From a4b16dad46576ce08ecb660fc923d0857dcae107 Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Tue, 28 Mar 2023 10:39:51 -0500 Subject: crypto: qat - Move driver to drivers/crypto/intel/qat With the growing number of Intel crypto drivers, it makes sense to group them all into a single drivers/crypto/intel/ directory. Signed-off-by: Tom Zanussi Signed-off-by: Herbert Xu --- drivers/crypto/qat/Kconfig | 97 - drivers/crypto/qat/Makefile | 9 - drivers/crypto/qat/qat_4xxx/Makefile | 4 - drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 417 ---- drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 82 - drivers/crypto/qat/qat_4xxx/adf_drv.c | 459 ----- drivers/crypto/qat/qat_c3xxx/Makefile | 4 - drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 139 -- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 28 - drivers/crypto/qat/qat_c3xxx/adf_drv.c | 258 --- drivers/crypto/qat/qat_c3xxxvf/Makefile | 4 - .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 102 - .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h | 18 - drivers/crypto/qat/qat_c3xxxvf/adf_drv.c | 232 --- drivers/crypto/qat/qat_c62x/Makefile | 4 - drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 141 -- drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h | 28 - drivers/crypto/qat/qat_c62x/adf_drv.c | 258 --- drivers/crypto/qat/qat_c62xvf/Makefile | 4 - drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 102 - drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h | 18 - drivers/crypto/qat/qat_c62xvf/adf_drv.c | 232 --- drivers/crypto/qat/qat_common/Makefile | 34 - drivers/crypto/qat/qat_common/adf_accel_devices.h | 319 --- drivers/crypto/qat/qat_common/adf_accel_engine.c | 212 -- drivers/crypto/qat/qat_common/adf_admin.c | 362 ---- drivers/crypto/qat/qat_common/adf_aer.c | 185 -- drivers/crypto/qat/qat_common/adf_cfg.c | 339 ---- drivers/crypto/qat/qat_common/adf_cfg.h | 43 - drivers/crypto/qat/qat_common/adf_cfg_common.h | 76 - drivers/crypto/qat/qat_common/adf_cfg_strings.h | 42 - drivers/crypto/qat/qat_common/adf_cfg_user.h | 38 - drivers/crypto/qat/qat_common/adf_common_drv.h | 249 --- drivers/crypto/qat/qat_common/adf_ctl_drv.c | 475 ----- drivers/crypto/qat/qat_common/adf_dev_mgr.c | 452 ----- drivers/crypto/qat/qat_common/adf_gen2_config.c | 206 -- drivers/crypto/qat/qat_common/adf_gen2_config.h | 10 - drivers/crypto/qat/qat_common/adf_gen2_dc.c | 70 - drivers/crypto/qat/qat_common/adf_gen2_dc.h | 10 - drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 268 --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 165 -- drivers/crypto/qat/qat_common/adf_gen2_pfvf.c | 399 ---- drivers/crypto/qat/qat_common/adf_gen2_pfvf.h | 29 - drivers/crypto/qat/qat_common/adf_gen4_dc.c | 83 - drivers/crypto/qat/qat_common/adf_gen4_dc.h | 10 - drivers/crypto/qat/qat_common/adf_gen4_hw_data.c | 194 -- drivers/crypto/qat/qat_common/adf_gen4_hw_data.h | 142 -- drivers/crypto/qat/qat_common/adf_gen4_pfvf.c | 147 -- drivers/crypto/qat/qat_common/adf_gen4_pfvf.h | 17 - drivers/crypto/qat/qat_common/adf_gen4_pm.c | 137 -- drivers/crypto/qat/qat_common/adf_gen4_pm.h | 44 - drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 105 - drivers/crypto/qat/qat_common/adf_init.c | 480 ----- drivers/crypto/qat/qat_common/adf_isr.c | 382 ---- drivers/crypto/qat/qat_common/adf_pfvf_msg.h | 259 --- drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c | 52 - drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h | 18 - drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c | 348 ---- drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h | 13 - drivers/crypto/qat/qat_common/adf_pfvf_utils.c | 65 - drivers/crypto/qat/qat_common/adf_pfvf_utils.h | 31 - drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c | 167 -- drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h | 23 - drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c | 368 ---- drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h | 17 - drivers/crypto/qat/qat_common/adf_sriov.c | 215 -- drivers/crypto/qat/qat_common/adf_sysfs.c | 174 -- drivers/crypto/qat/qat_common/adf_transport.c | 577 ------ drivers/crypto/qat/qat_common/adf_transport.h | 20 - .../qat/qat_common/adf_transport_access_macros.h | 58 - .../crypto/qat/qat_common/adf_transport_debug.c | 209 -- .../crypto/qat/qat_common/adf_transport_internal.h | 73 - drivers/crypto/qat/qat_common/adf_vf_isr.c | 313 --- drivers/crypto/qat/qat_common/icp_qat_fw.h | 298 --- drivers/crypto/qat/qat_common/icp_qat_fw_comp.h | 404 ---- .../crypto/qat/qat_common/icp_qat_fw_init_admin.h | 115 -- drivers/crypto/qat/qat_common/icp_qat_fw_la.h | 367 ---- .../qat/qat_common/icp_qat_fw_loader_handle.h | 68 - drivers/crypto/qat/qat_common/icp_qat_fw_pke.h | 68 - drivers/crypto/qat/qat_common/icp_qat_hal.h | 143 -- drivers/crypto/qat/qat_common/icp_qat_hw.h | 376 ---- drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h | 164 -- .../qat/qat_common/icp_qat_hw_20_comp_defs.h | 300 --- drivers/crypto/qat/qat_common/icp_qat_uclo.h | 585 ------ drivers/crypto/qat/qat_common/qat_algs.c | 1424 ------------- drivers/crypto/qat/qat_common/qat_algs_send.c | 87 - drivers/crypto/qat/qat_common/qat_algs_send.h | 25 - drivers/crypto/qat/qat_common/qat_asym_algs.c | 1309 ------------ drivers/crypto/qat/qat_common/qat_bl.c | 410 ---- drivers/crypto/qat/qat_common/qat_bl.h | 69 - drivers/crypto/qat/qat_common/qat_comp_algs.c | 489 ----- drivers/crypto/qat/qat_common/qat_comp_req.h | 123 -- drivers/crypto/qat/qat_common/qat_compression.c | 297 --- drivers/crypto/qat/qat_common/qat_compression.h | 37 - drivers/crypto/qat/qat_common/qat_crypto.c | 287 --- drivers/crypto/qat/qat_common/qat_crypto.h | 68 - drivers/crypto/qat/qat_common/qat_hal.c | 1594 --------------- drivers/crypto/qat/qat_common/qat_uclo.c | 2133 -------------------- drivers/crypto/qat/qat_dh895xcc/Makefile | 4 - .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 252 --- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 39 - drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 258 --- drivers/crypto/qat/qat_dh895xccvf/Makefile | 4 - .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 102 - .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h | 18 - drivers/crypto/qat/qat_dh895xccvf/adf_drv.c | 232 --- 106 files changed, 23513 deletions(-) delete mode 100644 drivers/crypto/qat/Kconfig delete mode 100644 drivers/crypto/qat/Makefile delete mode 100644 drivers/crypto/qat/qat_4xxx/Makefile delete mode 100644 drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c delete mode 100644 drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h delete mode 100644 drivers/crypto/qat/qat_4xxx/adf_drv.c delete mode 100644 drivers/crypto/qat/qat_c3xxx/Makefile delete mode 100644 drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c delete mode 100644 drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h delete mode 100644 drivers/crypto/qat/qat_c3xxx/adf_drv.c delete mode 100644 drivers/crypto/qat/qat_c3xxxvf/Makefile delete mode 100644 drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c delete mode 100644 drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h delete mode 100644 drivers/crypto/qat/qat_c3xxxvf/adf_drv.c delete mode 100644 drivers/crypto/qat/qat_c62x/Makefile delete mode 100644 drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c delete mode 100644 drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h delete mode 100644 drivers/crypto/qat/qat_c62x/adf_drv.c delete mode 100644 drivers/crypto/qat/qat_c62xvf/Makefile delete mode 100644 drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c delete mode 100644 drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h delete mode 100644 drivers/crypto/qat/qat_c62xvf/adf_drv.c delete mode 100644 drivers/crypto/qat/qat_common/Makefile delete mode 100644 drivers/crypto/qat/qat_common/adf_accel_devices.h delete mode 100644 drivers/crypto/qat/qat_common/adf_accel_engine.c delete mode 100644 drivers/crypto/qat/qat_common/adf_admin.c delete mode 100644 drivers/crypto/qat/qat_common/adf_aer.c delete mode 100644 drivers/crypto/qat/qat_common/adf_cfg.c delete mode 100644 drivers/crypto/qat/qat_common/adf_cfg.h delete mode 100644 drivers/crypto/qat/qat_common/adf_cfg_common.h delete mode 100644 drivers/crypto/qat/qat_common/adf_cfg_strings.h delete mode 100644 drivers/crypto/qat/qat_common/adf_cfg_user.h delete mode 100644 drivers/crypto/qat/qat_common/adf_common_drv.h delete mode 100644 drivers/crypto/qat/qat_common/adf_ctl_drv.c delete mode 100644 drivers/crypto/qat/qat_common/adf_dev_mgr.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_config.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_config.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_dc.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_dc.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_hw_data.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_hw_data.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_pfvf.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen2_pfvf.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_dc.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_dc.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_hw_data.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_hw_data.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_pfvf.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_pfvf.h delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_pm.c delete mode 100644 drivers/crypto/qat/qat_common/adf_gen4_pm.h delete mode 100644 drivers/crypto/qat/qat_common/adf_hw_arbiter.c delete mode 100644 drivers/crypto/qat/qat_common/adf_init.c delete mode 100644 drivers/crypto/qat/qat_common/adf_isr.c delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_msg.h delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_utils.c delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_utils.h delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c delete mode 100644 drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h delete mode 100644 drivers/crypto/qat/qat_common/adf_sriov.c delete mode 100644 drivers/crypto/qat/qat_common/adf_sysfs.c delete mode 100644 drivers/crypto/qat/qat_common/adf_transport.c delete mode 100644 drivers/crypto/qat/qat_common/adf_transport.h delete mode 100644 drivers/crypto/qat/qat_common/adf_transport_access_macros.h delete mode 100644 drivers/crypto/qat/qat_common/adf_transport_debug.c delete mode 100644 drivers/crypto/qat/qat_common/adf_transport_internal.h delete mode 100644 drivers/crypto/qat/qat_common/adf_vf_isr.c delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_fw.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_fw_comp.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_fw_la.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_fw_pke.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_hal.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_hw.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_hw_20_comp_defs.h delete mode 100644 drivers/crypto/qat/qat_common/icp_qat_uclo.h delete mode 100644 drivers/crypto/qat/qat_common/qat_algs.c delete mode 100644 drivers/crypto/qat/qat_common/qat_algs_send.c delete mode 100644 drivers/crypto/qat/qat_common/qat_algs_send.h delete mode 100644 drivers/crypto/qat/qat_common/qat_asym_algs.c delete mode 100644 drivers/crypto/qat/qat_common/qat_bl.c delete mode 100644 drivers/crypto/qat/qat_common/qat_bl.h delete mode 100644 drivers/crypto/qat/qat_common/qat_comp_algs.c delete mode 100644 drivers/crypto/qat/qat_common/qat_comp_req.h delete mode 100644 drivers/crypto/qat/qat_common/qat_compression.c delete mode 100644 drivers/crypto/qat/qat_common/qat_compression.h delete mode 100644 drivers/crypto/qat/qat_common/qat_crypto.c delete mode 100644 drivers/crypto/qat/qat_common/qat_crypto.h delete mode 100644 drivers/crypto/qat/qat_common/qat_hal.c delete mode 100644 drivers/crypto/qat/qat_common/qat_uclo.c delete mode 100644 drivers/crypto/qat/qat_dh895xcc/Makefile delete mode 100644 drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c delete mode 100644 drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h delete mode 100644 drivers/crypto/qat/qat_dh895xcc/adf_drv.c delete mode 100644 drivers/crypto/qat/qat_dh895xccvf/Makefile delete mode 100644 drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c delete mode 100644 drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h delete mode 100644 drivers/crypto/qat/qat_dh895xccvf/adf_drv.c (limited to 'drivers/crypto/qat') diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig deleted file mode 100644 index 1220cc86f910..000000000000 --- a/drivers/crypto/qat/Kconfig +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config CRYPTO_DEV_QAT - tristate - select CRYPTO_AEAD - select CRYPTO_AUTHENC - select CRYPTO_SKCIPHER - select CRYPTO_AKCIPHER - select CRYPTO_DH - select CRYPTO_HMAC - select CRYPTO_RSA - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 - select CRYPTO_LIB_AES - select FW_LOADER - select CRC8 - -config CRYPTO_DEV_QAT_DH895xCC - tristate "Support for Intel(R) DH895xCC" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select CRYPTO_DEV_QAT - help - Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology - for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_dh895xcc. - -config CRYPTO_DEV_QAT_C3XXX - tristate "Support for Intel(R) C3XXX" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select CRYPTO_DEV_QAT - help - Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology - for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_c3xxx. - -config CRYPTO_DEV_QAT_C62X - tristate "Support for Intel(R) C62X" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select CRYPTO_DEV_QAT - help - Support for Intel(R) C62x with Intel(R) QuickAssist Technology - for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_c62x. - -config CRYPTO_DEV_QAT_4XXX - tristate "Support for Intel(R) QAT_4XXX" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select CRYPTO_DEV_QAT - help - Support for Intel(R) QuickAssist Technology QAT_4xxx - for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_4xxx. - -config CRYPTO_DEV_QAT_DH895xCCVF - tristate "Support for Intel(R) DH895xCC Virtual Function" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select PCI_IOV - select CRYPTO_DEV_QAT - - help - Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology - Virtual Function for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_dh895xccvf. - -config CRYPTO_DEV_QAT_C3XXXVF - tristate "Support for Intel(R) C3XXX Virtual Function" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select PCI_IOV - select CRYPTO_DEV_QAT - help - Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology - Virtual Function for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_c3xxxvf. - -config CRYPTO_DEV_QAT_C62XVF - tristate "Support for Intel(R) C62X Virtual Function" - depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) - select PCI_IOV - select CRYPTO_DEV_QAT - help - Support for Intel(R) C62x with Intel(R) QuickAssist Technology - Virtual Function for accelerating crypto and compression workloads. - - To compile this as a module, choose M here: the module - will be called qat_c62xvf. diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile deleted file mode 100644 index 258c8a626ce0..000000000000 --- a/drivers/crypto/qat/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ -obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ -obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ -obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ -obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ -obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ -obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ -obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/qat/qat_4xxx/Makefile b/drivers/crypto/qat/qat_4xxx/Makefile deleted file mode 100644 index ff9c8b5897ea..000000000000 --- a/drivers/crypto/qat/qat_4xxx/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o -qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c deleted file mode 100644 index 7324b86a4f40..000000000000 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ /dev/null @@ -1,417 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2020 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_4xxx_hw_data.h" -#include "icp_qat_hw.h" - -struct adf_fw_config { - u32 ae_mask; - char *obj_name; -}; - -static struct adf_fw_config adf_4xxx_fw_cy_config[] = { - {0xF0, ADF_4XXX_SYM_OBJ}, - {0xF, ADF_4XXX_ASYM_OBJ}, - {0x100, ADF_4XXX_ADMIN_OBJ}, -}; - -static struct adf_fw_config adf_4xxx_fw_dc_config[] = { - {0xF0, ADF_4XXX_DC_OBJ}, - {0xF, ADF_4XXX_DC_OBJ}, - {0x100, ADF_4XXX_ADMIN_OBJ}, -}; - -static struct adf_fw_config adf_402xx_fw_cy_config[] = { - {0xF0, ADF_402XX_SYM_OBJ}, - {0xF, ADF_402XX_ASYM_OBJ}, - {0x100, ADF_402XX_ADMIN_OBJ}, -}; - -static struct adf_fw_config adf_402xx_fw_dc_config[] = { - {0xF0, ADF_402XX_DC_OBJ}, - {0xF, ADF_402XX_DC_OBJ}, - {0x100, ADF_402XX_ADMIN_OBJ}, -}; - -/* Worker thread to service arbiter mappings */ -static const u32 thrd_to_arb_map_cy[ADF_4XXX_MAX_ACCELENGINES] = { - 0x5555555, 0x5555555, 0x5555555, 0x5555555, - 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, - 0x0 -}; - -static const u32 thrd_to_arb_map_dc[ADF_4XXX_MAX_ACCELENGINES] = { - 0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF, - 0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF, - 0x0 -}; - -static struct adf_hw_device_class adf_4xxx_class = { - .name = ADF_4XXX_DEVICE_NAME, - .type = DEV_4XXX, - .instances = 0, -}; - -enum dev_services { - SVC_CY = 0, - SVC_DC, -}; - -static const char *const dev_cfg_services[] = { - [SVC_CY] = ADF_CFG_CY, - [SVC_DC] = ADF_CFG_DC, -}; - -static int get_service_enabled(struct adf_accel_dev *accel_dev) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - int ret; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - if (ret) { - dev_err(&GET_DEV(accel_dev), - ADF_SERVICES_ENABLED " param not found\n"); - return ret; - } - - ret = match_string(dev_cfg_services, ARRAY_SIZE(dev_cfg_services), - services); - if (ret < 0) - dev_err(&GET_DEV(accel_dev), - "Invalid value of " ADF_SERVICES_ENABLED " param: %s\n", - services); - - return ret; -} - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - return ADF_4XXX_ACCELERATORS_MASK; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - u32 me_disable = self->fuses; - - return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; -} - -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - return ADF_4XXX_MAX_ACCELERATORS; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - if (!self || !self->ae_mask) - return 0; - - return hweight32(self->ae_mask); -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_4XXX_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_4XXX_ETR_BAR; -} - -static u32 get_sram_bar_id(struct adf_hw_device_data *self) -{ - return ADF_4XXX_SRAM_BAR; -} - -/* - * The vector routing table is used to select the MSI-X entry to use for each - * interrupt source. - * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. - * The final entry corresponds to VF2PF or error interrupts. - * This vector table could be used to configure one MSI-X entry to be shared - * between multiple interrupt sources. - * - * The default routing is set to have a one to one correspondence between the - * interrupt source and the MSI-X entry used. - */ -static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) -{ - void __iomem *csr; - int i; - - csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; - for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) - ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); -} - -static u32 get_accel_cap(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; - u32 capabilities_cy, capabilities_dc; - u32 fusectl1; - - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); - - capabilities_cy = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | - ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | - ICP_ACCEL_CAPABILITIES_CIPHER | - ICP_ACCEL_CAPABILITIES_AUTHENTICATION | - ICP_ACCEL_CAPABILITIES_SHA3 | - ICP_ACCEL_CAPABILITIES_SHA3_EXT | - ICP_ACCEL_CAPABILITIES_HKDF | - ICP_ACCEL_CAPABILITIES_ECEDMONT | - ICP_ACCEL_CAPABILITIES_CHACHA_POLY | - ICP_ACCEL_CAPABILITIES_AESGCM_SPC | - ICP_ACCEL_CAPABILITIES_AES_V2; - - /* A set bit in fusectl1 means the feature is OFF in this SKU */ - if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) { - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_HKDF; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) { - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AES_V2; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) { - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) { - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; - capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; - } - - capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | - ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | - ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | - ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; - - if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) { - capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; - capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; - capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; - capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; - } - - switch (get_service_enabled(accel_dev)) { - case SVC_CY: - return capabilities_cy; - case SVC_DC: - return capabilities_dc; - } - - return 0; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - return DEV_SKU_1; -} - -static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) -{ - switch (get_service_enabled(accel_dev)) { - case SVC_CY: - return thrd_to_arb_map_cy; - case SVC_DC: - return thrd_to_arb_map_dc; - } - - return NULL; -} - -static void get_arb_info(struct arb_info *arb_info) -{ - arb_info->arb_cfg = ADF_4XXX_ARB_CONFIG; - arb_info->arb_offset = ADF_4XXX_ARB_OFFSET; - arb_info->wt2sam_offset = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; -} - -static void get_admin_info(struct admin_info *admin_csrs_info) -{ - admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; - admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; - admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; -} - -static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) -{ - struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; - void __iomem *csr = misc_bar->virt_addr; - - /* Enable all in errsou3 except VFLR notification on host */ - ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_VFLNOTIFY); -} - -static void adf_enable_ints(struct adf_accel_dev *accel_dev) -{ - void __iomem *addr; - - addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; - - /* Enable bundle interrupts */ - ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); - ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); - - /* Enable misc interrupts */ - ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); -} - -static int adf_init_device(struct adf_accel_dev *accel_dev) -{ - void __iomem *addr; - u32 status; - u32 csr; - int ret; - - addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; - - /* Temporarily mask PM interrupt */ - csr = ADF_CSR_RD(addr, ADF_GEN4_ERRMSK2); - csr |= ADF_GEN4_PM_SOU; - ADF_CSR_WR(addr, ADF_GEN4_ERRMSK2, csr); - - /* Set DRV_ACTIVE bit to power up the device */ - ADF_CSR_WR(addr, ADF_GEN4_PM_INTERRUPT, ADF_GEN4_PM_DRV_ACTIVE); - - /* Poll status register to make sure the device is powered up */ - ret = read_poll_timeout(ADF_CSR_RD, status, - status & ADF_GEN4_PM_INIT_STATE, - ADF_GEN4_PM_POLL_DELAY_US, - ADF_GEN4_PM_POLL_TIMEOUT_US, true, addr, - ADF_GEN4_PM_STATUS); - if (ret) - dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); - - return ret; -} - -static u32 uof_get_num_objs(void) -{ - BUILD_BUG_ON_MSG(ARRAY_SIZE(adf_4xxx_fw_cy_config) != - ARRAY_SIZE(adf_4xxx_fw_dc_config), - "Size mismatch between adf_4xxx_fw_*_config arrays"); - - return ARRAY_SIZE(adf_4xxx_fw_cy_config); -} - -static char *uof_get_name_4xxx(struct adf_accel_dev *accel_dev, u32 obj_num) -{ - switch (get_service_enabled(accel_dev)) { - case SVC_CY: - return adf_4xxx_fw_cy_config[obj_num].obj_name; - case SVC_DC: - return adf_4xxx_fw_dc_config[obj_num].obj_name; - } - - return NULL; -} - -static char *uof_get_name_402xx(struct adf_accel_dev *accel_dev, u32 obj_num) -{ - switch (get_service_enabled(accel_dev)) { - case SVC_CY: - return adf_402xx_fw_cy_config[obj_num].obj_name; - case SVC_DC: - return adf_402xx_fw_dc_config[obj_num].obj_name; - } - - return NULL; -} - -static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) -{ - switch (get_service_enabled(accel_dev)) { - case SVC_CY: - return adf_4xxx_fw_cy_config[obj_num].ae_mask; - case SVC_DC: - return adf_4xxx_fw_dc_config[obj_num].ae_mask; - } - - return 0; -} - -void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) -{ - hw_data->dev_class = &adf_4xxx_class; - hw_data->instance_id = adf_4xxx_class.instances++; - hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; - hw_data->num_banks_per_vf = ADF_4XXX_NUM_BANKS_PER_VF; - hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; - hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; - hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; - hw_data->num_logical_accel = 1; - hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_isr_resource_alloc; - hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_enable_error_correction; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; - hw_data->get_sram_bar_id = get_sram_bar_id; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_arb_info = get_arb_info; - hw_data->get_admin_info = get_admin_info; - hw_data->get_accel_cap = get_accel_cap; - hw_data->get_sku = get_sku; - hw_data->init_admin_comms = adf_init_admin_comms; - hw_data->exit_admin_comms = adf_exit_admin_comms; - hw_data->send_admin_init = adf_send_admin_init; - hw_data->init_arb = adf_init_arb; - hw_data->exit_arb = adf_exit_arb; - hw_data->get_arb_mapping = adf_get_arbiter_mapping; - hw_data->enable_ints = adf_enable_ints; - hw_data->init_device = adf_init_device; - hw_data->reset_device = adf_reset_flr; - hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; - switch (dev_id) { - case ADF_402XX_PCI_DEVICE_ID: - hw_data->fw_name = ADF_402XX_FW; - hw_data->fw_mmp_name = ADF_402XX_MMP; - hw_data->uof_get_name = uof_get_name_402xx; - break; - - default: - hw_data->fw_name = ADF_4XXX_FW; - hw_data->fw_mmp_name = ADF_4XXX_MMP; - hw_data->uof_get_name = uof_get_name_4xxx; - } - hw_data->uof_get_num_objs = uof_get_num_objs; - hw_data->uof_get_ae_mask = uof_get_ae_mask; - hw_data->set_msix_rttable = set_msix_default_rttable; - hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; - hw_data->disable_iov = adf_disable_sriov; - hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; - hw_data->enable_pm = adf_gen4_enable_pm; - hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; - hw_data->dev_config = adf_gen4_dev_config; - - adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); - adf_gen4_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; -} diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h deleted file mode 100644 index 085e259c245a..000000000000 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ /dev/null @@ -1,82 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_4XXX_HW_DATA_H_ -#define ADF_4XXX_HW_DATA_H_ - -#include - -/* PCIe configuration space */ -#define ADF_4XXX_SRAM_BAR 0 -#define ADF_4XXX_PMISC_BAR 1 -#define ADF_4XXX_ETR_BAR 2 -#define ADF_4XXX_RX_RINGS_OFFSET 1 -#define ADF_4XXX_TX_RINGS_MASK 0x1 -#define ADF_4XXX_MAX_ACCELERATORS 1 -#define ADF_4XXX_MAX_ACCELENGINES 9 -#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) - -/* Physical function fuses */ -#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) -#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) -#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) -#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) -#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) -#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) - -#define ADF_4XXX_ACCELERATORS_MASK (0x1) -#define ADF_4XXX_ACCELENGINES_MASK (0x1FF) -#define ADF_4XXX_ADMIN_AE_MASK (0x100) - -#define ADF_4XXX_ETR_MAX_BANKS 64 - -/* MSIX interrupt */ -#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) -#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) -#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) -#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) - -/* Bank and ring configuration */ -#define ADF_4XXX_NUM_RINGS_PER_BANK 2 -#define ADF_4XXX_NUM_BANKS_PER_VF 4 - -/* Arbiter configuration */ -#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) -#define ADF_4XXX_ARB_OFFSET (0x0) -#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) - -/* Admin Interface Reg Offset */ -#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) -#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) -#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) - -/* Firmware Binaries */ -#define ADF_4XXX_FW "qat_4xxx.bin" -#define ADF_4XXX_MMP "qat_4xxx_mmp.bin" -#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" -#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin" -#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" -#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" -/* Firmware for 402XXX */ -#define ADF_402XX_FW "qat_402xx.bin" -#define ADF_402XX_MMP "qat_402xx_mmp.bin" -#define ADF_402XX_SYM_OBJ "qat_402xx_sym.bin" -#define ADF_402XX_DC_OBJ "qat_402xx_dc.bin" -#define ADF_402XX_ASYM_OBJ "qat_402xx_asym.bin" -#define ADF_402XX_ADMIN_OBJ "qat_402xx_admin.bin" - -/* qat_4xxx fuse bits are different from old GENs, redefine them */ -enum icp_qat_4xxx_slice_mask { - ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), - ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), - ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), - ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), - ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), - ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), - ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6), -}; - -void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id); -void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); -int adf_gen4_dev_config(struct adf_accel_dev *accel_dev); - -#endif diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c deleted file mode 100644 index ceb87327a5fe..000000000000 --- a/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ /dev/null @@ -1,459 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2020 Intel Corporation */ -#include -#include -#include - -#include -#include -#include - -#include "adf_4xxx_hw_data.h" -#include "qat_compression.h" -#include "qat_crypto.h" -#include "adf_transport_access_macros.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), }, - { PCI_VDEVICE(INTEL, ADF_402XX_PCI_DEVICE_ID), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -enum configs { - DEV_CFG_CY = 0, - DEV_CFG_DC, -}; - -static const char * const services_operations[] = { - ADF_CFG_CY, - ADF_CFG_DC, -}; - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - if (accel_dev->hw_device) { - adf_clean_hw_data_4xxx(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev, NULL); -} - -static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev) -{ - const char *config; - int ret; - - config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; - - ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); - if (ret) - return ret; - - /* Default configuration is crypto only for even devices - * and compression for odd devices - */ - ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, config, - ADF_STR); - if (ret) - return ret; - - return 0; -} - -static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int banks = GET_MAX_BANKS(accel_dev); - int cpus = num_online_cpus(); - unsigned long bank, val; - int instances; - int ret; - int i; - - if (adf_hw_dev_has_crypto(accel_dev)) - instances = min(cpus, banks / 2); - else - instances = 0; - - for (i = 0; i < instances; i++) { - val = i; - bank = i * 2; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &bank, ADF_DEC); - if (ret) - goto err; - - bank += 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &bank, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, - i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - val = 128; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC); - if (ret) - goto err; - } - - val = i; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, - &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - goto err; - - return 0; -err: - dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); - return ret; -} - -static int adf_comp_dev_config(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int banks = GET_MAX_BANKS(accel_dev); - int cpus = num_online_cpus(); - unsigned long val; - int instances; - int ret; - int i; - - if (adf_hw_dev_has_compression(accel_dev)) - instances = min(cpus, banks); - else - instances = 0; - - for (i = 0; i < instances; i++) { - val = i; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 1; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC); - if (ret) - goto err; - } - - val = i; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, - &val, ADF_DEC); - if (ret) - goto err; - - return 0; -err: - dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); - return ret; -} - -int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - int ret; - - ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); - if (ret) - goto err; - - ret = adf_cfg_section_add(accel_dev, "Accelerator0"); - if (ret) - goto err; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - if (ret) - goto err; - - ret = sysfs_match_string(services_operations, services); - if (ret < 0) - goto err; - - switch (ret) { - case DEV_CFG_CY: - ret = adf_crypto_dev_config(accel_dev); - break; - case DEV_CFG_DC: - ret = adf_comp_dev_config(accel_dev); - break; - } - - if (ret) - goto err; - - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - - return ret; - -err: - dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); - return ret; -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - struct adf_bar *bar; - int ret; - - if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { - /* - * If the accelerator is connected to a node with no memory - * there is no point in using the accelerator since the remote - * memory transaction will be very slow. - */ - dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); - return -EINVAL; - } - - accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); - if (!accel_dev) - return -ENOMEM; - - INIT_LIST_HEAD(&accel_dev->crypto_list); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* - * Add accel device to accel table - * This should be called before adf_cleanup_accel is called - */ - if (adf_devmgr_add_dev(accel_dev, NULL)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - return -EFAULT; - } - - accel_dev->owner = THIS_MODULE; - /* Allocate and initialise device hardware meta-data structure */ - hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - - accel_dev->hw_device = hw_data; - adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); - - pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - /* If the device has no acceleration engines then ignore it */ - if (!hw_data->accel_mask || !hw_data->ae_mask || - (~hw_data->ae_mask & 0x01)) { - dev_err(&pdev->dev, "No acceleration units found.\n"); - ret = -EFAULT; - goto out_err; - } - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - /* Enable PCI device */ - ret = pcim_enable_device(pdev); - if (ret) { - dev_err(&pdev->dev, "Can't enable PCI device.\n"); - goto out_err; - } - - /* Set DMA identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration.\n"); - goto out_err; - } - - ret = adf_cfg_dev_init(accel_dev); - if (ret) { - dev_err(&pdev->dev, "Failed to initialize configuration.\n"); - goto out_err; - } - - /* Get accelerator capabilities mask */ - hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); - if (!hw_data->accel_capabilities_mask) { - dev_err(&pdev->dev, "Failed to get capabilities mask.\n"); - ret = -EINVAL; - goto out_err; - } - - /* Find and map all the device's BARS */ - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; - - ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); - if (ret) { - dev_err(&pdev->dev, "Failed to map pci regions.\n"); - goto out_err; - } - - i = 0; - for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { - bar = &accel_pci_dev->pci_bars[i++]; - bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; - } - - pci_set_master(pdev); - - if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state.\n"); - ret = -ENOMEM; - goto out_err; - } - - ret = adf_dev_up(accel_dev, true); - if (ret) - goto out_err_dev_stop; - - ret = adf_sysfs_init(accel_dev); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err: - adf_cleanup_accel(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); -} - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_4XXX_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - -module_pci_driver(adf_driver); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_FIRMWARE(ADF_4XXX_FW); -MODULE_FIRMWARE(ADF_4XXX_MMP); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); -MODULE_SOFTDEP("pre: crypto-intel_qat"); diff --git a/drivers/crypto/qat/qat_c3xxx/Makefile b/drivers/crypto/qat/qat_c3xxx/Makefile deleted file mode 100644 index 92ef416ccc78..000000000000 --- a/drivers/crypto/qat/qat_c3xxx/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o -qat_c3xxx-objs := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c deleted file mode 100644 index 475643654e64..000000000000 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include "adf_c3xxx_hw_data.h" -#include "icp_qat_hw.h" - -/* Worker thread to service arbiter mappings */ -static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { - 0x12222AAA, 0x11222AAA, 0x12222AAA, - 0x11222AAA, 0x12222AAA, 0x11222AAA -}; - -static struct adf_hw_device_class c3xxx_class = { - .name = ADF_C3XXX_DEVICE_NAME, - .type = DEV_C3XXX, - .instances = 0 -}; - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - u32 straps = self->straps; - u32 fuses = self->fuses; - u32 accel; - - accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; - accel &= ADF_C3XXX_ACCELERATORS_MASK; - - return accel; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - u32 straps = self->straps; - u32 fuses = self->fuses; - unsigned long disabled; - u32 ae_disable; - int accel; - - /* If an accel is disabled, then disable the corresponding two AEs */ - disabled = ~get_accel_mask(self) & ADF_C3XXX_ACCELERATORS_MASK; - ae_disable = BIT(1) | BIT(0); - for_each_set_bit(accel, &disabled, ADF_C3XXX_MAX_ACCELERATORS) - straps |= ae_disable << (accel << 1); - - return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C3XXX_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C3XXX_ETR_BAR; -} - -static u32 get_sram_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C3XXX_SRAM_BAR; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - int aes = self->get_num_aes(self); - - if (aes == 6) - return DEV_SKU_4; - - return DEV_SKU_UNKNOWN; -} - -static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) -{ - return thrd_to_arb_map; -} - -static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) -{ - adf_gen2_cfg_iov_thds(accel_dev, enable, - ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS, - ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS); -} - -void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class = &c3xxx_class; - hw_data->instance_id = c3xxx_class.instances++; - hw_data->num_banks = ADF_C3XXX_ETR_MAX_BANKS; - hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; - hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; - hw_data->num_logical_accel = 1; - hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_isr_resource_alloc; - hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_gen2_enable_error_correction; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_accel_cap = adf_gen2_get_accel_cap; - hw_data->get_num_accels = adf_gen2_get_num_accels; - hw_data->get_num_aes = adf_gen2_get_num_aes; - hw_data->get_sram_bar_id = get_sram_bar_id; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_admin_info = adf_gen2_get_admin_info; - hw_data->get_arb_info = adf_gen2_get_arb_info; - hw_data->get_sku = get_sku; - hw_data->fw_name = ADF_C3XXX_FW; - hw_data->fw_mmp_name = ADF_C3XXX_MMP; - hw_data->init_admin_comms = adf_init_admin_comms; - hw_data->exit_admin_comms = adf_exit_admin_comms; - hw_data->configure_iov_threads = configure_iov_threads; - hw_data->send_admin_init = adf_send_admin_init; - hw_data->init_arb = adf_init_arb; - hw_data->exit_arb = adf_exit_arb; - hw_data->get_arb_mapping = adf_get_arbiter_mapping; - hw_data->enable_ints = adf_gen2_enable_ints; - hw_data->reset_device = adf_reset_flr; - hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->disable_iov = adf_disable_sriov; - hw_data->dev_config = adf_gen2_dev_config; - - adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); - adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen2_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; -} diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h deleted file mode 100644 index 336a06f11dbd..000000000000 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_C3XXX_HW_DATA_H_ -#define ADF_C3XXX_HW_DATA_H_ - -/* PCIe configuration space */ -#define ADF_C3XXX_PMISC_BAR 0 -#define ADF_C3XXX_ETR_BAR 1 -#define ADF_C3XXX_SRAM_BAR 0 -#define ADF_C3XXX_MAX_ACCELERATORS 3 -#define ADF_C3XXX_MAX_ACCELENGINES 6 -#define ADF_C3XXX_ACCELERATORS_REG_OFFSET 16 -#define ADF_C3XXX_ACCELERATORS_MASK 0x7 -#define ADF_C3XXX_ACCELENGINES_MASK 0x3F -#define ADF_C3XXX_ETR_MAX_BANKS 16 -#define ADF_C3XXX_SOFTSTRAP_CSR_OFFSET 0x2EC - -/* AE to function mapping */ -#define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 -#define ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS 6 - -/* Firmware Binary */ -#define ADF_C3XXX_FW "qat_c3xxx.bin" -#define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" - -void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data); -#endif diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c deleted file mode 100644 index bb4dca735ab5..000000000000 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_c3xxx_hw_data.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C3XXX_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - -static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) -{ - pci_release_regions(accel_dev->accel_pci_dev.pci_dev); - pci_disable_device(accel_dev->accel_pci_dev.pci_dev); -} - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; - int i; - - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; - - if (bar->virt_addr) - pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); - } - - if (accel_dev->hw_device) { - switch (accel_pci_dev->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_C3XXX: - adf_clean_hw_data_c3xxx(accel_dev->hw_device); - break; - default: - break; - } - kfree(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev, NULL); -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - int ret; - - switch (ent->device) { - case PCI_DEVICE_ID_INTEL_QAT_C3XXX: - break; - default: - dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); - return -ENODEV; - } - - if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { - /* If the accelerator is connected to a node with no memory - * there is no point in using the accelerator since the remote - * memory transaction will be very slow. */ - dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); - return -EINVAL; - } - - accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!accel_dev) - return -ENOMEM; - - INIT_LIST_HEAD(&accel_dev->crypto_list); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* Add accel device to accel table. - * This should be called before adf_cleanup_accel is called */ - if (adf_devmgr_add_dev(accel_dev, NULL)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - kfree(accel_dev); - return -EFAULT; - } - - accel_dev->owner = THIS_MODULE; - /* Allocate and configure device configuration structure */ - hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - - accel_dev->hw_device = hw_data; - adf_init_hw_data_c3xxx(accel_dev->hw_device); - pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); - pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, - &hw_data->straps); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - /* If the device has no acceleration engines then ignore it. */ - if (!hw_data->accel_mask || !hw_data->ae_mask || - ((~hw_data->ae_mask) & 0x01)) { - dev_err(&pdev->dev, "No acceleration units found"); - ret = -EFAULT; - goto out_err; - } - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - /* enable PCI device */ - if (pci_enable_device(pdev)) { - ret = -EFAULT; - goto out_err; - } - - /* set dma identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration\n"); - goto out_err_disable; - } - - if (pci_request_regions(pdev, ADF_C3XXX_DEVICE_NAME)) { - ret = -EFAULT; - goto out_err_disable; - } - - /* Get accelerator capabilities mask */ - hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); - - /* Find and map all the device's BARS */ - i = 0; - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - - bar->base_addr = pci_resource_start(pdev, bar_nr); - if (!bar->base_addr) - break; - bar->size = pci_resource_len(pdev, bar_nr); - bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); - if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); - ret = -EFAULT; - goto out_err_free_reg; - } - } - pci_set_master(pdev); - - if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state\n"); - ret = -ENOMEM; - goto out_err_free_reg; - } - - ret = adf_dev_up(accel_dev, true); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err_free_reg: - pci_release_regions(accel_pci_dev->pci_dev); -out_err_disable: - pci_disable_device(accel_pci_dev->pci_dev); -out_err: - adf_cleanup_accel(accel_dev); - kfree(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); - adf_cleanup_pci_dev(accel_dev); - kfree(accel_dev); -} - -static int __init adfdrv_init(void) -{ - request_module("intel_qat"); - - if (pci_register_driver(&adf_driver)) { - pr_err("QAT: Driver initialization failed\n"); - return -EFAULT; - } - return 0; -} - -static void __exit adfdrv_release(void) -{ - pci_unregister_driver(&adf_driver); -} - -module_init(adfdrv_init); -module_exit(adfdrv_release); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_FIRMWARE(ADF_C3XXX_FW); -MODULE_FIRMWARE(ADF_C3XXX_MMP); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_c3xxxvf/Makefile b/drivers/crypto/qat/qat_c3xxxvf/Makefile deleted file mode 100644 index b6d76825a92c..000000000000 --- a/drivers/crypto/qat/qat_c3xxxvf/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o -qat_c3xxxvf-objs := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c deleted file mode 100644 index 84d9486e04de..000000000000 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include "adf_c3xxxvf_hw_data.h" - -static struct adf_hw_device_class c3xxxiov_class = { - .name = ADF_C3XXXVF_DEVICE_NAME, - .type = DEV_C3XXXVF, - .instances = 0 -}; - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - return ADF_C3XXXIOV_ACCELERATORS_MASK; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - return ADF_C3XXXIOV_ACCELENGINES_MASK; -} - -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - return ADF_C3XXXIOV_MAX_ACCELERATORS; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - return ADF_C3XXXIOV_MAX_ACCELENGINES; -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C3XXXIOV_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C3XXXIOV_ETR_BAR; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - return DEV_SKU_VF; -} - -static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) -{ - return 0; -} - -static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) -{ -} - -void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class = &c3xxxiov_class; - hw_data->num_banks = ADF_C3XXXIOV_ETR_MAX_BANKS; - hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; - hw_data->num_accel = ADF_C3XXXIOV_MAX_ACCELERATORS; - hw_data->num_logical_accel = 1; - hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_vf_isr_resource_alloc; - hw_data->free_irq = adf_vf_isr_resource_free; - hw_data->enable_error_correction = adf_vf_void_noop; - hw_data->init_admin_comms = adf_vf_int_noop; - hw_data->exit_admin_comms = adf_vf_void_noop; - hw_data->send_admin_init = adf_vf2pf_notify_init; - hw_data->init_arb = adf_vf_int_noop; - hw_data->exit_arb = adf_vf_void_noop; - hw_data->disable_iov = adf_vf2pf_notify_shutdown; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_sku = get_sku; - hw_data->enable_ints = adf_vf_void_noop; - hw_data->dev_class->instances++; - hw_data->dev_config = adf_gen2_dev_config; - adf_devmgr_update_class_index(hw_data); - adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); - adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen2_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; - adf_devmgr_update_class_index(hw_data); -} diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h deleted file mode 100644 index 6b4bf181d15b..000000000000 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2015 - 2020 Intel Corporation */ -#ifndef ADF_C3XXXVF_HW_DATA_H_ -#define ADF_C3XXXVF_HW_DATA_H_ - -#define ADF_C3XXXIOV_PMISC_BAR 1 -#define ADF_C3XXXIOV_ACCELERATORS_MASK 0x1 -#define ADF_C3XXXIOV_ACCELENGINES_MASK 0x1 -#define ADF_C3XXXIOV_MAX_ACCELERATORS 1 -#define ADF_C3XXXIOV_MAX_ACCELENGINES 1 -#define ADF_C3XXXIOV_RX_RINGS_OFFSET 8 -#define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF -#define ADF_C3XXXIOV_ETR_BAR 0 -#define ADF_C3XXXIOV_ETR_MAX_BANKS 1 - -void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); -#endif diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c deleted file mode 100644 index e8cc10f64134..000000000000 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_c3xxxvf_hw_data.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C3XXXVF_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, -}; - -static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) -{ - pci_release_regions(accel_dev->accel_pci_dev.pci_dev); - pci_disable_device(accel_dev->accel_pci_dev.pci_dev); -} - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; - struct adf_accel_dev *pf; - int i; - - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; - - if (bar->virt_addr) - pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); - } - - if (accel_dev->hw_device) { - switch (accel_pci_dev->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: - adf_clean_hw_data_c3xxxiov(accel_dev->hw_device); - break; - default: - break; - } - kfree(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); - adf_devmgr_rm_dev(accel_dev, pf); -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_dev *pf; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - int ret; - - switch (ent->device) { - case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: - break; - default: - dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); - return -ENODEV; - } - - accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!accel_dev) - return -ENOMEM; - - accel_dev->is_vf = true; - pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* Add accel device to accel table */ - if (adf_devmgr_add_dev(accel_dev, pf)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - kfree(accel_dev); - return -EFAULT; - } - INIT_LIST_HEAD(&accel_dev->crypto_list); - - accel_dev->owner = THIS_MODULE; - /* Allocate and configure device configuration structure */ - hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - accel_dev->hw_device = hw_data; - adf_init_hw_data_c3xxxiov(accel_dev->hw_device); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - /* enable PCI device */ - if (pci_enable_device(pdev)) { - ret = -EFAULT; - goto out_err; - } - - /* set dma identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration\n"); - goto out_err_disable; - } - - if (pci_request_regions(pdev, ADF_C3XXXVF_DEVICE_NAME)) { - ret = -EFAULT; - goto out_err_disable; - } - - /* Find and map all the device's BARS */ - i = 0; - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - - bar->base_addr = pci_resource_start(pdev, bar_nr); - if (!bar->base_addr) - break; - bar->size = pci_resource_len(pdev, bar_nr); - bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); - if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); - ret = -EFAULT; - goto out_err_free_reg; - } - } - pci_set_master(pdev); - /* Completion for VF2PF request/response message exchange */ - init_completion(&accel_dev->vf.msg_received); - - ret = adf_dev_up(accel_dev, false); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err_free_reg: - pci_release_regions(accel_pci_dev->pci_dev); -out_err_disable: - pci_disable_device(accel_pci_dev->pci_dev); -out_err: - adf_cleanup_accel(accel_dev); - kfree(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_flush_vf_wq(accel_dev); - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); - adf_cleanup_pci_dev(accel_dev); - kfree(accel_dev); -} - -static int __init adfdrv_init(void) -{ - request_module("intel_qat"); - - if (pci_register_driver(&adf_driver)) { - pr_err("QAT: Driver initialization failed\n"); - return -EFAULT; - } - return 0; -} - -static void __exit adfdrv_release(void) -{ - pci_unregister_driver(&adf_driver); - adf_clean_vf_map(true); -} - -module_init(adfdrv_init); -module_exit(adfdrv_release); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_c62x/Makefile b/drivers/crypto/qat/qat_c62x/Makefile deleted file mode 100644 index d581f7c87d6c..000000000000 --- a/drivers/crypto/qat/qat_c62x/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o -qat_c62x-objs := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c deleted file mode 100644 index e14270703670..000000000000 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include "adf_c62x_hw_data.h" -#include "icp_qat_hw.h" - -/* Worker thread to service arbiter mappings */ -static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { - 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, - 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA -}; - -static struct adf_hw_device_class c62x_class = { - .name = ADF_C62X_DEVICE_NAME, - .type = DEV_C62X, - .instances = 0 -}; - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - u32 straps = self->straps; - u32 fuses = self->fuses; - u32 accel; - - accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; - accel &= ADF_C62X_ACCELERATORS_MASK; - - return accel; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - u32 straps = self->straps; - u32 fuses = self->fuses; - unsigned long disabled; - u32 ae_disable; - int accel; - - /* If an accel is disabled, then disable the corresponding two AEs */ - disabled = ~get_accel_mask(self) & ADF_C62X_ACCELERATORS_MASK; - ae_disable = BIT(1) | BIT(0); - for_each_set_bit(accel, &disabled, ADF_C62X_MAX_ACCELERATORS) - straps |= ae_disable << (accel << 1); - - return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C62X_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C62X_ETR_BAR; -} - -static u32 get_sram_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C62X_SRAM_BAR; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - int aes = self->get_num_aes(self); - - if (aes == 8) - return DEV_SKU_2; - else if (aes == 10) - return DEV_SKU_4; - - return DEV_SKU_UNKNOWN; -} - -static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) -{ - return thrd_to_arb_map; -} - -static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) -{ - adf_gen2_cfg_iov_thds(accel_dev, enable, - ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS, - ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS); -} - -void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class = &c62x_class; - hw_data->instance_id = c62x_class.instances++; - hw_data->num_banks = ADF_C62X_ETR_MAX_BANKS; - hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; - hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; - hw_data->num_logical_accel = 1; - hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_isr_resource_alloc; - hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_gen2_enable_error_correction; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_accel_cap = adf_gen2_get_accel_cap; - hw_data->get_num_accels = adf_gen2_get_num_accels; - hw_data->get_num_aes = adf_gen2_get_num_aes; - hw_data->get_sram_bar_id = get_sram_bar_id; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_admin_info = adf_gen2_get_admin_info; - hw_data->get_arb_info = adf_gen2_get_arb_info; - hw_data->get_sku = get_sku; - hw_data->fw_name = ADF_C62X_FW; - hw_data->fw_mmp_name = ADF_C62X_MMP; - hw_data->init_admin_comms = adf_init_admin_comms; - hw_data->exit_admin_comms = adf_exit_admin_comms; - hw_data->configure_iov_threads = configure_iov_threads; - hw_data->send_admin_init = adf_send_admin_init; - hw_data->init_arb = adf_init_arb; - hw_data->exit_arb = adf_exit_arb; - hw_data->get_arb_mapping = adf_get_arbiter_mapping; - hw_data->enable_ints = adf_gen2_enable_ints; - hw_data->reset_device = adf_reset_flr; - hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->disable_iov = adf_disable_sriov; - hw_data->dev_config = adf_gen2_dev_config; - - adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); - adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen2_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; -} diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h deleted file mode 100644 index 008c0a3a9769..000000000000 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_C62X_HW_DATA_H_ -#define ADF_C62X_HW_DATA_H_ - -/* PCIe configuration space */ -#define ADF_C62X_SRAM_BAR 0 -#define ADF_C62X_PMISC_BAR 1 -#define ADF_C62X_ETR_BAR 2 -#define ADF_C62X_MAX_ACCELERATORS 5 -#define ADF_C62X_MAX_ACCELENGINES 10 -#define ADF_C62X_ACCELERATORS_REG_OFFSET 16 -#define ADF_C62X_ACCELERATORS_MASK 0x1F -#define ADF_C62X_ACCELENGINES_MASK 0x3FF -#define ADF_C62X_ETR_MAX_BANKS 16 -#define ADF_C62X_SOFTSTRAP_CSR_OFFSET 0x2EC - -/* AE to function mapping */ -#define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 -#define ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS 10 - -/* Firmware Binary */ -#define ADF_C62X_FW "qat_c62x.bin" -#define ADF_C62X_MMP "qat_c62x_mmp.bin" - -void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data); -#endif diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c deleted file mode 100644 index ca18ae14c099..000000000000 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_c62x_hw_data.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C62X_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - -static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) -{ - pci_release_regions(accel_dev->accel_pci_dev.pci_dev); - pci_disable_device(accel_dev->accel_pci_dev.pci_dev); -} - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; - int i; - - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; - - if (bar->virt_addr) - pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); - } - - if (accel_dev->hw_device) { - switch (accel_pci_dev->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_C62X: - adf_clean_hw_data_c62x(accel_dev->hw_device); - break; - default: - break; - } - kfree(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev, NULL); -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - int ret; - - switch (ent->device) { - case PCI_DEVICE_ID_INTEL_QAT_C62X: - break; - default: - dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); - return -ENODEV; - } - - if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { - /* If the accelerator is connected to a node with no memory - * there is no point in using the accelerator since the remote - * memory transaction will be very slow. */ - dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); - return -EINVAL; - } - - accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!accel_dev) - return -ENOMEM; - - INIT_LIST_HEAD(&accel_dev->crypto_list); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* Add accel device to accel table. - * This should be called before adf_cleanup_accel is called */ - if (adf_devmgr_add_dev(accel_dev, NULL)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - kfree(accel_dev); - return -EFAULT; - } - - accel_dev->owner = THIS_MODULE; - /* Allocate and configure device configuration structure */ - hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - - accel_dev->hw_device = hw_data; - adf_init_hw_data_c62x(accel_dev->hw_device); - pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); - pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, - &hw_data->straps); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - /* If the device has no acceleration engines then ignore it. */ - if (!hw_data->accel_mask || !hw_data->ae_mask || - ((~hw_data->ae_mask) & 0x01)) { - dev_err(&pdev->dev, "No acceleration units found"); - ret = -EFAULT; - goto out_err; - } - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - /* enable PCI device */ - if (pci_enable_device(pdev)) { - ret = -EFAULT; - goto out_err; - } - - /* set dma identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration\n"); - goto out_err_disable; - } - - if (pci_request_regions(pdev, ADF_C62X_DEVICE_NAME)) { - ret = -EFAULT; - goto out_err_disable; - } - - /* Get accelerator capabilities mask */ - hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); - - /* Find and map all the device's BARS */ - i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - - bar->base_addr = pci_resource_start(pdev, bar_nr); - if (!bar->base_addr) - break; - bar->size = pci_resource_len(pdev, bar_nr); - bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); - if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); - ret = -EFAULT; - goto out_err_free_reg; - } - } - pci_set_master(pdev); - - if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state\n"); - ret = -ENOMEM; - goto out_err_free_reg; - } - - ret = adf_dev_up(accel_dev, true); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err_free_reg: - pci_release_regions(accel_pci_dev->pci_dev); -out_err_disable: - pci_disable_device(accel_pci_dev->pci_dev); -out_err: - adf_cleanup_accel(accel_dev); - kfree(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); - adf_cleanup_pci_dev(accel_dev); - kfree(accel_dev); -} - -static int __init adfdrv_init(void) -{ - request_module("intel_qat"); - - if (pci_register_driver(&adf_driver)) { - pr_err("QAT: Driver initialization failed\n"); - return -EFAULT; - } - return 0; -} - -static void __exit adfdrv_release(void) -{ - pci_unregister_driver(&adf_driver); -} - -module_init(adfdrv_init); -module_exit(adfdrv_release); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_FIRMWARE(ADF_C62X_FW); -MODULE_FIRMWARE(ADF_C62X_MMP); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_c62xvf/Makefile b/drivers/crypto/qat/qat_c62xvf/Makefile deleted file mode 100644 index 446c3d638605..000000000000 --- a/drivers/crypto/qat/qat_c62xvf/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o -qat_c62xvf-objs := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c deleted file mode 100644 index 751d7aa57fc7..000000000000 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include "adf_c62xvf_hw_data.h" - -static struct adf_hw_device_class c62xiov_class = { - .name = ADF_C62XVF_DEVICE_NAME, - .type = DEV_C62XVF, - .instances = 0 -}; - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - return ADF_C62XIOV_ACCELERATORS_MASK; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - return ADF_C62XIOV_ACCELENGINES_MASK; -} - -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - return ADF_C62XIOV_MAX_ACCELERATORS; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - return ADF_C62XIOV_MAX_ACCELENGINES; -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C62XIOV_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_C62XIOV_ETR_BAR; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - return DEV_SKU_VF; -} - -static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) -{ - return 0; -} - -static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) -{ -} - -void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class = &c62xiov_class; - hw_data->num_banks = ADF_C62XIOV_ETR_MAX_BANKS; - hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; - hw_data->num_accel = ADF_C62XIOV_MAX_ACCELERATORS; - hw_data->num_logical_accel = 1; - hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_vf_isr_resource_alloc; - hw_data->free_irq = adf_vf_isr_resource_free; - hw_data->enable_error_correction = adf_vf_void_noop; - hw_data->init_admin_comms = adf_vf_int_noop; - hw_data->exit_admin_comms = adf_vf_void_noop; - hw_data->send_admin_init = adf_vf2pf_notify_init; - hw_data->init_arb = adf_vf_int_noop; - hw_data->exit_arb = adf_vf_void_noop; - hw_data->disable_iov = adf_vf2pf_notify_shutdown; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_sku = get_sku; - hw_data->enable_ints = adf_vf_void_noop; - hw_data->dev_class->instances++; - hw_data->dev_config = adf_gen2_dev_config; - adf_devmgr_update_class_index(hw_data); - adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); - adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen2_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; - adf_devmgr_update_class_index(hw_data); -} diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h deleted file mode 100644 index a1a62c003ebf..000000000000 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2015 - 2020 Intel Corporation */ -#ifndef ADF_C62XVF_HW_DATA_H_ -#define ADF_C62XVF_HW_DATA_H_ - -#define ADF_C62XIOV_PMISC_BAR 1 -#define ADF_C62XIOV_ACCELERATORS_MASK 0x1 -#define ADF_C62XIOV_ACCELENGINES_MASK 0x1 -#define ADF_C62XIOV_MAX_ACCELERATORS 1 -#define ADF_C62XIOV_MAX_ACCELENGINES 1 -#define ADF_C62XIOV_RX_RINGS_OFFSET 8 -#define ADF_C62XIOV_TX_RINGS_MASK 0xFF -#define ADF_C62XIOV_ETR_BAR 0 -#define ADF_C62XIOV_ETR_MAX_BANKS 1 - -void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data); -#endif diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c deleted file mode 100644 index 37566309df94..000000000000 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_c62xvf_hw_data.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X_VF), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_C62XVF_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, -}; - -static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) -{ - pci_release_regions(accel_dev->accel_pci_dev.pci_dev); - pci_disable_device(accel_dev->accel_pci_dev.pci_dev); -} - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; - struct adf_accel_dev *pf; - int i; - - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; - - if (bar->virt_addr) - pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); - } - - if (accel_dev->hw_device) { - switch (accel_pci_dev->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: - adf_clean_hw_data_c62xiov(accel_dev->hw_device); - break; - default: - break; - } - kfree(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); - adf_devmgr_rm_dev(accel_dev, pf); -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_dev *pf; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - int ret; - - switch (ent->device) { - case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: - break; - default: - dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); - return -ENODEV; - } - - accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!accel_dev) - return -ENOMEM; - - accel_dev->is_vf = true; - pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* Add accel device to accel table */ - if (adf_devmgr_add_dev(accel_dev, pf)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - kfree(accel_dev); - return -EFAULT; - } - INIT_LIST_HEAD(&accel_dev->crypto_list); - - accel_dev->owner = THIS_MODULE; - /* Allocate and configure device configuration structure */ - hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - accel_dev->hw_device = hw_data; - adf_init_hw_data_c62xiov(accel_dev->hw_device); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - /* enable PCI device */ - if (pci_enable_device(pdev)) { - ret = -EFAULT; - goto out_err; - } - - /* set dma identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration\n"); - goto out_err_disable; - } - - if (pci_request_regions(pdev, ADF_C62XVF_DEVICE_NAME)) { - ret = -EFAULT; - goto out_err_disable; - } - - /* Find and map all the device's BARS */ - i = 0; - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - - bar->base_addr = pci_resource_start(pdev, bar_nr); - if (!bar->base_addr) - break; - bar->size = pci_resource_len(pdev, bar_nr); - bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); - if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); - ret = -EFAULT; - goto out_err_free_reg; - } - } - pci_set_master(pdev); - /* Completion for VF2PF request/response message exchange */ - init_completion(&accel_dev->vf.msg_received); - - ret = adf_dev_up(accel_dev, false); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err_free_reg: - pci_release_regions(accel_pci_dev->pci_dev); -out_err_disable: - pci_disable_device(accel_pci_dev->pci_dev); -out_err: - adf_cleanup_accel(accel_dev); - kfree(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_flush_vf_wq(accel_dev); - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); - adf_cleanup_pci_dev(accel_dev); - kfree(accel_dev); -} - -static int __init adfdrv_init(void) -{ - request_module("intel_qat"); - - if (pci_register_driver(&adf_driver)) { - pr_err("QAT: Driver initialization failed\n"); - return -EFAULT; - } - return 0; -} - -static void __exit adfdrv_release(void) -{ - pci_unregister_driver(&adf_driver); - adf_clean_vf_map(true); -} - -module_init(adfdrv_init); -module_exit(adfdrv_release); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile deleted file mode 100644 index 1fb8d50f509f..000000000000 --- a/drivers/crypto/qat/qat_common/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o -intel_qat-objs := adf_cfg.o \ - adf_isr.o \ - adf_ctl_drv.o \ - adf_dev_mgr.o \ - adf_init.o \ - adf_accel_engine.o \ - adf_aer.o \ - adf_transport.o \ - adf_admin.o \ - adf_hw_arbiter.o \ - adf_sysfs.o \ - adf_gen2_hw_data.o \ - adf_gen2_config.o \ - adf_gen4_hw_data.o \ - adf_gen4_pm.o \ - adf_gen2_dc.o \ - adf_gen4_dc.o \ - qat_crypto.o \ - qat_compression.o \ - qat_comp_algs.o \ - qat_algs.o \ - qat_asym_algs.o \ - qat_algs_send.o \ - qat_uclo.o \ - qat_hal.o \ - qat_bl.o - -intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o -intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ - adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ - adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ - adf_gen2_pfvf.o adf_gen4_pfvf.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h deleted file mode 100644 index bd19e6460899..000000000000 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ /dev/null @@ -1,319 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_ACCEL_DEVICES_H_ -#define ADF_ACCEL_DEVICES_H_ -#include -#include -#include -#include -#include -#include "adf_cfg_common.h" -#include "adf_pfvf_msg.h" - -#define ADF_DH895XCC_DEVICE_NAME "dh895xcc" -#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" -#define ADF_C62X_DEVICE_NAME "c6xx" -#define ADF_C62XVF_DEVICE_NAME "c6xxvf" -#define ADF_C3XXX_DEVICE_NAME "c3xxx" -#define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" -#define ADF_4XXX_DEVICE_NAME "4xxx" -#define ADF_4XXX_PCI_DEVICE_ID 0x4940 -#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 -#define ADF_401XX_PCI_DEVICE_ID 0x4942 -#define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 -#define ADF_402XX_PCI_DEVICE_ID 0x4944 -#define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 -#define ADF_DEVICE_FUSECTL_OFFSET 0x40 -#define ADF_DEVICE_LEGFUSE_OFFSET 0x4C -#define ADF_DEVICE_FUSECTL_MASK 0x80000000 -#define ADF_PCI_MAX_BARS 3 -#define ADF_DEVICE_NAME_LENGTH 32 -#define ADF_ETR_MAX_RINGS_PER_BANK 16 -#define ADF_MAX_MSIX_VECTOR_NAME 16 -#define ADF_DEVICE_NAME_PREFIX "qat_" - -enum adf_accel_capabilities { - ADF_ACCEL_CAPABILITIES_NULL = 0, - ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, - ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, - ADF_ACCEL_CAPABILITIES_CIPHER = 4, - ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, - ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, - ADF_ACCEL_CAPABILITIES_LZS_COMPRESSION = 64, - ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 -}; - -struct adf_bar { - resource_size_t base_addr; - void __iomem *virt_addr; - resource_size_t size; -}; - -struct adf_irq { - bool enabled; - char name[ADF_MAX_MSIX_VECTOR_NAME]; -}; - -struct adf_accel_msix { - struct adf_irq *irqs; - u32 num_entries; -}; - -struct adf_accel_pci { - struct pci_dev *pci_dev; - struct adf_accel_msix msix_entries; - struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; - u8 revid; - u8 sku; -}; - -enum dev_state { - DEV_DOWN = 0, - DEV_UP -}; - -enum dev_sku_info { - DEV_SKU_1 = 0, - DEV_SKU_2, - DEV_SKU_3, - DEV_SKU_4, - DEV_SKU_VF, - DEV_SKU_UNKNOWN, -}; - -static inline const char *get_sku_info(enum dev_sku_info info) -{ - switch (info) { - case DEV_SKU_1: - return "SKU1"; - case DEV_SKU_2: - return "SKU2"; - case DEV_SKU_3: - return "SKU3"; - case DEV_SKU_4: - return "SKU4"; - case DEV_SKU_VF: - return "SKUVF"; - case DEV_SKU_UNKNOWN: - default: - break; - } - return "Unknown SKU"; -} - -struct adf_hw_device_class { - const char *name; - const enum adf_device_type type; - u32 instances; -}; - -struct arb_info { - u32 arb_cfg; - u32 arb_offset; - u32 wt2sam_offset; -}; - -struct admin_info { - u32 admin_msg_ur; - u32 admin_msg_lr; - u32 mailbox_offset; -}; - -struct adf_hw_csr_ops { - u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); - u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, - u32 ring); - void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, - u32 ring, u32 value); - u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, - u32 ring); - void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, - u32 ring, u32 value); - u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); - void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, - u32 ring, u32 value); - void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, - u32 ring, dma_addr_t addr); - void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, - u32 value); - void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); - void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, - u32 value); - void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, - u32 value); - void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, - u32 bank, u32 value); - void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, - u32 value); -}; - -struct adf_cfg_device_data; -struct adf_accel_dev; -struct adf_etr_data; -struct adf_etr_ring_data; - -struct adf_pfvf_ops { - int (*enable_comms)(struct adf_accel_dev *accel_dev); - u32 (*get_pf2vf_offset)(u32 i); - u32 (*get_vf2pf_offset)(u32 i); - void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); - void (*disable_all_vf2pf_interrupts)(void __iomem *pmisc_addr); - u32 (*disable_pending_vf2pf_interrupts)(void __iomem *pmisc_addr); - int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, - u32 pfvf_offset, struct mutex *csr_lock); - struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, - u32 pfvf_offset, u8 compat_ver); -}; - -struct adf_dc_ops { - void (*build_deflate_ctx)(void *ctx); -}; - -struct adf_hw_device_data { - struct adf_hw_device_class *dev_class; - u32 (*get_accel_mask)(struct adf_hw_device_data *self); - u32 (*get_ae_mask)(struct adf_hw_device_data *self); - u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); - u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); - u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); - u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); - u32 (*get_num_aes)(struct adf_hw_device_data *self); - u32 (*get_num_accels)(struct adf_hw_device_data *self); - void (*get_arb_info)(struct arb_info *arb_csrs_info); - void (*get_admin_info)(struct admin_info *admin_csrs_info); - enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); - int (*alloc_irq)(struct adf_accel_dev *accel_dev); - void (*free_irq)(struct adf_accel_dev *accel_dev); - void (*enable_error_correction)(struct adf_accel_dev *accel_dev); - int (*init_admin_comms)(struct adf_accel_dev *accel_dev); - void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); - int (*send_admin_init)(struct adf_accel_dev *accel_dev); - int (*init_arb)(struct adf_accel_dev *accel_dev); - void (*exit_arb)(struct adf_accel_dev *accel_dev); - const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev); - int (*init_device)(struct adf_accel_dev *accel_dev); - int (*enable_pm)(struct adf_accel_dev *accel_dev); - bool (*handle_pm_interrupt)(struct adf_accel_dev *accel_dev); - void (*disable_iov)(struct adf_accel_dev *accel_dev); - void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, - bool enable); - void (*enable_ints)(struct adf_accel_dev *accel_dev); - void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); - int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); - void (*reset_device)(struct adf_accel_dev *accel_dev); - void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); - char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); - u32 (*uof_get_num_objs)(void); - u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); - int (*dev_config)(struct adf_accel_dev *accel_dev); - struct adf_pfvf_ops pfvf_ops; - struct adf_hw_csr_ops csr_ops; - struct adf_dc_ops dc_ops; - const char *fw_name; - const char *fw_mmp_name; - u32 fuses; - u32 straps; - u32 accel_capabilities_mask; - u32 extended_dc_capabilities; - u32 clock_frequency; - u32 instance_id; - u16 accel_mask; - u32 ae_mask; - u32 admin_ae_mask; - u16 tx_rings_mask; - u16 ring_to_svc_map; - u8 tx_rx_gap; - u8 num_banks; - u16 num_banks_per_vf; - u8 num_rings_per_bank; - u8 num_accel; - u8 num_logical_accel; - u8 num_engines; -}; - -/* CSR write macro */ -#define ADF_CSR_WR(csr_base, csr_offset, val) \ - __raw_writel(val, csr_base + csr_offset) - -/* CSR read macro */ -#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) - -#define ADF_CFG_NUM_SERVICES 4 -#define ADF_SRV_TYPE_BIT_LEN 3 -#define ADF_SRV_TYPE_MASK 0x7 - -#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) -#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) -#define GET_HW_DATA(accel_dev) (accel_dev->hw_device) -#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) -#define GET_NUM_RINGS_PER_BANK(accel_dev) \ - GET_HW_DATA(accel_dev)->num_rings_per_bank -#define GET_SRV_TYPE(accel_dev, idx) \ - (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \ - & ADF_SRV_TYPE_MASK) -#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) -#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) -#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) -#define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops) -#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev - -struct adf_admin_comms; -struct icp_qat_fw_loader_handle; -struct adf_fw_loader_data { - struct icp_qat_fw_loader_handle *fw_loader; - const struct firmware *uof_fw; - const struct firmware *mmp_fw; -}; - -struct adf_accel_vf_info { - struct adf_accel_dev *accel_dev; - struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ - struct ratelimit_state vf2pf_ratelimit; - u32 vf_nr; - bool init; - u8 vf_compat_ver; -}; - -struct adf_dc_data { - u8 *ovf_buff; - size_t ovf_buff_sz; - dma_addr_t ovf_buff_p; -}; - -struct adf_accel_dev { - struct adf_etr_data *transport; - struct adf_hw_device_data *hw_device; - struct adf_cfg_device_data *cfg; - struct adf_fw_loader_data *fw_loader; - struct adf_admin_comms *admin; - struct adf_dc_data *dc_data; - struct list_head crypto_list; - struct list_head compression_list; - unsigned long status; - atomic_t ref_count; - struct dentry *debugfs_dir; - struct list_head list; - struct module *owner; - struct adf_accel_pci accel_pci_dev; - union { - struct { - /* protects VF2PF interrupts access */ - spinlock_t vf2pf_ints_lock; - /* vf_info is non-zero when SR-IOV is init'ed */ - struct adf_accel_vf_info *vf_info; - } pf; - struct { - bool irq_enabled; - char irq_name[ADF_MAX_MSIX_VECTOR_NAME]; - struct tasklet_struct pf2vf_bh_tasklet; - struct mutex vf2pf_lock; /* protect CSR access */ - struct completion msg_received; - struct pfvf_message response; /* temp field holding pf2vf response */ - u8 pf_compat_ver; - } vf; - }; - struct mutex state_lock; /* protect state of the device */ - bool is_vf; - u32 accel_id; -}; -#endif diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c deleted file mode 100644 index 4ce2b666929e..000000000000 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ /dev/null @@ -1,212 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include "adf_cfg.h" -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "icp_qat_uclo.h" - -static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, - u32 fw_size) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct icp_qat_fw_loader_handle *loader; - char *obj_name; - u32 num_objs; - u32 ae_mask; - int i; - - loader = loader_data->fw_loader; - num_objs = hw_device->uof_get_num_objs(); - - for (i = 0; i < num_objs; i++) { - obj_name = hw_device->uof_get_name(accel_dev, i); - ae_mask = hw_device->uof_get_ae_mask(accel_dev, i); - if (!obj_name || !ae_mask) { - dev_err(&GET_DEV(accel_dev), "Invalid UOF image\n"); - goto out_err; - } - - if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) { - dev_err(&GET_DEV(accel_dev), - "Invalid mask for UOF image\n"); - goto out_err; - } - if (qat_uclo_map_obj(loader, fw_addr, fw_size, obj_name)) { - dev_err(&GET_DEV(accel_dev), - "Failed to map UOF firmware\n"); - goto out_err; - } - if (qat_uclo_wr_all_uimage(loader)) { - dev_err(&GET_DEV(accel_dev), - "Failed to load UOF firmware\n"); - goto out_err; - } - qat_uclo_del_obj(loader); - } - - return 0; - -out_err: - adf_ae_fw_release(accel_dev); - return -EFAULT; -} - -int adf_ae_fw_load(struct adf_accel_dev *accel_dev) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - void *fw_addr, *mmp_addr; - u32 fw_size, mmp_size; - - if (!hw_device->fw_name) - return 0; - - if (request_firmware(&loader_data->mmp_fw, hw_device->fw_mmp_name, - &accel_dev->accel_pci_dev.pci_dev->dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to load MMP firmware %s\n", - hw_device->fw_mmp_name); - return -EFAULT; - } - if (request_firmware(&loader_data->uof_fw, hw_device->fw_name, - &accel_dev->accel_pci_dev.pci_dev->dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to load UOF firmware %s\n", - hw_device->fw_name); - goto out_err; - } - - fw_size = loader_data->uof_fw->size; - fw_addr = (void *)loader_data->uof_fw->data; - mmp_size = loader_data->mmp_fw->size; - mmp_addr = (void *)loader_data->mmp_fw->data; - - if (qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size)) { - dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); - goto out_err; - } - - if (hw_device->uof_get_num_objs) - return adf_ae_fw_load_images(accel_dev, fw_addr, fw_size); - - if (qat_uclo_map_obj(loader_data->fw_loader, fw_addr, fw_size, NULL)) { - dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); - goto out_err; - } - if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { - dev_err(&GET_DEV(accel_dev), "Failed to load UOF\n"); - goto out_err; - } - return 0; - -out_err: - adf_ae_fw_release(accel_dev); - return -EFAULT; -} - -void adf_ae_fw_release(struct adf_accel_dev *accel_dev) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - - if (!hw_device->fw_name) - return; - - qat_uclo_del_obj(loader_data->fw_loader); - qat_hal_deinit(loader_data->fw_loader); - release_firmware(loader_data->uof_fw); - release_firmware(loader_data->mmp_fw); - loader_data->uof_fw = NULL; - loader_data->mmp_fw = NULL; - loader_data->fw_loader = NULL; -} - -int adf_ae_start(struct adf_accel_dev *accel_dev) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 ae_ctr; - - if (!hw_data->fw_name) - return 0; - - ae_ctr = qat_hal_start(loader_data->fw_loader); - dev_info(&GET_DEV(accel_dev), - "qat_dev%d started %d acceleration engines\n", - accel_dev->accel_id, ae_ctr); - return 0; -} - -int adf_ae_stop(struct adf_accel_dev *accel_dev) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); - - if (!hw_data->fw_name) - return 0; - - for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { - if (hw_data->ae_mask & (1 << ae)) { - qat_hal_stop(loader_data->fw_loader, ae, 0xFF); - ae_ctr++; - } - } - dev_info(&GET_DEV(accel_dev), - "qat_dev%d stopped %d acceleration engines\n", - accel_dev->accel_id, ae_ctr); - return 0; -} - -static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - - qat_hal_reset(loader_data->fw_loader); - if (qat_hal_clr_reset(loader_data->fw_loader)) - return -EFAULT; - - return 0; -} - -int adf_ae_init(struct adf_accel_dev *accel_dev) -{ - struct adf_fw_loader_data *loader_data; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - - if (!hw_device->fw_name) - return 0; - - loader_data = kzalloc(sizeof(*loader_data), GFP_KERNEL); - if (!loader_data) - return -ENOMEM; - - accel_dev->fw_loader = loader_data; - if (qat_hal_init(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to init the AEs\n"); - kfree(loader_data); - return -EFAULT; - } - if (adf_ae_reset(accel_dev, 0)) { - dev_err(&GET_DEV(accel_dev), "Failed to reset the AEs\n"); - qat_hal_deinit(loader_data->fw_loader); - kfree(loader_data); - return -EFAULT; - } - return 0; -} - -int adf_ae_shutdown(struct adf_accel_dev *accel_dev) -{ - struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - - if (!hw_device->fw_name) - return 0; - - qat_hal_deinit(loader_data->fw_loader); - kfree(accel_dev->fw_loader); - accel_dev->fw_loader = NULL; - return 0; -} diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c deleted file mode 100644 index 3b6184c35081..000000000000 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ /dev/null @@ -1,362 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "icp_qat_fw_init_admin.h" - -#define ADF_ADMIN_MAILBOX_STRIDE 0x1000 -#define ADF_ADMINMSG_LEN 32 -#define ADF_CONST_TABLE_SIZE 1024 -#define ADF_ADMIN_POLL_DELAY_US 20 -#define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC) - -static const u8 const_tab[1024] __aligned(1024) = { -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, -0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, -0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, -0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, -0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, -0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, -0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, -0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, -0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, -0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, -0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, -0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, -0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, -0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, -0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, -0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, -0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, -0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, -0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, -0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, -0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; - -struct adf_admin_comms { - dma_addr_t phy_addr; - dma_addr_t const_tbl_addr; - void *virt_addr; - void *virt_tbl_addr; - void __iomem *mailbox_addr; - struct mutex lock; /* protects adf_admin_comms struct */ -}; - -static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, - void *in, void *out) -{ - int ret; - u32 status; - struct adf_admin_comms *admin = accel_dev->admin; - int offset = ae * ADF_ADMINMSG_LEN * 2; - void __iomem *mailbox = admin->mailbox_addr; - int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; - struct icp_qat_fw_init_admin_req *request = in; - - mutex_lock(&admin->lock); - - if (ADF_CSR_RD(mailbox, mb_offset) == 1) { - mutex_unlock(&admin->lock); - return -EAGAIN; - } - - memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); - ADF_CSR_WR(mailbox, mb_offset, 1); - - ret = read_poll_timeout(ADF_CSR_RD, status, status == 0, - ADF_ADMIN_POLL_DELAY_US, - ADF_ADMIN_POLL_TIMEOUT_US, true, - mailbox, mb_offset); - if (ret < 0) { - /* Response timeout */ - dev_err(&GET_DEV(accel_dev), - "Failed to send admin msg %d to accelerator %d\n", - request->cmd_id, ae); - } else { - /* Response received from admin message, we can now - * make response data available in "out" parameter. - */ - memcpy(out, admin->virt_addr + offset + - ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); - } - - mutex_unlock(&admin->lock); - return ret; -} - -static int adf_send_admin(struct adf_accel_dev *accel_dev, - struct icp_qat_fw_init_admin_req *req, - struct icp_qat_fw_init_admin_resp *resp, - const unsigned long ae_mask) -{ - u32 ae; - - for_each_set_bit(ae, &ae_mask, ICP_QAT_HW_AE_DELIMITER) - if (adf_put_admin_msg_sync(accel_dev, ae, req, resp) || - resp->status) - return -EFAULT; - - return 0; -} - -static int adf_init_ae(struct adf_accel_dev *accel_dev) -{ - struct icp_qat_fw_init_admin_req req; - struct icp_qat_fw_init_admin_resp resp; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; - - memset(&req, 0, sizeof(req)); - memset(&resp, 0, sizeof(resp)); - req.cmd_id = ICP_QAT_FW_INIT_AE; - - return adf_send_admin(accel_dev, &req, &resp, ae_mask); -} - -static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) -{ - struct icp_qat_fw_init_admin_req req; - struct icp_qat_fw_init_admin_resp resp; - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; - - memset(&req, 0, sizeof(req)); - memset(&resp, 0, sizeof(resp)); - req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; - - req.init_cfg_sz = ADF_CONST_TABLE_SIZE; - req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; - - return adf_send_admin(accel_dev, &req, &resp, ae_mask); -} - -static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, - u32 *capabilities) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct icp_qat_fw_init_admin_resp resp; - struct icp_qat_fw_init_admin_req req; - unsigned long ae_mask; - unsigned long ae; - int ret; - - /* Target only service accelerator engines */ - ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask; - - memset(&req, 0, sizeof(req)); - memset(&resp, 0, sizeof(resp)); - req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; - - *capabilities = 0; - for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { - ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae); - if (ret) - return ret; - - *capabilities |= resp.extended_features; - } - - return 0; -} - -/** - * adf_send_admin_init() - Function sends init message to FW - * @accel_dev: Pointer to acceleration device. - * - * Function sends admin init message to the FW - * - * Return: 0 on success, error code otherwise. - */ -int adf_send_admin_init(struct adf_accel_dev *accel_dev) -{ - u32 dc_capabilities = 0; - int ret; - - ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities); - if (ret) { - dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n"); - return ret; - } - accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; - - ret = adf_set_fw_constants(accel_dev); - if (ret) - return ret; - - return adf_init_ae(accel_dev); -} -EXPORT_SYMBOL_GPL(adf_send_admin_init); - -/** - * adf_init_admin_pm() - Function sends PM init message to FW - * @accel_dev: Pointer to acceleration device. - * @idle_delay: QAT HW idle time before power gating is initiated. - * 000 - 64us - * 001 - 128us - * 010 - 256us - * 011 - 512us - * 100 - 1ms - * 101 - 2ms - * 110 - 4ms - * 111 - 8ms - * - * Function sends to the FW the admin init message for the PM state - * configuration. - * - * Return: 0 on success, error code otherwise. - */ -int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct icp_qat_fw_init_admin_resp resp = {0}; - struct icp_qat_fw_init_admin_req req = {0}; - u32 ae_mask = hw_data->admin_ae_mask; - - if (!accel_dev->admin) { - dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); - return -EFAULT; - } - - req.cmd_id = ICP_QAT_FW_PM_STATE_CONFIG; - req.idle_filter = idle_delay; - - return adf_send_admin(accel_dev, &req, &resp, ae_mask); -} -EXPORT_SYMBOL_GPL(adf_init_admin_pm); - -int adf_init_admin_comms(struct adf_accel_dev *accel_dev) -{ - struct adf_admin_comms *admin; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - struct admin_info admin_csrs_info; - u32 mailbox_offset, adminmsg_u, adminmsg_l; - void __iomem *mailbox; - u64 reg_val; - - admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!admin) - return -ENOMEM; - admin->virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - &admin->phy_addr, GFP_KERNEL); - if (!admin->virt_addr) { - dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); - kfree(admin); - return -ENOMEM; - } - - admin->virt_tbl_addr = dma_alloc_coherent(&GET_DEV(accel_dev), - PAGE_SIZE, - &admin->const_tbl_addr, - GFP_KERNEL); - if (!admin->virt_tbl_addr) { - dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n"); - dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - admin->virt_addr, admin->phy_addr); - kfree(admin); - return -ENOMEM; - } - - memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); - hw_data->get_admin_info(&admin_csrs_info); - - mailbox_offset = admin_csrs_info.mailbox_offset; - mailbox = pmisc_addr + mailbox_offset; - adminmsg_u = admin_csrs_info.admin_msg_ur; - adminmsg_l = admin_csrs_info.admin_msg_lr; - - reg_val = (u64)admin->phy_addr; - ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val)); - ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val)); - - mutex_init(&admin->lock); - admin->mailbox_addr = mailbox; - accel_dev->admin = admin; - return 0; -} -EXPORT_SYMBOL_GPL(adf_init_admin_comms); - -void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) -{ - struct adf_admin_comms *admin = accel_dev->admin; - - if (!admin) - return; - - if (admin->virt_addr) - dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - admin->virt_addr, admin->phy_addr); - if (admin->virt_tbl_addr) - dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - admin->virt_tbl_addr, admin->const_tbl_addr); - - mutex_destroy(&admin->lock); - kfree(admin); - accel_dev->admin = NULL; -} -EXPORT_SYMBOL_GPL(adf_exit_admin_comms); diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c deleted file mode 100644 index 04af32a2811c..000000000000 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" - -static struct workqueue_struct *device_reset_wq; - -static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - dev_info(&pdev->dev, "Acceleration driver hardware error detected.\n"); - if (!accel_dev) { - dev_err(&pdev->dev, "Can't find acceleration device\n"); - return PCI_ERS_RESULT_DISCONNECT; - } - - if (state == pci_channel_io_perm_failure) { - dev_err(&pdev->dev, "Can't recover from device error\n"); - return PCI_ERS_RESULT_DISCONNECT; - } - - return PCI_ERS_RESULT_NEED_RESET; -} - -/* reset dev data */ -struct adf_reset_dev_data { - int mode; - struct adf_accel_dev *accel_dev; - struct completion compl; - struct work_struct reset_work; -}; - -void adf_reset_sbr(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - struct pci_dev *parent = pdev->bus->self; - u16 bridge_ctl = 0; - - if (!parent) - parent = pdev; - - if (!pci_wait_for_pending_transaction(pdev)) - dev_info(&GET_DEV(accel_dev), - "Transaction still in progress. Proceeding\n"); - - dev_info(&GET_DEV(accel_dev), "Secondary bus reset\n"); - - pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); - bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; - pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); - msleep(100); - bridge_ctl &= ~PCI_BRIDGE_CTL_BUS_RESET; - pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); - msleep(100); -} -EXPORT_SYMBOL_GPL(adf_reset_sbr); - -void adf_reset_flr(struct adf_accel_dev *accel_dev) -{ - pcie_flr(accel_to_pci_dev(accel_dev)); -} -EXPORT_SYMBOL_GPL(adf_reset_flr); - -void adf_dev_restore(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - - if (hw_device->reset_device) { - dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", - accel_dev->accel_id); - hw_device->reset_device(accel_dev); - pci_restore_state(pdev); - pci_save_state(pdev); - } -} - -static void adf_device_reset_worker(struct work_struct *work) -{ - struct adf_reset_dev_data *reset_data = - container_of(work, struct adf_reset_dev_data, reset_work); - struct adf_accel_dev *accel_dev = reset_data->accel_dev; - - adf_dev_restarting_notify(accel_dev); - if (adf_dev_restart(accel_dev)) { - /* The device hanged and we can't restart it so stop here */ - dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); - kfree(reset_data); - WARN(1, "QAT: device restart failed. Device is unusable\n"); - return; - } - adf_dev_restarted_notify(accel_dev); - clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - - /* The dev is back alive. Notify the caller if in sync mode */ - if (reset_data->mode == ADF_DEV_RESET_SYNC) - complete(&reset_data->compl); - else - kfree(reset_data); -} - -static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, - enum adf_dev_reset_mode mode) -{ - struct adf_reset_dev_data *reset_data; - - if (!adf_dev_started(accel_dev) || - test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) - return 0; - - set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - reset_data = kzalloc(sizeof(*reset_data), GFP_KERNEL); - if (!reset_data) - return -ENOMEM; - reset_data->accel_dev = accel_dev; - init_completion(&reset_data->compl); - reset_data->mode = mode; - INIT_WORK(&reset_data->reset_work, adf_device_reset_worker); - queue_work(device_reset_wq, &reset_data->reset_work); - - /* If in sync mode wait for the result */ - if (mode == ADF_DEV_RESET_SYNC) { - int ret = 0; - /* Maximum device reset time is 10 seconds */ - unsigned long wait_jiffies = msecs_to_jiffies(10000); - unsigned long timeout = wait_for_completion_timeout( - &reset_data->compl, wait_jiffies); - if (!timeout) { - dev_err(&GET_DEV(accel_dev), - "Reset device timeout expired\n"); - ret = -EFAULT; - } - kfree(reset_data); - return ret; - } - return 0; -} - -static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Can't find acceleration device\n"); - return PCI_ERS_RESULT_DISCONNECT; - } - if (adf_dev_aer_schedule_reset(accel_dev, ADF_DEV_RESET_SYNC)) - return PCI_ERS_RESULT_DISCONNECT; - - return PCI_ERS_RESULT_RECOVERED; -} - -static void adf_resume(struct pci_dev *pdev) -{ - dev_info(&pdev->dev, "Acceleration driver reset completed\n"); - dev_info(&pdev->dev, "Device is up and running\n"); -} - -const struct pci_error_handlers adf_err_handler = { - .error_detected = adf_error_detected, - .slot_reset = adf_slot_reset, - .resume = adf_resume, -}; -EXPORT_SYMBOL_GPL(adf_err_handler); - -int adf_init_aer(void) -{ - device_reset_wq = alloc_workqueue("qat_device_reset_wq", - WQ_MEM_RECLAIM, 0); - return !device_reset_wq ? -EFAULT : 0; -} - -void adf_exit_aer(void) -{ - if (device_reset_wq) - destroy_workqueue(device_reset_wq); - device_reset_wq = NULL; -} diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c deleted file mode 100644 index 1931e5b37f2b..000000000000 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ /dev/null @@ -1,339 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_cfg.h" -#include "adf_common_drv.h" - -static DEFINE_MUTEX(qat_cfg_read_lock); - -static void *qat_dev_cfg_start(struct seq_file *sfile, loff_t *pos) -{ - struct adf_cfg_device_data *dev_cfg = sfile->private; - - mutex_lock(&qat_cfg_read_lock); - return seq_list_start(&dev_cfg->sec_list, *pos); -} - -static int qat_dev_cfg_show(struct seq_file *sfile, void *v) -{ - struct list_head *list; - struct adf_cfg_section *sec = - list_entry(v, struct adf_cfg_section, list); - - seq_printf(sfile, "[%s]\n", sec->name); - list_for_each(list, &sec->param_head) { - struct adf_cfg_key_val *ptr = - list_entry(list, struct adf_cfg_key_val, list); - seq_printf(sfile, "%s = %s\n", ptr->key, ptr->val); - } - return 0; -} - -static void *qat_dev_cfg_next(struct seq_file *sfile, void *v, loff_t *pos) -{ - struct adf_cfg_device_data *dev_cfg = sfile->private; - - return seq_list_next(v, &dev_cfg->sec_list, pos); -} - -static void qat_dev_cfg_stop(struct seq_file *sfile, void *v) -{ - mutex_unlock(&qat_cfg_read_lock); -} - -static const struct seq_operations qat_dev_cfg_sops = { - .start = qat_dev_cfg_start, - .next = qat_dev_cfg_next, - .stop = qat_dev_cfg_stop, - .show = qat_dev_cfg_show -}; - -DEFINE_SEQ_ATTRIBUTE(qat_dev_cfg); - -/** - * adf_cfg_dev_add() - Create an acceleration device configuration table. - * @accel_dev: Pointer to acceleration device. - * - * Function creates a configuration table for the given acceleration device. - * The table stores device specific config values. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ -int adf_cfg_dev_add(struct adf_accel_dev *accel_dev) -{ - struct adf_cfg_device_data *dev_cfg_data; - - dev_cfg_data = kzalloc(sizeof(*dev_cfg_data), GFP_KERNEL); - if (!dev_cfg_data) - return -ENOMEM; - INIT_LIST_HEAD(&dev_cfg_data->sec_list); - init_rwsem(&dev_cfg_data->lock); - accel_dev->cfg = dev_cfg_data; - - /* accel_dev->debugfs_dir should always be non-NULL here */ - dev_cfg_data->debug = debugfs_create_file("dev_cfg", S_IRUSR, - accel_dev->debugfs_dir, - dev_cfg_data, - &qat_dev_cfg_fops); - return 0; -} -EXPORT_SYMBOL_GPL(adf_cfg_dev_add); - -static void adf_cfg_section_del_all(struct list_head *head); - -void adf_cfg_del_all(struct adf_accel_dev *accel_dev) -{ - struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; - - down_write(&dev_cfg_data->lock); - adf_cfg_section_del_all(&dev_cfg_data->sec_list); - up_write(&dev_cfg_data->lock); - clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -} - -/** - * adf_cfg_dev_remove() - Clears acceleration device configuration table. - * @accel_dev: Pointer to acceleration device. - * - * Function removes configuration table from the given acceleration device - * and frees all allocated memory. - * To be used by QAT device specific drivers. - * - * Return: void - */ -void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev) -{ - struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; - - if (!dev_cfg_data) - return; - - down_write(&dev_cfg_data->lock); - adf_cfg_section_del_all(&dev_cfg_data->sec_list); - up_write(&dev_cfg_data->lock); - debugfs_remove(dev_cfg_data->debug); - kfree(dev_cfg_data); - accel_dev->cfg = NULL; -} -EXPORT_SYMBOL_GPL(adf_cfg_dev_remove); - -static void adf_cfg_keyval_add(struct adf_cfg_key_val *new, - struct adf_cfg_section *sec) -{ - list_add_tail(&new->list, &sec->param_head); -} - -static void adf_cfg_keyval_remove(const char *key, struct adf_cfg_section *sec) -{ - struct list_head *head = &sec->param_head; - struct list_head *list_ptr, *tmp; - - list_for_each_prev_safe(list_ptr, tmp, head) { - struct adf_cfg_key_val *ptr = - list_entry(list_ptr, struct adf_cfg_key_val, list); - - if (strncmp(ptr->key, key, sizeof(ptr->key))) - continue; - - list_del(list_ptr); - kfree(ptr); - break; - } -} - -static void adf_cfg_keyval_del_all(struct list_head *head) -{ - struct list_head *list_ptr, *tmp; - - list_for_each_prev_safe(list_ptr, tmp, head) { - struct adf_cfg_key_val *ptr = - list_entry(list_ptr, struct adf_cfg_key_val, list); - list_del(list_ptr); - kfree(ptr); - } -} - -static void adf_cfg_section_del_all(struct list_head *head) -{ - struct adf_cfg_section *ptr; - struct list_head *list, *tmp; - - list_for_each_prev_safe(list, tmp, head) { - ptr = list_entry(list, struct adf_cfg_section, list); - adf_cfg_keyval_del_all(&ptr->param_head); - list_del(list); - kfree(ptr); - } -} - -static struct adf_cfg_key_val *adf_cfg_key_value_find(struct adf_cfg_section *s, - const char *key) -{ - struct list_head *list; - - list_for_each(list, &s->param_head) { - struct adf_cfg_key_val *ptr = - list_entry(list, struct adf_cfg_key_val, list); - if (!strcmp(ptr->key, key)) - return ptr; - } - return NULL; -} - -static struct adf_cfg_section *adf_cfg_sec_find(struct adf_accel_dev *accel_dev, - const char *sec_name) -{ - struct adf_cfg_device_data *cfg = accel_dev->cfg; - struct list_head *list; - - list_for_each(list, &cfg->sec_list) { - struct adf_cfg_section *ptr = - list_entry(list, struct adf_cfg_section, list); - if (!strcmp(ptr->name, sec_name)) - return ptr; - } - return NULL; -} - -static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, - const char *sec_name, - const char *key_name, - char *val) -{ - struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, sec_name); - struct adf_cfg_key_val *keyval = NULL; - - if (sec) - keyval = adf_cfg_key_value_find(sec, key_name); - if (keyval) { - memcpy(val, keyval->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES); - return 0; - } - return -ENODATA; -} - -/** - * adf_cfg_add_key_value_param() - Add key-value config entry to config table. - * @accel_dev: Pointer to acceleration device. - * @section_name: Name of the section where the param will be added - * @key: The key string - * @val: Value pain for the given @key - * @type: Type - string, int or address - * - * Function adds configuration key - value entry in the appropriate section - * in the given acceleration device. If the key exists already, the value - * is updated. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ -int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, - const char *section_name, - const char *key, const void *val, - enum adf_cfg_val_type type) -{ - struct adf_cfg_device_data *cfg = accel_dev->cfg; - struct adf_cfg_key_val *key_val; - struct adf_cfg_section *section = adf_cfg_sec_find(accel_dev, - section_name); - char temp_val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - - if (!section) - return -EFAULT; - - key_val = kzalloc(sizeof(*key_val), GFP_KERNEL); - if (!key_val) - return -ENOMEM; - - INIT_LIST_HEAD(&key_val->list); - strscpy(key_val->key, key, sizeof(key_val->key)); - - if (type == ADF_DEC) { - snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, - "%ld", (*((long *)val))); - } else if (type == ADF_STR) { - strscpy(key_val->val, (char *)val, sizeof(key_val->val)); - } else if (type == ADF_HEX) { - snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, - "0x%lx", (unsigned long)val); - } else { - dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); - kfree(key_val); - return -EINVAL; - } - key_val->type = type; - - /* Add the key-value pair as below policy: - * 1. if the key doesn't exist, add it; - * 2. if the key already exists with a different value then update it - * to the new value (the key is deleted and the newly created - * key_val containing the new value is added to the database); - * 3. if the key exists with the same value, then return without doing - * anything (the newly created key_val is freed). - */ - if (!adf_cfg_key_val_get(accel_dev, section_name, key, temp_val)) { - if (strncmp(temp_val, key_val->val, sizeof(temp_val))) { - adf_cfg_keyval_remove(key, section); - } else { - kfree(key_val); - return 0; - } - } - - down_write(&cfg->lock); - adf_cfg_keyval_add(key_val, section); - up_write(&cfg->lock); - return 0; -} -EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param); - -/** - * adf_cfg_section_add() - Add config section entry to config table. - * @accel_dev: Pointer to acceleration device. - * @name: Name of the section - * - * Function adds configuration section where key - value entries - * will be stored. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ -int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) -{ - struct adf_cfg_device_data *cfg = accel_dev->cfg; - struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, name); - - if (sec) - return 0; - - sec = kzalloc(sizeof(*sec), GFP_KERNEL); - if (!sec) - return -ENOMEM; - - strscpy(sec->name, name, sizeof(sec->name)); - INIT_LIST_HEAD(&sec->param_head); - down_write(&cfg->lock); - list_add_tail(&sec->list, &cfg->sec_list); - up_write(&cfg->lock); - return 0; -} -EXPORT_SYMBOL_GPL(adf_cfg_section_add); - -int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, - const char *section, const char *name, - char *value) -{ - struct adf_cfg_device_data *cfg = accel_dev->cfg; - int ret; - - down_read(&cfg->lock); - ret = adf_cfg_key_val_get(accel_dev, section, name, value); - up_read(&cfg->lock); - return ret; -} -EXPORT_SYMBOL_GPL(adf_cfg_get_param_value); diff --git a/drivers/crypto/qat/qat_common/adf_cfg.h b/drivers/crypto/qat/qat_common/adf_cfg.h deleted file mode 100644 index 376cde61a60e..000000000000 --- a/drivers/crypto/qat/qat_common/adf_cfg.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_CFG_H_ -#define ADF_CFG_H_ - -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_cfg_common.h" -#include "adf_cfg_strings.h" - -struct adf_cfg_key_val { - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - enum adf_cfg_val_type type; - struct list_head list; -}; - -struct adf_cfg_section { - char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; - struct list_head list; - struct list_head param_head; -}; - -struct adf_cfg_device_data { - struct list_head sec_list; - struct dentry *debug; - struct rw_semaphore lock; -}; - -int adf_cfg_dev_add(struct adf_accel_dev *accel_dev); -void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev); -int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name); -void adf_cfg_del_all(struct adf_accel_dev *accel_dev); -int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, - const char *section_name, - const char *key, const void *val, - enum adf_cfg_val_type type); -int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, - const char *section, const char *name, char *value); - -#endif diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h deleted file mode 100644 index 6e5de1dab97b..000000000000 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ /dev/null @@ -1,76 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_CFG_COMMON_H_ -#define ADF_CFG_COMMON_H_ - -#include -#include - -#define ADF_CFG_MAX_STR_LEN 64 -#define ADF_CFG_MAX_KEY_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN -#define ADF_CFG_MAX_VAL_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN -#define ADF_CFG_MAX_SECTION_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN -#define ADF_CFG_BASE_DEC 10 -#define ADF_CFG_BASE_HEX 16 -#define ADF_CFG_ALL_DEVICES 0xFE -#define ADF_CFG_NO_DEVICE 0xFF -#define ADF_CFG_AFFINITY_WHATEVER 0xFF -#define MAX_DEVICE_NAME_SIZE 32 -#define ADF_MAX_DEVICES (32 * 32) -#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES) - -#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0 -#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3 -#define ADF_CFG_SERV_RING_PAIR_2_SHIFT 6 -#define ADF_CFG_SERV_RING_PAIR_3_SHIFT 9 -enum adf_cfg_service_type { - UNUSED = 0, - CRYPTO, - COMP, - SYM, - ASYM, - USED -}; - -enum adf_cfg_val_type { - ADF_DEC, - ADF_HEX, - ADF_STR -}; - -enum adf_device_type { - DEV_UNKNOWN = 0, - DEV_DH895XCC, - DEV_DH895XCCVF, - DEV_C62X, - DEV_C62XVF, - DEV_C3XXX, - DEV_C3XXXVF, - DEV_4XXX, -}; - -struct adf_dev_status_info { - enum adf_device_type type; - __u32 accel_id; - __u32 instance_id; - __u8 num_ae; - __u8 num_accel; - __u8 num_logical_accel; - __u8 banks_per_accel; - __u8 state; - __u8 bus; - __u8 dev; - __u8 fun; - char name[MAX_DEVICE_NAME_SIZE]; -}; - -#define ADF_CTL_IOC_MAGIC 'a' -#define IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS _IOW(ADF_CTL_IOC_MAGIC, 0, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_STOP_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 1, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_START_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 2, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_STATUS_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 3, __u32) -#define IOCTL_GET_NUM_DEVICES _IOW(ADF_CTL_IOC_MAGIC, 4, __s32) -#endif diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h deleted file mode 100644 index 5d8c3bdb258c..000000000000 --- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_CFG_STRINGS_H_ -#define ADF_CFG_STRINGS_H_ - -#define ADF_GENERAL_SEC "GENERAL" -#define ADF_KERNEL_SEC "KERNEL" -#define ADF_ACCEL_SEC "Accelerator" -#define ADF_NUM_CY "NumberCyInstances" -#define ADF_NUM_DC "NumberDcInstances" -#define ADF_RING_SYM_SIZE "NumConcurrentSymRequests" -#define ADF_RING_ASYM_SIZE "NumConcurrentAsymRequests" -#define ADF_RING_DC_SIZE "NumConcurrentRequests" -#define ADF_RING_ASYM_TX "RingAsymTx" -#define ADF_RING_SYM_TX "RingSymTx" -#define ADF_RING_ASYM_RX "RingAsymRx" -#define ADF_RING_SYM_RX "RingSymRx" -#define ADF_RING_DC_TX "RingTx" -#define ADF_RING_DC_RX "RingRx" -#define ADF_ETRMGR_BANK "Bank" -#define ADF_RING_SYM_BANK_NUM "BankSymNumber" -#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber" -#define ADF_RING_DC_BANK_NUM "BankDcNumber" -#define ADF_CY "Cy" -#define ADF_DC "Dc" -#define ADF_CFG_DC "dc" -#define ADF_CFG_CY "sym;asym" -#define ADF_SERVICES_ENABLED "ServicesEnabled" -#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" -#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \ - ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED -#define ADF_ETRMGR_COALESCE_TIMER "InterruptCoalescingTimerNs" -#define ADF_ETRMGR_COALESCE_TIMER_FORMAT \ - ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCE_TIMER -#define ADF_ETRMGR_COALESCING_MSG_ENABLED "InterruptCoalescingNumResponses" -#define ADF_ETRMGR_COALESCING_MSG_ENABLED_FORMAT \ - ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_MSG_ENABLED -#define ADF_ETRMGR_CORE_AFFINITY "CoreAffinity" -#define ADF_ETRMGR_CORE_AFFINITY_FORMAT \ - ADF_ETRMGR_BANK "%d" ADF_ETRMGR_CORE_AFFINITY -#define ADF_ACCEL_STR "Accelerator%d" -#endif diff --git a/drivers/crypto/qat/qat_common/adf_cfg_user.h b/drivers/crypto/qat/qat_common/adf_cfg_user.h deleted file mode 100644 index 421f4fb8b4dd..000000000000 --- a/drivers/crypto/qat/qat_common/adf_cfg_user.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_CFG_USER_H_ -#define ADF_CFG_USER_H_ - -#include "adf_cfg_common.h" -#include "adf_cfg_strings.h" - -struct adf_user_cfg_key_val { - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - union { - struct adf_user_cfg_key_val *next; - __u64 padding3; - }; - enum adf_cfg_val_type type; -} __packed; - -struct adf_user_cfg_section { - char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; - union { - struct adf_user_cfg_key_val *params; - __u64 padding1; - }; - union { - struct adf_user_cfg_section *next; - __u64 padding3; - }; -} __packed; - -struct adf_user_cfg_ctl_data { - union { - struct adf_user_cfg_section *config_section; - __u64 padding; - }; - __u8 device_id; -} __packed; -#endif diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h deleted file mode 100644 index db79759bee61..000000000000 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ /dev/null @@ -1,249 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2021 Intel Corporation */ -#ifndef ADF_DRV_H -#define ADF_DRV_H - -#include -#include -#include "adf_accel_devices.h" -#include "icp_qat_fw_loader_handle.h" -#include "icp_qat_hal.h" - -#define ADF_MAJOR_VERSION 0 -#define ADF_MINOR_VERSION 6 -#define ADF_BUILD_VERSION 0 -#define ADF_DRV_VERSION __stringify(ADF_MAJOR_VERSION) "." \ - __stringify(ADF_MINOR_VERSION) "." \ - __stringify(ADF_BUILD_VERSION) - -#define ADF_STATUS_RESTARTING 0 -#define ADF_STATUS_STARTING 1 -#define ADF_STATUS_CONFIGURED 2 -#define ADF_STATUS_STARTED 3 -#define ADF_STATUS_AE_INITIALISED 4 -#define ADF_STATUS_AE_UCODE_LOADED 5 -#define ADF_STATUS_AE_STARTED 6 -#define ADF_STATUS_PF_RUNNING 7 -#define ADF_STATUS_IRQ_ALLOCATED 8 - -enum adf_dev_reset_mode { - ADF_DEV_RESET_ASYNC = 0, - ADF_DEV_RESET_SYNC -}; - -enum adf_event { - ADF_EVENT_INIT = 0, - ADF_EVENT_START, - ADF_EVENT_STOP, - ADF_EVENT_SHUTDOWN, - ADF_EVENT_RESTARTING, - ADF_EVENT_RESTARTED, -}; - -struct service_hndl { - int (*event_hld)(struct adf_accel_dev *accel_dev, - enum adf_event event); - unsigned long init_status[ADF_DEVS_ARRAY_SIZE]; - unsigned long start_status[ADF_DEVS_ARRAY_SIZE]; - char *name; - struct list_head list; -}; - -int adf_service_register(struct service_hndl *service); -int adf_service_unregister(struct service_hndl *service); - -int adf_dev_up(struct adf_accel_dev *accel_dev, bool init_config); -int adf_dev_down(struct adf_accel_dev *accel_dev, bool cache_config); -int adf_dev_restart(struct adf_accel_dev *accel_dev); - -void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); -void adf_clean_vf_map(bool); - -int adf_ctl_dev_register(void); -void adf_ctl_dev_unregister(void); -int adf_processes_dev_register(void); -void adf_processes_dev_unregister(void); - -int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, - struct adf_accel_dev *pf); -void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, - struct adf_accel_dev *pf); -struct list_head *adf_devmgr_get_head(void); -struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id); -struct adf_accel_dev *adf_devmgr_get_first(void); -struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev); -int adf_devmgr_verify_id(u32 id); -void adf_devmgr_get_num_dev(u32 *num); -int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev); -int adf_dev_started(struct adf_accel_dev *accel_dev); -int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev); -int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev); -int adf_ae_init(struct adf_accel_dev *accel_dev); -int adf_ae_shutdown(struct adf_accel_dev *accel_dev); -int adf_ae_fw_load(struct adf_accel_dev *accel_dev); -void adf_ae_fw_release(struct adf_accel_dev *accel_dev); -int adf_ae_start(struct adf_accel_dev *accel_dev); -int adf_ae_stop(struct adf_accel_dev *accel_dev); - -extern const struct pci_error_handlers adf_err_handler; -void adf_reset_sbr(struct adf_accel_dev *accel_dev); -void adf_reset_flr(struct adf_accel_dev *accel_dev); -void adf_dev_restore(struct adf_accel_dev *accel_dev); -int adf_init_aer(void); -void adf_exit_aer(void); -int adf_init_admin_comms(struct adf_accel_dev *accel_dev); -void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); -int adf_send_admin_init(struct adf_accel_dev *accel_dev); -int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay); -int adf_init_arb(struct adf_accel_dev *accel_dev); -void adf_exit_arb(struct adf_accel_dev *accel_dev); -void adf_update_ring_arb(struct adf_etr_ring_data *ring); - -int adf_dev_get(struct adf_accel_dev *accel_dev); -void adf_dev_put(struct adf_accel_dev *accel_dev); -int adf_dev_in_use(struct adf_accel_dev *accel_dev); -int adf_init_etr_data(struct adf_accel_dev *accel_dev); -void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev); -int qat_crypto_register(void); -int qat_crypto_unregister(void); -int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev); -struct qat_crypto_instance *qat_crypto_get_instance_node(int node); -void qat_crypto_put_instance(struct qat_crypto_instance *inst); -void qat_alg_callback(void *resp); -void qat_alg_asym_callback(void *resp); -int qat_algs_register(void); -void qat_algs_unregister(void); -int qat_asym_algs_register(void); -void qat_asym_algs_unregister(void); - -struct qat_compression_instance *qat_compression_get_instance_node(int node); -void qat_compression_put_instance(struct qat_compression_instance *inst); -int qat_compression_register(void); -int qat_compression_unregister(void); -int qat_comp_algs_register(void); -void qat_comp_algs_unregister(void); -void qat_comp_alg_callback(void *resp); - -int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); -void adf_isr_resource_free(struct adf_accel_dev *accel_dev); -int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); -void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); - -int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev); - -int adf_sysfs_init(struct adf_accel_dev *accel_dev); - -int qat_hal_init(struct adf_accel_dev *accel_dev); -void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); -int qat_hal_start(struct icp_qat_fw_loader_handle *handle); -void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask); -void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); -int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle); -void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask); -int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, - unsigned int ae); -int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, enum icp_qat_uof_regtype lm_type, - unsigned char mode); -int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char mode); -int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char mode); -void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask, unsigned int upc); -void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int uaddr, - unsigned int words_num, u64 *uword); -void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int uword_addr, unsigned int words_num, - unsigned int *data); -int qat_hal_get_ins_num(void); -int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, - struct icp_qat_uof_batch_init *lm_init_header); -int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int regdata); -int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int regdata); -int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int regdata); -int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - unsigned short reg_num, unsigned int regdata); -int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned short lm_addr, unsigned int value); -void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char mode); -int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); -void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); -int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, - int mem_size); -int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, u32 mem_size, char *obj_name); -int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, - unsigned int cfg_ae_mask); -int adf_init_misc_wq(void); -void adf_exit_misc_wq(void); -bool adf_misc_wq_queue_work(struct work_struct *work); -#if defined(CONFIG_PCI_IOV) -int adf_sriov_configure(struct pci_dev *pdev, int numvfs); -void adf_disable_sriov(struct adf_accel_dev *accel_dev); -void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); -void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev); -bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); -bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr); -int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev); -void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); -void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); -void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info); -int adf_init_pf_wq(void); -void adf_exit_pf_wq(void); -int adf_init_vf_wq(void); -void adf_exit_vf_wq(void); -void adf_flush_vf_wq(struct adf_accel_dev *accel_dev); -#else -#define adf_sriov_configure NULL - -static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) -{ -} - -static inline int adf_init_pf_wq(void) -{ - return 0; -} - -static inline void adf_exit_pf_wq(void) -{ -} - -static inline int adf_init_vf_wq(void) -{ - return 0; -} - -static inline void adf_exit_vf_wq(void) -{ -} - -#endif - -static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc; - - pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - - return pmisc->virt_addr; -} - -#endif diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c deleted file mode 100644 index 88c41d6fbb7c..000000000000 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ /dev/null @@ -1,475 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_cfg.h" -#include "adf_cfg_common.h" -#include "adf_cfg_user.h" - -#define ADF_CFG_MAX_SECTION 512 -#define ADF_CFG_MAX_KEY_VAL 256 - -#define DEVICE_NAME "qat_adf_ctl" - -static DEFINE_MUTEX(adf_ctl_lock); -static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); - -static const struct file_operations adf_ctl_ops = { - .owner = THIS_MODULE, - .unlocked_ioctl = adf_ctl_ioctl, - .compat_ioctl = compat_ptr_ioctl, -}; - -struct adf_ctl_drv_info { - unsigned int major; - struct cdev drv_cdev; - struct class *drv_class; -}; - -static struct adf_ctl_drv_info adf_ctl_drv; - -static void adf_chr_drv_destroy(void) -{ - device_destroy(adf_ctl_drv.drv_class, MKDEV(adf_ctl_drv.major, 0)); - cdev_del(&adf_ctl_drv.drv_cdev); - class_destroy(adf_ctl_drv.drv_class); - unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1); -} - -static int adf_chr_drv_create(void) -{ - dev_t dev_id; - struct device *drv_device; - - if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) { - pr_err("QAT: unable to allocate chrdev region\n"); - return -EFAULT; - } - - adf_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME); - if (IS_ERR(adf_ctl_drv.drv_class)) { - pr_err("QAT: class_create failed for adf_ctl\n"); - goto err_chrdev_unreg; - } - adf_ctl_drv.major = MAJOR(dev_id); - cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops); - if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) { - pr_err("QAT: cdev add failed\n"); - goto err_class_destr; - } - - drv_device = device_create(adf_ctl_drv.drv_class, NULL, - MKDEV(adf_ctl_drv.major, 0), - NULL, DEVICE_NAME); - if (IS_ERR(drv_device)) { - pr_err("QAT: failed to create device\n"); - goto err_cdev_del; - } - return 0; -err_cdev_del: - cdev_del(&adf_ctl_drv.drv_cdev); -err_class_destr: - class_destroy(adf_ctl_drv.drv_class); -err_chrdev_unreg: - unregister_chrdev_region(dev_id, 1); - return -EFAULT; -} - -static int adf_ctl_alloc_resources(struct adf_user_cfg_ctl_data **ctl_data, - unsigned long arg) -{ - struct adf_user_cfg_ctl_data *cfg_data; - - cfg_data = kzalloc(sizeof(*cfg_data), GFP_KERNEL); - if (!cfg_data) - return -ENOMEM; - - /* Initialize device id to NO DEVICE as 0 is a valid device id */ - cfg_data->device_id = ADF_CFG_NO_DEVICE; - - if (copy_from_user(cfg_data, (void __user *)arg, sizeof(*cfg_data))) { - pr_err("QAT: failed to copy from user cfg_data.\n"); - kfree(cfg_data); - return -EIO; - } - - *ctl_data = cfg_data; - return 0; -} - -static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, - const char *section, - const struct adf_user_cfg_key_val *key_val) -{ - if (key_val->type == ADF_HEX) { - long *ptr = (long *)key_val->val; - long val = *ptr; - - if (adf_cfg_add_key_value_param(accel_dev, section, - key_val->key, (void *)val, - key_val->type)) { - dev_err(&GET_DEV(accel_dev), - "failed to add hex keyvalue.\n"); - return -EFAULT; - } - } else { - if (adf_cfg_add_key_value_param(accel_dev, section, - key_val->key, key_val->val, - key_val->type)) { - dev_err(&GET_DEV(accel_dev), - "failed to add keyvalue.\n"); - return -EFAULT; - } - } - return 0; -} - -static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, - struct adf_user_cfg_ctl_data *ctl_data) -{ - struct adf_user_cfg_key_val key_val; - struct adf_user_cfg_key_val *params_head; - struct adf_user_cfg_section section, *section_head; - int i, j; - - section_head = ctl_data->config_section; - - for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) { - if (copy_from_user(§ion, (void __user *)section_head, - sizeof(*section_head))) { - dev_err(&GET_DEV(accel_dev), - "failed to copy section info\n"); - goto out_err; - } - - if (adf_cfg_section_add(accel_dev, section.name)) { - dev_err(&GET_DEV(accel_dev), - "failed to add section.\n"); - goto out_err; - } - - params_head = section.params; - - for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) { - if (copy_from_user(&key_val, (void __user *)params_head, - sizeof(key_val))) { - dev_err(&GET_DEV(accel_dev), - "Failed to copy keyvalue.\n"); - goto out_err; - } - if (adf_add_key_value_data(accel_dev, section.name, - &key_val)) { - goto out_err; - } - params_head = key_val.next; - } - section_head = section.next; - } - return 0; -out_err: - adf_cfg_del_all(accel_dev); - return -EFAULT; -} - -static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - struct adf_accel_dev *accel_dev; - - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; - - accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) { - ret = -EFAULT; - goto out; - } - - if (adf_dev_started(accel_dev)) { - ret = -EFAULT; - goto out; - } - - if (adf_copy_key_value_data(accel_dev, ctl_data)) { - ret = -EFAULT; - goto out; - } - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_is_device_in_use(int id) -{ - struct adf_accel_dev *dev; - - list_for_each_entry(dev, adf_devmgr_get_head(), list) { - if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { - dev_info(&GET_DEV(dev), - "device qat_dev%d is busy\n", - dev->accel_id); - return -EBUSY; - } - } - } - return 0; -} - -static void adf_ctl_stop_devices(u32 id) -{ - struct adf_accel_dev *accel_dev; - - list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { - if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (!adf_dev_started(accel_dev)) - continue; - - /* First stop all VFs */ - if (!accel_dev->is_vf) - continue; - - adf_dev_down(accel_dev, false); - } - } - - list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { - if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (!adf_dev_started(accel_dev)) - continue; - - adf_dev_down(accel_dev, false); - } - } -} - -static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; - - if (adf_devmgr_verify_id(ctl_data->device_id)) { - pr_err("QAT: Device %d not found\n", ctl_data->device_id); - ret = -ENODEV; - goto out; - } - - ret = adf_ctl_is_device_in_use(ctl_data->device_id); - if (ret) - goto out; - - if (ctl_data->device_id == ADF_CFG_ALL_DEVICES) - pr_info("QAT: Stopping all acceleration devices.\n"); - else - pr_info("QAT: Stopping acceleration device qat_dev%d.\n", - ctl_data->device_id); - - adf_ctl_stop_devices(ctl_data->device_id); - -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - struct adf_accel_dev *accel_dev; - - ret = adf_ctl_alloc_resources(&ctl_data, arg); - if (ret) - return ret; - - ret = -ENODEV; - accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) - goto out; - - dev_info(&GET_DEV(accel_dev), - "Starting acceleration device qat_dev%d.\n", - ctl_data->device_id); - - ret = adf_dev_up(accel_dev, false); - - if (ret) { - dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", - ctl_data->device_id); - adf_dev_down(accel_dev, false); - } -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - u32 num_devices = 0; - - adf_devmgr_get_num_dev(&num_devices); - if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices))) - return -EFAULT; - - return 0; -} - -static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - struct adf_hw_device_data *hw_data; - struct adf_dev_status_info dev_info; - struct adf_accel_dev *accel_dev; - - if (copy_from_user(&dev_info, (void __user *)arg, - sizeof(struct adf_dev_status_info))) { - pr_err("QAT: failed to copy from user.\n"); - return -EFAULT; - } - - accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); - if (!accel_dev) - return -ENODEV; - - hw_data = accel_dev->hw_device; - dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; - dev_info.num_ae = hw_data->get_num_aes(hw_data); - dev_info.num_accel = hw_data->get_num_accels(hw_data); - dev_info.num_logical_accel = hw_data->num_logical_accel; - dev_info.banks_per_accel = hw_data->num_banks - / hw_data->num_logical_accel; - strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); - dev_info.instance_id = hw_data->instance_id; - dev_info.type = hw_data->dev_class->type; - dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; - dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn); - dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn); - - if (copy_to_user((void __user *)arg, &dev_info, - sizeof(struct adf_dev_status_info))) { - dev_err(&GET_DEV(accel_dev), "failed to copy status.\n"); - return -EFAULT; - } - return 0; -} - -static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) -{ - int ret; - - if (mutex_lock_interruptible(&adf_ctl_lock)) - return -EFAULT; - - switch (cmd) { - case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS: - ret = adf_ctl_ioctl_dev_config(fp, cmd, arg); - break; - - case IOCTL_STOP_ACCEL_DEV: - ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg); - break; - - case IOCTL_START_ACCEL_DEV: - ret = adf_ctl_ioctl_dev_start(fp, cmd, arg); - break; - - case IOCTL_GET_NUM_DEVICES: - ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg); - break; - - case IOCTL_STATUS_ACCEL_DEV: - ret = adf_ctl_ioctl_get_status(fp, cmd, arg); - break; - default: - pr_err_ratelimited("QAT: Invalid ioctl %d\n", cmd); - ret = -EFAULT; - break; - } - mutex_unlock(&adf_ctl_lock); - return ret; -} - -static int __init adf_register_ctl_device_driver(void) -{ - if (adf_chr_drv_create()) - goto err_chr_dev; - - if (adf_init_misc_wq()) - goto err_misc_wq; - - if (adf_init_aer()) - goto err_aer; - - if (adf_init_pf_wq()) - goto err_pf_wq; - - if (adf_init_vf_wq()) - goto err_vf_wq; - - if (qat_crypto_register()) - goto err_crypto_register; - - if (qat_compression_register()) - goto err_compression_register; - - return 0; - -err_compression_register: - qat_crypto_unregister(); -err_crypto_register: - adf_exit_vf_wq(); -err_vf_wq: - adf_exit_pf_wq(); -err_pf_wq: - adf_exit_aer(); -err_aer: - adf_exit_misc_wq(); -err_misc_wq: - adf_chr_drv_destroy(); -err_chr_dev: - mutex_destroy(&adf_ctl_lock); - return -EFAULT; -} - -static void __exit adf_unregister_ctl_device_driver(void) -{ - adf_chr_drv_destroy(); - adf_exit_misc_wq(); - adf_exit_aer(); - adf_exit_vf_wq(); - adf_exit_pf_wq(); - qat_crypto_unregister(); - qat_compression_unregister(); - adf_clean_vf_map(false); - mutex_destroy(&adf_ctl_lock); -} - -module_init(adf_register_ctl_device_driver); -module_exit(adf_unregister_ctl_device_driver); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_ALIAS_CRYPTO("intel_qat"); -MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c deleted file mode 100644 index 86ee36feefad..000000000000 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ /dev/null @@ -1,452 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include "adf_cfg.h" -#include "adf_common_drv.h" - -static LIST_HEAD(accel_table); -static LIST_HEAD(vfs_table); -static DEFINE_MUTEX(table_lock); -static u32 num_devices; -static u8 id_map[ADF_MAX_DEVICES]; - -struct vf_id_map { - u32 bdf; - u32 id; - u32 fake_id; - bool attached; - struct list_head list; -}; - -static int adf_get_vf_id(struct adf_accel_dev *vf) -{ - return ((7 * (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)) + - PCI_FUNC(accel_to_pci_dev(vf)->devfn) + - (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)); -} - -static int adf_get_vf_num(struct adf_accel_dev *vf) -{ - return (accel_to_pci_dev(vf)->bus->number << 8) | adf_get_vf_id(vf); -} - -static struct vf_id_map *adf_find_vf(u32 bdf) -{ - struct list_head *itr; - - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - - if (ptr->bdf == bdf) - return ptr; - } - return NULL; -} - -static int adf_get_vf_real_id(u32 fake) -{ - struct list_head *itr; - - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - if (ptr->fake_id == fake) - return ptr->id; - } - return -1; -} - -/** - * adf_clean_vf_map() - Cleans VF id mapings - * - * Function cleans internal ids for virtual functions. - * @vf: flag indicating whether mappings is cleaned - * for vfs only or for vfs and pfs - */ -void adf_clean_vf_map(bool vf) -{ - struct vf_id_map *map; - struct list_head *ptr, *tmp; - - mutex_lock(&table_lock); - list_for_each_safe(ptr, tmp, &vfs_table) { - map = list_entry(ptr, struct vf_id_map, list); - if (map->bdf != -1) { - id_map[map->id] = 0; - num_devices--; - } - - if (vf && map->bdf == -1) - continue; - - list_del(ptr); - kfree(map); - } - mutex_unlock(&table_lock); -} -EXPORT_SYMBOL_GPL(adf_clean_vf_map); - -/** - * adf_devmgr_update_class_index() - Update internal index - * @hw_data: Pointer to internal device data. - * - * Function updates internal dev index for VFs - */ -void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data) -{ - struct adf_hw_device_class *class = hw_data->dev_class; - struct list_head *itr; - int i = 0; - - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = - list_entry(itr, struct adf_accel_dev, list); - - if (ptr->hw_device->dev_class == class) - ptr->hw_device->instance_id = i++; - - if (i == class->instances) - break; - } -} -EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index); - -static unsigned int adf_find_free_id(void) -{ - unsigned int i; - - for (i = 0; i < ADF_MAX_DEVICES; i++) { - if (!id_map[i]) { - id_map[i] = 1; - return i; - } - } - return ADF_MAX_DEVICES + 1; -} - -/** - * adf_devmgr_add_dev() - Add accel_dev to the acceleration framework - * @accel_dev: Pointer to acceleration device. - * @pf: Corresponding PF if the accel_dev is a VF - * - * Function adds acceleration device to the acceleration framework. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ -int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, - struct adf_accel_dev *pf) -{ - struct list_head *itr; - int ret = 0; - - if (num_devices == ADF_MAX_DEVICES) { - dev_err(&GET_DEV(accel_dev), "Only support up to %d devices\n", - ADF_MAX_DEVICES); - return -EFAULT; - } - - mutex_lock(&table_lock); - atomic_set(&accel_dev->ref_count, 0); - - /* PF on host or VF on guest - optimized to remove redundant is_vf */ - if (!accel_dev->is_vf || !pf) { - struct vf_id_map *map; - - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = - list_entry(itr, struct adf_accel_dev, list); - - if (ptr == accel_dev) { - ret = -EEXIST; - goto unlock; - } - } - - list_add_tail(&accel_dev->list, &accel_table); - accel_dev->accel_id = adf_find_free_id(); - if (accel_dev->accel_id > ADF_MAX_DEVICES) { - ret = -EFAULT; - goto unlock; - } - num_devices++; - map = kzalloc(sizeof(*map), GFP_KERNEL); - if (!map) { - ret = -ENOMEM; - goto unlock; - } - map->bdf = ~0; - map->id = accel_dev->accel_id; - map->fake_id = map->id; - map->attached = true; - list_add_tail(&map->list, &vfs_table); - } else if (accel_dev->is_vf && pf) { - /* VF on host */ - struct vf_id_map *map; - - map = adf_find_vf(adf_get_vf_num(accel_dev)); - if (map) { - struct vf_id_map *next; - - accel_dev->accel_id = map->id; - list_add_tail(&accel_dev->list, &accel_table); - map->fake_id++; - map->attached = true; - next = list_next_entry(map, list); - while (next && &next->list != &vfs_table) { - next->fake_id++; - next = list_next_entry(next, list); - } - - ret = 0; - goto unlock; - } - - map = kzalloc(sizeof(*map), GFP_KERNEL); - if (!map) { - ret = -ENOMEM; - goto unlock; - } - accel_dev->accel_id = adf_find_free_id(); - if (accel_dev->accel_id > ADF_MAX_DEVICES) { - kfree(map); - ret = -EFAULT; - goto unlock; - } - num_devices++; - list_add_tail(&accel_dev->list, &accel_table); - map->bdf = adf_get_vf_num(accel_dev); - map->id = accel_dev->accel_id; - map->fake_id = map->id; - map->attached = true; - list_add_tail(&map->list, &vfs_table); - } - mutex_init(&accel_dev->state_lock); -unlock: - mutex_unlock(&table_lock); - return ret; -} -EXPORT_SYMBOL_GPL(adf_devmgr_add_dev); - -struct list_head *adf_devmgr_get_head(void) -{ - return &accel_table; -} - -/** - * adf_devmgr_rm_dev() - Remove accel_dev from the acceleration framework. - * @accel_dev: Pointer to acceleration device. - * @pf: Corresponding PF if the accel_dev is a VF - * - * Function removes acceleration device from the acceleration framework. - * To be used by QAT device specific drivers. - * - * Return: void - */ -void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, - struct adf_accel_dev *pf) -{ - mutex_lock(&table_lock); - /* PF on host or VF on guest - optimized to remove redundant is_vf */ - if (!accel_dev->is_vf || !pf) { - id_map[accel_dev->accel_id] = 0; - num_devices--; - } else if (accel_dev->is_vf && pf) { - struct vf_id_map *map, *next; - - map = adf_find_vf(adf_get_vf_num(accel_dev)); - if (!map) { - dev_err(&GET_DEV(accel_dev), "Failed to find VF map\n"); - goto unlock; - } - map->fake_id--; - map->attached = false; - next = list_next_entry(map, list); - while (next && &next->list != &vfs_table) { - next->fake_id--; - next = list_next_entry(next, list); - } - } -unlock: - mutex_destroy(&accel_dev->state_lock); - list_del(&accel_dev->list); - mutex_unlock(&table_lock); -} -EXPORT_SYMBOL_GPL(adf_devmgr_rm_dev); - -struct adf_accel_dev *adf_devmgr_get_first(void) -{ - struct adf_accel_dev *dev = NULL; - - if (!list_empty(&accel_table)) - dev = list_first_entry(&accel_table, struct adf_accel_dev, - list); - return dev; -} - -/** - * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. - * @pci_dev: Pointer to PCI device. - * - * Function returns acceleration device associated with the given PCI device. - * To be used by QAT device specific drivers. - * - * Return: pointer to accel_dev or NULL if not found. - */ -struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev) -{ - struct list_head *itr; - - mutex_lock(&table_lock); - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = - list_entry(itr, struct adf_accel_dev, list); - - if (ptr->accel_pci_dev.pci_dev == pci_dev) { - mutex_unlock(&table_lock); - return ptr; - } - } - mutex_unlock(&table_lock); - return NULL; -} -EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); - -struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id) -{ - struct list_head *itr; - int real_id; - - mutex_lock(&table_lock); - real_id = adf_get_vf_real_id(id); - if (real_id < 0) - goto unlock; - - id = real_id; - - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = - list_entry(itr, struct adf_accel_dev, list); - if (ptr->accel_id == id) { - mutex_unlock(&table_lock); - return ptr; - } - } -unlock: - mutex_unlock(&table_lock); - return NULL; -} - -int adf_devmgr_verify_id(u32 id) -{ - if (id == ADF_CFG_ALL_DEVICES) - return 0; - - if (adf_devmgr_get_dev_by_id(id)) - return 0; - - return -ENODEV; -} - -static int adf_get_num_dettached_vfs(void) -{ - struct list_head *itr; - int vfs = 0; - - mutex_lock(&table_lock); - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - if (ptr->bdf != ~0 && !ptr->attached) - vfs++; - } - mutex_unlock(&table_lock); - return vfs; -} - -void adf_devmgr_get_num_dev(u32 *num) -{ - *num = num_devices - adf_get_num_dettached_vfs(); -} - -/** - * adf_dev_in_use() - Check whether accel_dev is currently in use - * @accel_dev: Pointer to acceleration device. - * - * To be used by QAT device specific drivers. - * - * Return: 1 when device is in use, 0 otherwise. - */ -int adf_dev_in_use(struct adf_accel_dev *accel_dev) -{ - return atomic_read(&accel_dev->ref_count) != 0; -} -EXPORT_SYMBOL_GPL(adf_dev_in_use); - -/** - * adf_dev_get() - Increment accel_dev reference count - * @accel_dev: Pointer to acceleration device. - * - * Increment the accel_dev refcount and if this is the first time - * incrementing it during this period the accel_dev is in use, - * increment the module refcount too. - * To be used by QAT device specific drivers. - * - * Return: 0 when successful, EFAULT when fail to bump module refcount - */ -int adf_dev_get(struct adf_accel_dev *accel_dev) -{ - if (atomic_add_return(1, &accel_dev->ref_count) == 1) - if (!try_module_get(accel_dev->owner)) - return -EFAULT; - return 0; -} -EXPORT_SYMBOL_GPL(adf_dev_get); - -/** - * adf_dev_put() - Decrement accel_dev reference count - * @accel_dev: Pointer to acceleration device. - * - * Decrement the accel_dev refcount and if this is the last time - * decrementing it during this period the accel_dev is in use, - * decrement the module refcount too. - * To be used by QAT device specific drivers. - * - * Return: void - */ -void adf_dev_put(struct adf_accel_dev *accel_dev) -{ - if (atomic_sub_return(1, &accel_dev->ref_count) == 0) - module_put(accel_dev->owner); -} -EXPORT_SYMBOL_GPL(adf_dev_put); - -/** - * adf_devmgr_in_reset() - Check whether device is in reset - * @accel_dev: Pointer to acceleration device. - * - * To be used by QAT device specific drivers. - * - * Return: 1 when the device is being reset, 0 otherwise. - */ -int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev) -{ - return test_bit(ADF_STATUS_RESTARTING, &accel_dev->status); -} -EXPORT_SYMBOL_GPL(adf_devmgr_in_reset); - -/** - * adf_dev_started() - Check whether device has started - * @accel_dev: Pointer to acceleration device. - * - * To be used by QAT device specific drivers. - * - * Return: 1 when the device has started, 0 otherwise - */ -int adf_dev_started(struct adf_accel_dev *accel_dev) -{ - return test_bit(ADF_STATUS_STARTED, &accel_dev->status); -} -EXPORT_SYMBOL_GPL(adf_dev_started); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_config.c b/drivers/crypto/qat/qat_common/adf_gen2_config.c deleted file mode 100644 index eeb30da7587a..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_config.c +++ /dev/null @@ -1,206 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include "adf_accel_devices.h" -#include "adf_cfg.h" -#include "adf_cfg_strings.h" -#include "adf_gen2_config.h" -#include "adf_common_drv.h" -#include "qat_crypto.h" -#include "qat_compression.h" -#include "adf_transport_access_macros.h" - -static int adf_gen2_crypto_dev_config(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int banks = GET_MAX_BANKS(accel_dev); - int cpus = num_online_cpus(); - unsigned long val; - int instances; - int ret; - int i; - - if (adf_hw_dev_has_crypto(accel_dev)) - instances = min(cpus, banks); - else - instances = 0; - - for (i = 0; i < instances; i++) { - val = i; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, - i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - val = 128; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 2; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 8; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 10; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC); - if (ret) - goto err; - } - - val = i; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, - &val, ADF_DEC); - if (ret) - goto err; - - return ret; - -err: - dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); - return ret; -} - -static int adf_gen2_comp_dev_config(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int banks = GET_MAX_BANKS(accel_dev); - int cpus = num_online_cpus(); - unsigned long val; - int instances; - int ret; - int i; - - if (adf_hw_dev_has_compression(accel_dev)) - instances = min(cpus, banks); - else - instances = 0; - - for (i = 0; i < instances; i++) { - val = i; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 6; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 14; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - } - - val = i; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - return ret; - - return ret; - -err: - dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); - return ret; -} - -/** - * adf_gen2_dev_config() - create dev config required to create instances - * - * @accel_dev: Pointer to acceleration device. - * - * Function creates device configuration required to create instances - * - * Return: 0 on success, error code otherwise. - */ -int adf_gen2_dev_config(struct adf_accel_dev *accel_dev) -{ - int ret; - - ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); - if (ret) - goto err; - - ret = adf_cfg_section_add(accel_dev, "Accelerator0"); - if (ret) - goto err; - - ret = adf_gen2_crypto_dev_config(accel_dev); - if (ret) - goto err; - - ret = adf_gen2_comp_dev_config(accel_dev); - if (ret) - goto err; - - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - - return ret; - -err: - dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); - return ret; -} -EXPORT_SYMBOL_GPL(adf_gen2_dev_config); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_config.h b/drivers/crypto/qat/qat_common/adf_gen2_config.h deleted file mode 100644 index 4bf9da2de68a..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_config.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN2_CONFIG_H_ -#define ADF_GEN2_CONFIG_H_ - -#include "adf_accel_devices.h" - -int adf_gen2_dev_config(struct adf_accel_dev *accel_dev); - -#endif diff --git a/drivers/crypto/qat/qat_common/adf_gen2_dc.c b/drivers/crypto/qat/qat_common/adf_gen2_dc.c deleted file mode 100644 index 47261b1c1da6..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_dc.c +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include "adf_accel_devices.h" -#include "adf_gen2_dc.h" -#include "icp_qat_fw_comp.h" - -static void qat_comp_build_deflate_ctx(void *ctx) -{ - struct icp_qat_fw_comp_req *req_tmpl = (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; - struct icp_qat_fw_comp_cd_hdr *comp_cd_ctrl = &req_tmpl->comp_cd_ctrl; - - memset(req_tmpl, 0, sizeof(*req_tmpl)); - header->hdr_flags = - ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); - header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; - header->comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, - QAT_COMN_PTR_TYPE_SGL); - header->serv_specif_flags = - ICP_QAT_FW_COMP_FLAGS_BUILD(ICP_QAT_FW_COMP_STATELESS_SESSION, - ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); - req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; - req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; - req_pars->req_par_flags = - ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(ICP_QAT_FW_COMP_SOP, - ICP_QAT_FW_COMP_EOP, - ICP_QAT_FW_COMP_BFINAL, - ICP_QAT_FW_COMP_CNV, - ICP_QAT_FW_COMP_CNV_RECOVERY, - ICP_QAT_FW_COMP_NO_CNV_DFX, - ICP_QAT_FW_COMP_CRC_MODE_LEGACY, - ICP_QAT_FW_COMP_NO_XXHASH_ACC, - ICP_QAT_FW_COMP_CNV_ERROR_NONE, - ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); - ICP_QAT_FW_COMN_NEXT_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); - ICP_QAT_FW_COMN_CURR_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_COMP); - - /* Fill second half of the template for decompression */ - memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); - req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); -} - -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate_ctx; -} -EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_dc.h b/drivers/crypto/qat/qat_common/adf_gen2_dc.h deleted file mode 100644 index 6eae023354d7..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN2_DC_H -#define ADF_GEN2_DC_H - -#include "adf_accel_devices.h" - -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN2_DC_H */ diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c deleted file mode 100644 index d1884547b5a1..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2020 Intel Corporation */ -#include "adf_common_drv.h" -#include "adf_gen2_hw_data.h" -#include "icp_qat_hw.h" -#include - -u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self) -{ - if (!self || !self->accel_mask) - return 0; - - return hweight16(self->accel_mask); -} -EXPORT_SYMBOL_GPL(adf_gen2_get_num_accels); - -u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self) -{ - if (!self || !self->ae_mask) - return 0; - - return hweight32(self->ae_mask); -} -EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes); - -void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - unsigned long accel_mask = hw_data->accel_mask; - unsigned long ae_mask = hw_data->ae_mask; - unsigned int val, i; - - /* Enable Accel Engine error detection & correction */ - for_each_set_bit(i, &ae_mask, hw_data->num_engines) { - val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i)); - val |= ADF_GEN2_ENABLE_AE_ECC_ERR; - ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i), val); - val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i)); - val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR; - ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i), val); - } - - /* Enable shared memory error detection & correction */ - for_each_set_bit(i, &accel_mask, hw_data->num_accel) { - val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_UERRSSMSH(i)); - val |= ADF_GEN2_ERRSSMSH_EN; - ADF_CSR_WR(pmisc_addr, ADF_GEN2_UERRSSMSH(i), val); - val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_CERRSSMSH(i)); - val |= ADF_GEN2_ERRSSMSH_EN; - ADF_CSR_WR(pmisc_addr, ADF_GEN2_CERRSSMSH(i), val); - } -} -EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction); - -void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, - int num_a_regs, int num_b_regs) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - u32 reg; - int i; - - /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */ - for (i = 0; i < num_a_regs; i++) { - reg = READ_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i); - if (enable) - reg |= AE2FUNCTION_MAP_VALID; - else - reg &= ~AE2FUNCTION_MAP_VALID; - WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group B */ - for (i = 0; i < num_b_regs; i++) { - reg = READ_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i); - if (enable) - reg |= AE2FUNCTION_MAP_VALID; - else - reg &= ~AE2FUNCTION_MAP_VALID; - WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i, reg); - } -} -EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); - -void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) -{ - admin_csrs_info->mailbox_offset = ADF_MAILBOX_BASE_OFFSET; - admin_csrs_info->admin_msg_ur = ADF_ADMINMSGUR_OFFSET; - admin_csrs_info->admin_msg_lr = ADF_ADMINMSGLR_OFFSET; -} -EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); - -void adf_gen2_get_arb_info(struct arb_info *arb_info) -{ - arb_info->arb_cfg = ADF_ARB_CONFIG; - arb_info->arb_offset = ADF_ARB_OFFSET; - arb_info->wt2sam_offset = ADF_ARB_WRK_2_SER_MAP_OFFSET; -} -EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); - -void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev) -{ - void __iomem *addr = adf_get_pmisc_base(accel_dev); - u32 val; - - val = accel_dev->pf.vf_info ? 0 : BIT_ULL(GET_MAX_BANKS(accel_dev)) - 1; - - /* Enable bundle and misc interrupts */ - ADF_CSR_WR(addr, ADF_GEN2_SMIAPF0_MASK_OFFSET, val); - ADF_CSR_WR(addr, ADF_GEN2_SMIAPF1_MASK_OFFSET, ADF_GEN2_SMIA1_MASK); -} -EXPORT_SYMBOL_GPL(adf_gen2_enable_ints); - -static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) -{ - return BUILD_RING_BASE_ADDR(addr, size); -} - -static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); -} - -static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); -} - -static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) -{ - return READ_CSR_E_STAT(csr_base_addr, bank); -} - -static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, - u32 ring, u32 value) -{ - WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); -} - -static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, - dma_addr_t addr) -{ - WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); -} - -static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) -{ - WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); -} - -static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) -{ - WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); -} - -static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); -} - -static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); -} - -static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); -} - -static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); -} - -void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) -{ - csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; - csr_ops->read_csr_ring_head = read_csr_ring_head; - csr_ops->write_csr_ring_head = write_csr_ring_head; - csr_ops->read_csr_ring_tail = read_csr_ring_tail; - csr_ops->write_csr_ring_tail = write_csr_ring_tail; - csr_ops->read_csr_e_stat = read_csr_e_stat; - csr_ops->write_csr_ring_config = write_csr_ring_config; - csr_ops->write_csr_ring_base = write_csr_ring_base; - csr_ops->write_csr_int_flag = write_csr_int_flag; - csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; - csr_ops->write_csr_int_col_en = write_csr_int_col_en; - csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; - csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; - csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; -} -EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); - -u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; - u32 straps = hw_data->straps; - u32 fuses = hw_data->fuses; - u32 legfuses; - u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | - ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | - ICP_ACCEL_CAPABILITIES_AUTHENTICATION | - ICP_ACCEL_CAPABILITIES_CIPHER | - ICP_ACCEL_CAPABILITIES_COMPRESSION; - - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); - - /* A set bit in legfuses means the feature is OFF in this SKU */ - if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; - if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) - capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; - - if ((straps | fuses) & ADF_POWERGATE_PKE) - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; - - if ((straps | fuses) & ADF_POWERGATE_DC) - capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; - - return capabilities; -} -EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); - -void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; - u32 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; - unsigned long accel_mask = hw_data->accel_mask; - u32 i = 0; - - /* Configures WDT timers */ - for_each_set_bit(i, &accel_mask, hw_data->num_accel) { - /* Enable WDT for sym and dc */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDT(i), timer_val); - /* Enable WDT for pke */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKE(i), timer_val_pke); - } -} -EXPORT_SYMBOL_GPL(adf_gen2_set_ssm_wdtimer); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h deleted file mode 100644 index e4bc07529be4..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ /dev/null @@ -1,165 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2020 Intel Corporation */ -#ifndef ADF_GEN2_HW_DATA_H_ -#define ADF_GEN2_HW_DATA_H_ - -#include "adf_accel_devices.h" -#include "adf_cfg_common.h" - -/* Transport access */ -#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL -#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL -#define ADF_RING_CSR_RING_CONFIG 0x000 -#define ADF_RING_CSR_RING_LBASE 0x040 -#define ADF_RING_CSR_RING_UBASE 0x080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_SRCSEL_2 0x178 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_BUNDLE_SIZE 0x1000 -#define ADF_GEN2_RX_RINGS_OFFSET 8 -#define ADF_GEN2_TX_RINGS_MASK 0xFF - -#define BUILD_RING_BASE_ADDR(addr, size) \ - (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - u32 l_base = 0, u_base = 0; \ - l_base = (u32)((value) & 0xFFFFFFFF); \ - u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ -} while (0) - -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG, value) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ -do { \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ -} while (0) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_COL_EN, value) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, value) - -/* AE to function map */ -#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) -#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) -#define AE2FUNCTION_MAP_REG_SIZE 4 -#define AE2FUNCTION_MAP_VALID BIT(7) - -#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ - AE2FUNCTION_MAP_REG_SIZE * (index)) -#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ - AE2FUNCTION_MAP_REG_SIZE * (index), value) -#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ - AE2FUNCTION_MAP_REG_SIZE * (index)) -#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ - AE2FUNCTION_MAP_REG_SIZE * (index), value) - -/* Admin Interface Offsets */ -#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_MAILBOX_BASE_OFFSET 0x20970 - -/* Arbiter configuration */ -#define ADF_ARB_OFFSET 0x30000 -#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 -#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C - -#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * (index)), value) - -/* Power gating */ -#define ADF_POWERGATE_DC BIT(23) -#define ADF_POWERGATE_PKE BIT(24) - -/* Default ring mapping */ -#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \ - (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ - CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ - UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ - COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) - -/* WDT timers - * - * Timeout is in cycles. Clock speed may vary across products but this - * value should be a few milli-seconds. - */ -#define ADF_SSM_WDT_DEFAULT_VALUE 0x200000 -#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x2000000 -#define ADF_SSMWDT_OFFSET 0x54 -#define ADF_SSMWDTPKE_OFFSET 0x58 -#define ADF_SSMWDT(i) (ADF_SSMWDT_OFFSET + ((i) * 0x4000)) -#define ADF_SSMWDTPKE(i) (ADF_SSMWDTPKE_OFFSET + ((i) * 0x4000)) - -/* Error detection and correction */ -#define ADF_GEN2_AE_CTX_ENABLES(i) ((i) * 0x1000 + 0x20818) -#define ADF_GEN2_AE_MISC_CONTROL(i) ((i) * 0x1000 + 0x20960) -#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28) -#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) -#define ADF_GEN2_UERRSSMSH(i) ((i) * 0x4000 + 0x18) -#define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10) -#define ADF_GEN2_ERRSSMSH_EN BIT(3) - -/* Interrupts */ -#define ADF_GEN2_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) -#define ADF_GEN2_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) -#define ADF_GEN2_SMIA1_MASK 0x1 - -u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self); -u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self); -void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev); -void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, - int num_a_regs, int num_b_regs); -void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); -void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); -void adf_gen2_get_arb_info(struct arb_info *arb_info); -void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev); -u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); -void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); - -#endif diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c deleted file mode 100644 index 70ef11963938..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c +++ /dev/null @@ -1,399 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2021 Intel Corporation */ -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_gen2_pfvf.h" -#include "adf_pfvf_msg.h" -#include "adf_pfvf_pf_proto.h" -#include "adf_pfvf_vf_proto.h" -#include "adf_pfvf_utils.h" - - /* VF2PF interrupts */ -#define ADF_GEN2_VF_MSK 0xFFFF -#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) -#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & ADF_GEN2_VF_MSK) << 9) - -#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) -#define ADF_GEN2_VF_PF2VF_OFFSET 0x200 - -#define ADF_GEN2_CSR_IN_USE 0x6AC2 -#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE - -enum gen2_csr_pos { - ADF_GEN2_CSR_PF2VF_OFFSET = 0, - ADF_GEN2_CSR_VF2PF_OFFSET = 16, -}; - -#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2 -#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F -#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6 -#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF - -static const struct pfvf_csr_format csr_gen2_fmt = { - { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK }, - { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, -}; - -#define ADF_PFVF_MSG_RETRY_DELAY 5 -#define ADF_PFVF_MSG_MAX_RETRIES 3 - -static u32 adf_gen2_pf_get_pfvf_offset(u32 i) -{ - return ADF_GEN2_PF_PF2VF_OFFSET(i); -} - -static u32 adf_gen2_vf_get_pfvf_offset(u32 i) -{ - return ADF_GEN2_VF_PF2VF_OFFSET; -} - -static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) -{ - /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ - if (vf_mask & ADF_GEN2_VF_MSK) { - u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) - & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); - } -} - -static void adf_gen2_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) -{ - /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ - u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) - | ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); -} - -static u32 adf_gen2_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) -{ - u32 sources, disabled, pending; - u32 errsou3, errmsk3; - - /* Get the interrupt sources triggered by VFs */ - errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); - sources = ADF_GEN2_ERR_REG_VF2PF(errsou3); - - if (!sources) - return 0; - - /* Get the already disabled interrupts */ - errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); - disabled = ADF_GEN2_ERR_REG_VF2PF(errmsk3); - - pending = sources & ~disabled; - if (!pending) - return 0; - - /* Due to HW limitations, when disabling the interrupts, we can't - * just disable the requested sources, as this would lead to missed - * interrupts if ERRSOU3 changes just before writing to ERRMSK3. - * To work around it, disable all and re-enable only the sources that - * are not in vf_mask and were not already disabled. Re-enabling will - * trigger a new interrupt for the sources that have changed in the - * meantime, if any. - */ - errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); - - errmsk3 &= ADF_GEN2_ERR_MSK_VF2PF(sources | disabled); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); - - /* Return the sources of the (new) interrupt(s) */ - return pending; -} - -static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset) -{ - return ADF_PFVF_INT << offset; -} - -static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset) -{ - return (csr_msg & 0xFFFF) << offset; -} - -static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset) -{ - return (csr_val >> offset) & 0xFFFF; -} - -static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset) -{ - return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE; -} - -static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset) -{ - *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset); -} - -static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset) -{ - *msg |= (ADF_GEN2_CSR_IN_USE << offset); -} - -static bool is_legacy_user_pfvf_message(u32 msg) -{ - return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); -} - -static bool is_pf2vf_notification(u8 msg_type) -{ - switch (msg_type) { - case ADF_PF2VF_MSGTYPE_RESTARTING: - return true; - default: - return false; - } -} - -static bool is_vf2pf_notification(u8 msg_type) -{ - switch (msg_type) { - case ADF_VF2PF_MSGTYPE_INIT: - case ADF_VF2PF_MSGTYPE_SHUTDOWN: - return true; - default: - return false; - } -} - -struct pfvf_gen2_params { - u32 pfvf_offset; - struct mutex *csr_lock; /* lock preventing concurrent access of CSR */ - enum gen2_csr_pos local_offset; - enum gen2_csr_pos remote_offset; - bool (*is_notification_message)(u8 msg_type); - u8 compat_ver; -}; - -static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, - struct pfvf_message msg, - struct pfvf_gen2_params *params) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - enum gen2_csr_pos remote_offset = params->remote_offset; - enum gen2_csr_pos local_offset = params->local_offset; - unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; - struct mutex *lock = params->csr_lock; - u32 pfvf_offset = params->pfvf_offset; - u32 int_bit; - u32 csr_val; - u32 csr_msg; - int ret; - - /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This - * allows us to build and read messages as if they where all 0 based. - * However, send and receive are in a single shared 32 bits register, - * so we need to shift and/or mask the message half before decoding - * it and after encoding it. Which one to shift depends on the - * direction. - */ - - int_bit = gen2_csr_get_int_bit(local_offset); - - csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt); - if (unlikely(!csr_msg)) - return -EINVAL; - - /* Prepare for CSR format, shifting the wire message in place and - * setting the in use pattern - */ - csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset); - gen2_csr_set_in_use(&csr_msg, remote_offset); - - mutex_lock(lock); - -start: - /* Check if the PFVF CSR is in use by remote function */ - csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); - if (gen2_csr_is_in_use(csr_val, local_offset)) { - dev_dbg(&GET_DEV(accel_dev), - "PFVF CSR in use by remote function\n"); - goto retry; - } - - /* Attempt to get ownership of the PFVF CSR */ - ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit); - - /* Wait for confirmation from remote func it received the message */ - ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit), - ADF_PFVF_MSG_ACK_DELAY_US, - ADF_PFVF_MSG_ACK_MAX_DELAY_US, - true, pmisc_addr, pfvf_offset); - if (unlikely(ret < 0)) { - dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); - csr_val &= ~int_bit; - } - - /* For fire-and-forget notifications, the receiver does not clear - * the in-use pattern. This is used to detect collisions. - */ - if (params->is_notification_message(msg.type) && csr_val != csr_msg) { - /* Collision must have overwritten the message */ - dev_err(&GET_DEV(accel_dev), - "Collision on notification - PFVF CSR overwritten by remote function\n"); - goto retry; - } - - /* If the far side did not clear the in-use pattern it is either - * 1) Notification - message left intact to detect collision - * 2) Older protocol (compatibility version < 3) on the far side - * where the sender is responsible for clearing the in-use - * pattern after the received has acknowledged receipt. - * In either case, clear the in-use pattern now. - */ - if (gen2_csr_is_in_use(csr_val, remote_offset)) { - gen2_csr_clear_in_use(&csr_val, remote_offset); - ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); - } - -out: - mutex_unlock(lock); - return ret; - -retry: - if (--retries) { - msleep(ADF_PFVF_MSG_RETRY_DELAY); - goto start; - } else { - ret = -EBUSY; - goto out; - } -} - -static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, - struct pfvf_gen2_params *params) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - enum gen2_csr_pos remote_offset = params->remote_offset; - enum gen2_csr_pos local_offset = params->local_offset; - u32 pfvf_offset = params->pfvf_offset; - struct pfvf_message msg = { 0 }; - u32 int_bit; - u32 csr_val; - u16 csr_msg; - - int_bit = gen2_csr_get_int_bit(local_offset); - - /* Read message */ - csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); - if (!(csr_val & int_bit)) { - dev_info(&GET_DEV(accel_dev), - "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); - return msg; - } - - /* Extract the message from the CSR */ - csr_msg = gen2_csr_msg_from_position(csr_val, local_offset); - - /* Ignore legacy non-system (non-kernel) messages */ - if (unlikely(is_legacy_user_pfvf_message(csr_msg))) { - dev_dbg(&GET_DEV(accel_dev), - "Ignored non-system message (0x%.8x);\n", csr_val); - /* Because this must be a legacy message, the far side - * must clear the in-use pattern, so don't do it. - */ - return msg; - } - - /* Return the pfvf_message format */ - msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt); - - /* The in-use pattern is not cleared for notifications (so that - * it can be used for collision detection) or older implementations - */ - if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK && - !params->is_notification_message(msg.type)) - gen2_csr_clear_in_use(&csr_val, remote_offset); - - /* To ACK, clear the INT bit */ - csr_val &= ~int_bit; - ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); - - return msg; -} - -static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, - u32 pfvf_offset, struct mutex *csr_lock) -{ - struct pfvf_gen2_params params = { - .csr_lock = csr_lock, - .pfvf_offset = pfvf_offset, - .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, - .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, - .is_notification_message = is_pf2vf_notification, - }; - - return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); -} - -static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, - u32 pfvf_offset, struct mutex *csr_lock) -{ - struct pfvf_gen2_params params = { - .csr_lock = csr_lock, - .pfvf_offset = pfvf_offset, - .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, - .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, - .is_notification_message = is_vf2pf_notification, - }; - - return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); -} - -static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, - u32 pfvf_offset, u8 compat_ver) -{ - struct pfvf_gen2_params params = { - .pfvf_offset = pfvf_offset, - .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, - .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, - .is_notification_message = is_pf2vf_notification, - .compat_ver = compat_ver, - }; - - return adf_gen2_pfvf_recv(accel_dev, ¶ms); -} - -static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, - u32 pfvf_offset, u8 compat_ver) -{ - struct pfvf_gen2_params params = { - .pfvf_offset = pfvf_offset, - .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, - .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, - .is_notification_message = is_vf2pf_notification, - .compat_ver = compat_ver, - }; - - return adf_gen2_pfvf_recv(accel_dev, ¶ms); -} - -void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) -{ - pfvf_ops->enable_comms = adf_enable_pf2vf_comms; - pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset; - pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset; - pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; - pfvf_ops->disable_all_vf2pf_interrupts = adf_gen2_disable_all_vf2pf_interrupts; - pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen2_disable_pending_vf2pf_interrupts; - pfvf_ops->send_msg = adf_gen2_pf2vf_send; - pfvf_ops->recv_msg = adf_gen2_vf2pf_recv; -} -EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); - -void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) -{ - pfvf_ops->enable_comms = adf_enable_vf2pf_comms; - pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; - pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; - pfvf_ops->send_msg = adf_gen2_vf2pf_send; - pfvf_ops->recv_msg = adf_gen2_pf2vf_recv; -} -EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h deleted file mode 100644 index a716545a764c..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_GEN2_PFVF_H -#define ADF_GEN2_PFVF_H - -#include -#include "adf_accel_devices.h" - -#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) -#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) -#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) -#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) - -#if defined(CONFIG_PCI_IOV) -void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); -void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); -#else -static inline void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) -{ - pfvf_ops->enable_comms = adf_pfvf_comms_disabled; -} - -static inline void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) -{ - pfvf_ops->enable_comms = adf_pfvf_comms_disabled; -} -#endif - -#endif /* ADF_GEN2_PFVF_H */ diff --git a/drivers/crypto/qat/qat_common/adf_gen4_dc.c b/drivers/crypto/qat/qat_common/adf_gen4_dc.c deleted file mode 100644 index 5859238e37de..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_dc.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include "adf_accel_devices.h" -#include "icp_qat_fw_comp.h" -#include "icp_qat_hw_20_comp.h" -#include "adf_gen4_dc.h" - -static void qat_comp_build_deflate(void *ctx) -{ - struct icp_qat_fw_comp_req *req_tmpl = - (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; - struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = {0}; - struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = {0}; - struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = {0}; - u32 upper_val; - u32 lower_val; - - memset(req_tmpl, 0, sizeof(*req_tmpl)); - header->hdr_flags = - ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); - header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; - header->comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, - QAT_COMN_PTR_TYPE_SGL); - header->serv_specif_flags = - ICP_QAT_FW_COMP_FLAGS_BUILD(ICP_QAT_FW_COMP_STATELESS_SESSION, - ICP_QAT_FW_COMP_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; - hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; - hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; - hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; - hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; - hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; - hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; - hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; - - upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); - lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; - - req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; - req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; - req_pars->req_par_flags = - ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(ICP_QAT_FW_COMP_SOP, - ICP_QAT_FW_COMP_EOP, - ICP_QAT_FW_COMP_BFINAL, - ICP_QAT_FW_COMP_CNV, - ICP_QAT_FW_COMP_CNV_RECOVERY, - ICP_QAT_FW_COMP_NO_CNV_DFX, - ICP_QAT_FW_COMP_CRC_MODE_LEGACY, - ICP_QAT_FW_COMP_NO_XXHASH_ACC, - ICP_QAT_FW_COMP_CNV_ERROR_NONE, - ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); - - /* Fill second half of the template for decompression */ - memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); - req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - - hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; - lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = 0; -} - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_dc.h b/drivers/crypto/qat/qat_common/adf_gen4_dc.h deleted file mode 100644 index 0b1a6774412e..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN4_DC_H -#define ADF_GEN4_DC_H - -#include "adf_accel_devices.h" - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN4_DC_H */ diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c deleted file mode 100644 index 3148a62938fd..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +++ /dev/null @@ -1,194 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2020 Intel Corporation */ -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_gen4_hw_data.h" - -static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) -{ - return BUILD_RING_BASE_ADDR(addr, size); -} - -static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); -} - -static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); -} - -static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) -{ - return READ_CSR_E_STAT(csr_base_addr, bank); -} - -static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); -} - -static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, - dma_addr_t addr) -{ - WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); -} - -static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); -} - -static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) -{ - WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); -} - -static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) -{ - WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); -} - -static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); -} - -static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); -} - -static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); -} - -void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) -{ - csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; - csr_ops->read_csr_ring_head = read_csr_ring_head; - csr_ops->write_csr_ring_head = write_csr_ring_head; - csr_ops->read_csr_ring_tail = read_csr_ring_tail; - csr_ops->write_csr_ring_tail = write_csr_ring_tail; - csr_ops->read_csr_e_stat = read_csr_e_stat; - csr_ops->write_csr_ring_config = write_csr_ring_config; - csr_ops->write_csr_ring_base = write_csr_ring_base; - csr_ops->write_csr_int_flag = write_csr_int_flag; - csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; - csr_ops->write_csr_int_col_en = write_csr_int_col_en; - csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; - csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; - csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); - -static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, - u32 *lower) -{ - *lower = lower_32_bits(value); - *upper = upper_32_bits(value); -} - -void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; - u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; - u32 ssm_wdt_pke_high = 0; - u32 ssm_wdt_pke_low = 0; - u32 ssm_wdt_high = 0; - u32 ssm_wdt_low = 0; - - /* Convert 64bit WDT timer value into 32bit values for - * mmio write to 32bit CSRs. - */ - adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low); - adf_gen4_unpack_ssm_wdtimer(timer_val_pke, &ssm_wdt_pke_high, - &ssm_wdt_pke_low); - - /* Enable WDT for sym and dc */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low); - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high); - /* Enable WDT for pke */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low); - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high); -} -EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); - -int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev) -{ - return 0; -} -EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled); - -static int reset_ring_pair(void __iomem *csr, u32 bank_number) -{ - u32 status; - int ret; - - /* Write rpresetctl register BIT(0) as 1 - * Since rpresetctl registers have no RW fields, no need to preserve - * values for other bits. Just write directly. - */ - ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), - ADF_WQM_CSR_RPRESETCTL_RESET); - - /* Read rpresetsts register and wait for rp reset to complete */ - ret = read_poll_timeout(ADF_CSR_RD, status, - status & ADF_WQM_CSR_RPRESETSTS_STATUS, - ADF_RPRESET_POLL_DELAY_US, - ADF_RPRESET_POLL_TIMEOUT_US, true, - csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); - if (!ret) { - /* When rp reset is done, clear rpresetsts */ - ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), - ADF_WQM_CSR_RPRESETSTS_STATUS); - } - - return ret; -} - -int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data); - void __iomem *csr; - int ret; - - if (bank_number >= hw_data->num_banks) - return -EINVAL; - - dev_dbg(&GET_DEV(accel_dev), - "ring pair reset for bank:%d\n", bank_number); - - csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr; - ret = reset_ring_pair(csr, bank_number); - if (ret) - dev_err(&GET_DEV(accel_dev), - "ring pair reset failed (timeout)\n"); - else - dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n"); - - return ret; -} -EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h deleted file mode 100644 index 4fb4b3df5a18..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +++ /dev/null @@ -1,142 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2020 Intel Corporation */ -#ifndef ADF_GEN4_HW_CSR_DATA_H_ -#define ADF_GEN4_HW_CSR_DATA_H_ - -#include "adf_accel_devices.h" -#include "adf_cfg_common.h" - -/* Transport access */ -#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL -#define ADF_RING_CSR_RING_CONFIG 0x1000 -#define ADF_RING_CSR_RING_LBASE 0x1040 -#define ADF_RING_CSR_RING_UBASE 0x1080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_ADDR_OFFSET 0x100000 -#define ADF_RING_BUNDLE_SIZE 0x2000 - -#define BUILD_RING_BASE_ADDR(addr, size) \ - ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - void __iomem *_csr_base_addr = csr_base_addr; \ - u32 _bank = bank; \ - u32 _ring = ring; \ - dma_addr_t _value = value; \ - u32 l_base = 0, u_base = 0; \ - l_base = lower_32_bits(_value); \ - u_base = upper_32_bits(_value); \ - ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (_bank) + \ - ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ - ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (_bank) + \ - ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ -} while (0) - -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_FLAG, (value)) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_COL_EN, (value)) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, (value)) - -/* Arbiter configuration */ -#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C - -#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_SRV_ARB_EN, (value)) - -/* Default ring mapping */ -#define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \ - (ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ - SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ - ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ - SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) - -/* WDT timers - * - * Timeout is in cycles. Clock speed may vary across products but this - * value should be a few milli-seconds. - */ -#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL -#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000 -#define ADF_SSMWDTL_OFFSET 0x54 -#define ADF_SSMWDTH_OFFSET 0x5C -#define ADF_SSMWDTPKEL_OFFSET 0x58 -#define ADF_SSMWDTPKEH_OFFSET 0x60 - -/* Ring reset */ -#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) -#define ADF_RPRESET_POLL_DELAY_US 20 -#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) -#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3)) -#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) -#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) - -/* Error source registers */ -#define ADF_GEN4_ERRSOU0 (0x41A200) -#define ADF_GEN4_ERRSOU1 (0x41A204) -#define ADF_GEN4_ERRSOU2 (0x41A208) -#define ADF_GEN4_ERRSOU3 (0x41A20C) - -/* Error source mask registers */ -#define ADF_GEN4_ERRMSK0 (0x41A210) -#define ADF_GEN4_ERRMSK1 (0x41A214) -#define ADF_GEN4_ERRMSK2 (0x41A218) -#define ADF_GEN4_ERRMSK3 (0x41A21C) - -#define ADF_GEN4_VFLNOTIFY BIT(7) - -void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); -void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); -int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); -#endif diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c deleted file mode 100644 index 8e8efe93f3ee..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2021 Intel Corporation */ -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_gen4_pfvf.h" -#include "adf_pfvf_pf_proto.h" -#include "adf_pfvf_utils.h" - -#define ADF_4XXX_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20)) -#define ADF_4XXX_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20)) - -/* VF2PF interrupt source registers */ -#define ADF_4XXX_VM2PF_SOU 0x41A180 -#define ADF_4XXX_VM2PF_MSK 0x41A1C0 -#define ADF_GEN4_VF_MSK 0xFFFF - -#define ADF_PFVF_GEN4_MSGTYPE_SHIFT 2 -#define ADF_PFVF_GEN4_MSGTYPE_MASK 0x3F -#define ADF_PFVF_GEN4_MSGDATA_SHIFT 8 -#define ADF_PFVF_GEN4_MSGDATA_MASK 0xFFFFFF - -static const struct pfvf_csr_format csr_gen4_fmt = { - { ADF_PFVF_GEN4_MSGTYPE_SHIFT, ADF_PFVF_GEN4_MSGTYPE_MASK }, - { ADF_PFVF_GEN4_MSGDATA_SHIFT, ADF_PFVF_GEN4_MSGDATA_MASK }, -}; - -static u32 adf_gen4_pf_get_pf2vf_offset(u32 i) -{ - return ADF_4XXX_PF2VM_OFFSET(i); -} - -static u32 adf_gen4_pf_get_vf2pf_offset(u32 i) -{ - return ADF_4XXX_VM2PF_OFFSET(i); -} - -static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) -{ - u32 val; - - val = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK) & ~vf_mask; - ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, val); -} - -static void adf_gen4_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) -{ - ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, ADF_GEN4_VF_MSK); -} - -static u32 adf_gen4_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) -{ - u32 sources, disabled, pending; - - /* Get the interrupt sources triggered by VFs */ - sources = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_SOU); - if (!sources) - return 0; - - /* Get the already disabled interrupts */ - disabled = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK); - - pending = sources & ~disabled; - if (!pending) - return 0; - - /* Due to HW limitations, when disabling the interrupts, we can't - * just disable the requested sources, as this would lead to missed - * interrupts if VM2PF_SOU changes just before writing to VM2PF_MSK. - * To work around it, disable all and re-enable only the sources that - * are not in vf_mask and were not already disabled. Re-enabling will - * trigger a new interrupt for the sources that have changed in the - * meantime, if any. - */ - ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, ADF_GEN4_VF_MSK); - ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, disabled | sources); - - /* Return the sources of the (new) interrupt(s) */ - return pending; -} - -static int adf_gen4_pfvf_send(struct adf_accel_dev *accel_dev, - struct pfvf_message msg, u32 pfvf_offset, - struct mutex *csr_lock) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - u32 csr_val; - int ret; - - csr_val = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen4_fmt); - if (unlikely(!csr_val)) - return -EINVAL; - - mutex_lock(csr_lock); - - ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val | ADF_PFVF_INT); - - /* Wait for confirmation from remote that it received the message */ - ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & ADF_PFVF_INT), - ADF_PFVF_MSG_ACK_DELAY_US, - ADF_PFVF_MSG_ACK_MAX_DELAY_US, - true, pmisc_addr, pfvf_offset); - if (ret < 0) - dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); - - mutex_unlock(csr_lock); - return ret; -} - -static struct pfvf_message adf_gen4_pfvf_recv(struct adf_accel_dev *accel_dev, - u32 pfvf_offset, u8 compat_ver) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - struct pfvf_message msg = { 0 }; - u32 csr_val; - - /* Read message from the CSR */ - csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); - if (!(csr_val & ADF_PFVF_INT)) { - dev_info(&GET_DEV(accel_dev), - "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); - return msg; - } - - /* We can now acknowledge the message reception by clearing the - * interrupt bit - */ - ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val & ~ADF_PFVF_INT); - - /* Return the pfvf_message format */ - return adf_pfvf_message_of(accel_dev, csr_val, &csr_gen4_fmt); -} - -void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) -{ - pfvf_ops->enable_comms = adf_enable_pf2vf_comms; - pfvf_ops->get_pf2vf_offset = adf_gen4_pf_get_pf2vf_offset; - pfvf_ops->get_vf2pf_offset = adf_gen4_pf_get_vf2pf_offset; - pfvf_ops->enable_vf2pf_interrupts = adf_gen4_enable_vf2pf_interrupts; - pfvf_ops->disable_all_vf2pf_interrupts = adf_gen4_disable_all_vf2pf_interrupts; - pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen4_disable_pending_vf2pf_interrupts; - pfvf_ops->send_msg = adf_gen4_pfvf_send; - pfvf_ops->recv_msg = adf_gen4_pfvf_recv; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_pf_pfvf_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h deleted file mode 100644 index 17d1b774d4a8..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_GEN4_PFVF_H -#define ADF_GEN4_PFVF_H - -#include "adf_accel_devices.h" - -#ifdef CONFIG_PCI_IOV -void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); -#else -static inline void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) -{ - pfvf_ops->enable_comms = adf_pfvf_comms_disabled; -} -#endif - -#endif /* ADF_GEN4_PFVF_H */ diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pm.c b/drivers/crypto/qat/qat_common/adf_gen4_pm.c deleted file mode 100644 index 7037c0892a8a..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_pm.c +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2022 Intel Corporation */ -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_gen4_pm.h" -#include "adf_cfg_strings.h" -#include "icp_qat_fw_init_admin.h" -#include "adf_gen4_hw_data.h" -#include "adf_cfg.h" - -enum qat_pm_host_msg { - PM_NO_CHANGE = 0, - PM_SET_MIN, -}; - -struct adf_gen4_pm_data { - struct work_struct pm_irq_work; - struct adf_accel_dev *accel_dev; - u32 pm_int_sts; -}; - -static int send_host_msg(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc = adf_get_pmisc_base(accel_dev); - u32 msg; - - msg = ADF_CSR_RD(pmisc, ADF_GEN4_PM_HOST_MSG); - if (msg & ADF_GEN4_PM_MSG_PENDING) - return -EBUSY; - - /* Send HOST_MSG */ - msg = FIELD_PREP(ADF_GEN4_PM_MSG_PAYLOAD_BIT_MASK, PM_SET_MIN); - msg |= ADF_GEN4_PM_MSG_PENDING; - ADF_CSR_WR(pmisc, ADF_GEN4_PM_HOST_MSG, msg); - - /* Poll status register to make sure the HOST_MSG has been processed */ - return read_poll_timeout(ADF_CSR_RD, msg, - !(msg & ADF_GEN4_PM_MSG_PENDING), - ADF_GEN4_PM_MSG_POLL_DELAY_US, - ADF_GEN4_PM_POLL_TIMEOUT_US, true, pmisc, - ADF_GEN4_PM_HOST_MSG); -} - -static void pm_bh_handler(struct work_struct *work) -{ - struct adf_gen4_pm_data *pm_data = - container_of(work, struct adf_gen4_pm_data, pm_irq_work); - struct adf_accel_dev *accel_dev = pm_data->accel_dev; - void __iomem *pmisc = adf_get_pmisc_base(accel_dev); - u32 pm_int_sts = pm_data->pm_int_sts; - u32 val; - - /* PM Idle interrupt */ - if (pm_int_sts & ADF_GEN4_PM_IDLE_STS) { - /* Issue host message to FW */ - if (send_host_msg(accel_dev)) - dev_warn_ratelimited(&GET_DEV(accel_dev), - "Failed to send host msg to FW\n"); - } - - /* Clear interrupt status */ - ADF_CSR_WR(pmisc, ADF_GEN4_PM_INTERRUPT, pm_int_sts); - - /* Reenable PM interrupt */ - val = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); - val &= ~ADF_GEN4_PM_SOU; - ADF_CSR_WR(pmisc, ADF_GEN4_ERRMSK2, val); - - kfree(pm_data); -} - -bool adf_gen4_handle_pm_interrupt(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc = adf_get_pmisc_base(accel_dev); - struct adf_gen4_pm_data *pm_data = NULL; - u32 errsou2; - u32 errmsk2; - u32 val; - - /* Only handle the interrupt triggered by PM */ - errmsk2 = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); - if (errmsk2 & ADF_GEN4_PM_SOU) - return false; - - errsou2 = ADF_CSR_RD(pmisc, ADF_GEN4_ERRSOU2); - if (!(errsou2 & ADF_GEN4_PM_SOU)) - return false; - - /* Disable interrupt */ - val = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); - val |= ADF_GEN4_PM_SOU; - ADF_CSR_WR(pmisc, ADF_GEN4_ERRMSK2, val); - - val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_INTERRUPT); - - pm_data = kzalloc(sizeof(*pm_data), GFP_ATOMIC); - if (!pm_data) - return false; - - pm_data->pm_int_sts = val; - pm_data->accel_dev = accel_dev; - - INIT_WORK(&pm_data->pm_irq_work, pm_bh_handler); - adf_misc_wq_queue_work(&pm_data->pm_irq_work); - - return true; -} -EXPORT_SYMBOL_GPL(adf_gen4_handle_pm_interrupt); - -int adf_gen4_enable_pm(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc = adf_get_pmisc_base(accel_dev); - int ret; - u32 val; - - ret = adf_init_admin_pm(accel_dev, ADF_GEN4_PM_DEFAULT_IDLE_FILTER); - if (ret) - return ret; - - /* Enable default PM interrupts: IDLE, THROTTLE */ - val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_INTERRUPT); - val |= ADF_GEN4_PM_INT_EN_DEFAULT; - - /* Clear interrupt status */ - val |= ADF_GEN4_PM_INT_STS_MASK; - ADF_CSR_WR(pmisc, ADF_GEN4_PM_INTERRUPT, val); - - /* Unmask PM Interrupt */ - val = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); - val &= ~ADF_GEN4_PM_SOU; - ADF_CSR_WR(pmisc, ADF_GEN4_ERRMSK2, val); - - return 0; -} -EXPORT_SYMBOL_GPL(adf_gen4_enable_pm); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pm.h b/drivers/crypto/qat/qat_common/adf_gen4_pm.h deleted file mode 100644 index f8f8a9ee29e5..000000000000 --- a/drivers/crypto/qat/qat_common/adf_gen4_pm.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN4_PM_H -#define ADF_GEN4_PM_H - -#include "adf_accel_devices.h" - -/* Power management registers */ -#define ADF_GEN4_PM_HOST_MSG (0x50A01C) - -/* Power management */ -#define ADF_GEN4_PM_POLL_DELAY_US 20 -#define ADF_GEN4_PM_POLL_TIMEOUT_US USEC_PER_SEC -#define ADF_GEN4_PM_MSG_POLL_DELAY_US (10 * USEC_PER_MSEC) -#define ADF_GEN4_PM_STATUS (0x50A00C) -#define ADF_GEN4_PM_INTERRUPT (0x50A028) - -/* Power management source in ERRSOU2 and ERRMSK2 */ -#define ADF_GEN4_PM_SOU BIT(18) - -#define ADF_GEN4_PM_IDLE_INT_EN BIT(18) -#define ADF_GEN4_PM_THROTTLE_INT_EN BIT(19) -#define ADF_GEN4_PM_DRV_ACTIVE BIT(20) -#define ADF_GEN4_PM_INIT_STATE BIT(21) -#define ADF_GEN4_PM_INT_EN_DEFAULT (ADF_GEN4_PM_IDLE_INT_EN | \ - ADF_GEN4_PM_THROTTLE_INT_EN) - -#define ADF_GEN4_PM_THR_STS BIT(0) -#define ADF_GEN4_PM_IDLE_STS BIT(1) -#define ADF_GEN4_PM_FW_INT_STS BIT(2) -#define ADF_GEN4_PM_INT_STS_MASK (ADF_GEN4_PM_THR_STS | \ - ADF_GEN4_PM_IDLE_STS | \ - ADF_GEN4_PM_FW_INT_STS) - -#define ADF_GEN4_PM_MSG_PENDING BIT(0) -#define ADF_GEN4_PM_MSG_PAYLOAD_BIT_MASK GENMASK(28, 1) - -#define ADF_GEN4_PM_DEFAULT_IDLE_FILTER (0x0) -#define ADF_GEN4_PM_MAX_IDLE_FILTER (0x7) - -int adf_gen4_enable_pm(struct adf_accel_dev *accel_dev); -bool adf_gen4_handle_pm_interrupt(struct adf_accel_dev *accel_dev); - -#endif diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c deleted file mode 100644 index da6956699246..000000000000 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_transport_internal.h" - -#define ADF_ARB_NUM 4 -#define ADF_ARB_REG_SIZE 0x4 - -#define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ - ADF_CSR_WR(csr_addr, (arb_offset) + \ - (ADF_ARB_REG_SIZE * (index)), value) - -#define WRITE_CSR_ARB_WT2SAM(csr_addr, arb_offset, wt_offset, index, value) \ - ADF_CSR_WR(csr_addr, ((arb_offset) + (wt_offset)) + \ - (ADF_ARB_REG_SIZE * (index)), value) - -int adf_init_arb(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - unsigned long ae_mask = hw_data->ae_mask; - u32 arb_off, wt_off, arb_cfg; - const u32 *thd_2_arb_cfg; - struct arb_info info; - int arb, i; - - hw_data->get_arb_info(&info); - arb_cfg = info.arb_cfg; - arb_off = info.arb_offset; - wt_off = info.wt2sam_offset; - - /* Service arb configured for 32 bytes responses and - * ring flow control check enabled. */ - for (arb = 0; arb < ADF_ARB_NUM; arb++) - WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); - - /* Map worker threads to service arbiters */ - thd_2_arb_cfg = hw_data->get_arb_mapping(accel_dev); - - for_each_set_bit(i, &ae_mask, hw_data->num_engines) - WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); - - return 0; -} -EXPORT_SYMBOL_GPL(adf_init_arb); - -void adf_update_ring_arb(struct adf_etr_ring_data *ring) -{ - struct adf_accel_dev *accel_dev = ring->bank->accel_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); - u32 tx_ring_mask = hw_data->tx_rings_mask; - u32 shift = hw_data->tx_rx_gap; - u32 arben, arben_tx, arben_rx; - u32 rx_ring_mask; - - /* - * Enable arbitration on a ring only if the TX half of the ring mask - * matches the RX part. This results in writes to CSR on both TX and - * RX update - only one is necessary, but both are done for - * simplicity. - */ - rx_ring_mask = tx_ring_mask << shift; - arben_tx = (ring->bank->ring_mask & tx_ring_mask) >> 0; - arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; - arben = arben_tx & arben_rx; - - csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr, - ring->bank->bank_number, arben); -} - -void adf_exit_arb(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); - u32 arb_off, wt_off; - struct arb_info info; - void __iomem *csr; - unsigned int i; - - hw_data->get_arb_info(&info); - arb_off = info.arb_offset; - wt_off = info.wt2sam_offset; - - if (!accel_dev->transport) - return; - - csr = accel_dev->transport->banks[0].csr_addr; - - hw_data->get_arb_info(&info); - - /* Reset arbiter configuration */ - for (i = 0; i < ADF_ARB_NUM; i++) - WRITE_CSR_ARB_SARCONFIG(csr, arb_off, i, 0); - - /* Unmap worker threads to service arbiters */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, 0); - - /* Disable arbitration on all rings */ - for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); -} -EXPORT_SYMBOL_GPL(adf_exit_arb); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c deleted file mode 100644 index 0985f64ab11a..000000000000 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ /dev/null @@ -1,480 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_cfg.h" -#include "adf_common_drv.h" - -static LIST_HEAD(service_table); -static DEFINE_MUTEX(service_lock); - -static void adf_service_add(struct service_hndl *service) -{ - mutex_lock(&service_lock); - list_add(&service->list, &service_table); - mutex_unlock(&service_lock); -} - -int adf_service_register(struct service_hndl *service) -{ - memset(service->init_status, 0, sizeof(service->init_status)); - memset(service->start_status, 0, sizeof(service->start_status)); - adf_service_add(service); - return 0; -} - -static void adf_service_remove(struct service_hndl *service) -{ - mutex_lock(&service_lock); - list_del(&service->list); - mutex_unlock(&service_lock); -} - -int adf_service_unregister(struct service_hndl *service) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(service->init_status); i++) { - if (service->init_status[i] || service->start_status[i]) { - pr_err("QAT: Could not remove active service\n"); - return -EFAULT; - } - } - adf_service_remove(service); - return 0; -} - -/** - * adf_dev_init() - Init data structures and services for the given accel device - * @accel_dev: Pointer to acceleration device. - * - * Initialize the ring data structures and the admin comms and arbitration - * services. - * - * Return: 0 on success, error code otherwise. - */ -static int adf_dev_init(struct adf_accel_dev *accel_dev) -{ - struct service_hndl *service; - struct list_head *list_itr; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int ret; - - if (!hw_data) { - dev_err(&GET_DEV(accel_dev), - "Failed to init device - hw_data not set\n"); - return -EFAULT; - } - - if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && - !accel_dev->is_vf) { - dev_err(&GET_DEV(accel_dev), "Device not configured\n"); - return -EFAULT; - } - - if (adf_init_etr_data(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n"); - return -EFAULT; - } - - if (hw_data->init_device && hw_data->init_device(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n"); - return -EFAULT; - } - - if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n"); - return -EFAULT; - } - - if (hw_data->init_arb && hw_data->init_arb(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n"); - return -EFAULT; - } - - if (adf_ae_init(accel_dev)) { - dev_err(&GET_DEV(accel_dev), - "Failed to initialise Acceleration Engine\n"); - return -EFAULT; - } - set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status); - - if (adf_ae_fw_load(accel_dev)) { - dev_err(&GET_DEV(accel_dev), - "Failed to load acceleration FW\n"); - return -EFAULT; - } - set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); - - if (hw_data->alloc_irq(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n"); - return -EFAULT; - } - set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); - - hw_data->enable_ints(accel_dev); - hw_data->enable_error_correction(accel_dev); - - ret = hw_data->pfvf_ops.enable_comms(accel_dev); - if (ret) - return ret; - - if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && - accel_dev->is_vf) { - if (qat_crypto_vf_dev_config(accel_dev)) - return -EFAULT; - } - - /* - * Subservice initialisation is divided into two stages: init and start. - * This is to facilitate any ordering dependencies between services - * prior to starting any of the accelerators. - */ - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { - dev_err(&GET_DEV(accel_dev), - "Failed to initialise service %s\n", - service->name); - return -EFAULT; - } - set_bit(accel_dev->accel_id, service->init_status); - } - - return 0; -} - -/** - * adf_dev_start() - Start acceleration service for the given accel device - * @accel_dev: Pointer to acceleration device. - * - * Function notifies all the registered services that the acceleration device - * is ready to be used. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ -static int adf_dev_start(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct service_hndl *service; - struct list_head *list_itr; - - set_bit(ADF_STATUS_STARTING, &accel_dev->status); - - if (adf_ae_start(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "AE Start Failed\n"); - return -EFAULT; - } - set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); - - if (hw_data->send_admin_init(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to send init message\n"); - return -EFAULT; - } - - /* Set ssm watch dog timer */ - if (hw_data->set_ssm_wdtimer) - hw_data->set_ssm_wdtimer(accel_dev); - - /* Enable Power Management */ - if (hw_data->enable_pm && hw_data->enable_pm(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to configure Power Management\n"); - return -EFAULT; - } - - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (service->event_hld(accel_dev, ADF_EVENT_START)) { - dev_err(&GET_DEV(accel_dev), - "Failed to start service %s\n", - service->name); - return -EFAULT; - } - set_bit(accel_dev->accel_id, service->start_status); - } - - clear_bit(ADF_STATUS_STARTING, &accel_dev->status); - set_bit(ADF_STATUS_STARTED, &accel_dev->status); - - if (!list_empty(&accel_dev->crypto_list) && - (qat_algs_register() || qat_asym_algs_register())) { - dev_err(&GET_DEV(accel_dev), - "Failed to register crypto algs\n"); - set_bit(ADF_STATUS_STARTING, &accel_dev->status); - clear_bit(ADF_STATUS_STARTED, &accel_dev->status); - return -EFAULT; - } - - if (!list_empty(&accel_dev->compression_list) && qat_comp_algs_register()) { - dev_err(&GET_DEV(accel_dev), - "Failed to register compression algs\n"); - set_bit(ADF_STATUS_STARTING, &accel_dev->status); - clear_bit(ADF_STATUS_STARTED, &accel_dev->status); - return -EFAULT; - } - return 0; -} - -/** - * adf_dev_stop() - Stop acceleration service for the given accel device - * @accel_dev: Pointer to acceleration device. - * - * Function notifies all the registered services that the acceleration device - * is shuting down. - * To be used by QAT device specific drivers. - * - * Return: void - */ -static void adf_dev_stop(struct adf_accel_dev *accel_dev) -{ - struct service_hndl *service; - struct list_head *list_itr; - bool wait = false; - int ret; - - if (!adf_dev_started(accel_dev) && - !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) - return; - - clear_bit(ADF_STATUS_STARTING, &accel_dev->status); - clear_bit(ADF_STATUS_STARTED, &accel_dev->status); - - if (!list_empty(&accel_dev->crypto_list)) { - qat_algs_unregister(); - qat_asym_algs_unregister(); - } - - if (!list_empty(&accel_dev->compression_list)) - qat_comp_algs_unregister(); - - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!test_bit(accel_dev->accel_id, service->start_status)) - continue; - ret = service->event_hld(accel_dev, ADF_EVENT_STOP); - if (!ret) { - clear_bit(accel_dev->accel_id, service->start_status); - } else if (ret == -EAGAIN) { - wait = true; - clear_bit(accel_dev->accel_id, service->start_status); - } - } - - if (wait) - msleep(100); - - if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) { - if (adf_ae_stop(accel_dev)) - dev_err(&GET_DEV(accel_dev), "failed to stop AE\n"); - else - clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); - } -} - -/** - * adf_dev_shutdown() - shutdown acceleration services and data strucutures - * @accel_dev: Pointer to acceleration device - * - * Cleanup the ring data structures and the admin comms and arbitration - * services. - */ -static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct service_hndl *service; - struct list_head *list_itr; - - if (!hw_data) { - dev_err(&GET_DEV(accel_dev), - "QAT: Failed to shutdown device - hw_data not set\n"); - return; - } - - if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { - adf_ae_fw_release(accel_dev); - clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); - } - - if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) { - if (adf_ae_shutdown(accel_dev)) - dev_err(&GET_DEV(accel_dev), - "Failed to shutdown Accel Engine\n"); - else - clear_bit(ADF_STATUS_AE_INITIALISED, - &accel_dev->status); - } - - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!test_bit(accel_dev->accel_id, service->init_status)) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) - dev_err(&GET_DEV(accel_dev), - "Failed to shutdown service %s\n", - service->name); - else - clear_bit(accel_dev->accel_id, service->init_status); - } - - hw_data->disable_iov(accel_dev); - - if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { - hw_data->free_irq(accel_dev); - clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); - } - - /* Delete configuration only if not restarting */ - if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) - adf_cfg_del_all(accel_dev); - - if (hw_data->exit_arb) - hw_data->exit_arb(accel_dev); - - if (hw_data->exit_admin_comms) - hw_data->exit_admin_comms(accel_dev); - - adf_cleanup_etr_data(accel_dev); - adf_dev_restore(accel_dev); -} - -int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) -{ - struct service_hndl *service; - struct list_head *list_itr; - - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) - dev_err(&GET_DEV(accel_dev), - "Failed to restart service %s.\n", - service->name); - } - return 0; -} - -int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) -{ - struct service_hndl *service; - struct list_head *list_itr; - - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) - dev_err(&GET_DEV(accel_dev), - "Failed to restart service %s.\n", - service->name); - } - return 0; -} - -static int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - int ret; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - - adf_dev_stop(accel_dev); - adf_dev_shutdown(accel_dev); - - if (!ret) { - ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); - if (ret) - return ret; - - ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, - services, ADF_STR); - if (ret) - return ret; - } - - return 0; -} - -int adf_dev_down(struct adf_accel_dev *accel_dev, bool reconfig) -{ - int ret = 0; - - if (!accel_dev) - return -EINVAL; - - mutex_lock(&accel_dev->state_lock); - - if (!adf_dev_started(accel_dev)) { - dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already down\n", - accel_dev->accel_id); - ret = -EINVAL; - goto out; - } - - if (reconfig) { - ret = adf_dev_shutdown_cache_cfg(accel_dev); - goto out; - } - - adf_dev_stop(accel_dev); - adf_dev_shutdown(accel_dev); - -out: - mutex_unlock(&accel_dev->state_lock); - return ret; -} -EXPORT_SYMBOL_GPL(adf_dev_down); - -int adf_dev_up(struct adf_accel_dev *accel_dev, bool config) -{ - int ret = 0; - - if (!accel_dev) - return -EINVAL; - - mutex_lock(&accel_dev->state_lock); - - if (adf_dev_started(accel_dev)) { - dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already up\n", - accel_dev->accel_id); - ret = -EALREADY; - goto out; - } - - if (config && GET_HW_DATA(accel_dev)->dev_config) { - ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev); - if (unlikely(ret)) - goto out; - } - - ret = adf_dev_init(accel_dev); - if (unlikely(ret)) - goto out; - - ret = adf_dev_start(accel_dev); - -out: - mutex_unlock(&accel_dev->state_lock); - return ret; -} -EXPORT_SYMBOL_GPL(adf_dev_up); - -int adf_dev_restart(struct adf_accel_dev *accel_dev) -{ - int ret = 0; - - if (!accel_dev) - return -EFAULT; - - adf_dev_down(accel_dev, false); - - ret = adf_dev_up(accel_dev, false); - /* if device is already up return success*/ - if (ret == -EALREADY) - return 0; - - return ret; -} -EXPORT_SYMBOL_GPL(adf_dev_restart); diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c deleted file mode 100644 index ad9e135b8560..000000000000 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ /dev/null @@ -1,382 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_cfg.h" -#include "adf_cfg_strings.h" -#include "adf_cfg_common.h" -#include "adf_transport_access_macros.h" -#include "adf_transport_internal.h" - -#define ADF_MAX_NUM_VFS 32 -static struct workqueue_struct *adf_misc_wq; - -static int adf_enable_msix(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 msix_num_entries = hw_data->num_banks + 1; - int ret; - - if (hw_data->set_msix_rttable) - hw_data->set_msix_rttable(accel_dev); - - ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries, - msix_num_entries, PCI_IRQ_MSIX); - if (unlikely(ret < 0)) { - dev_err(&GET_DEV(accel_dev), - "Failed to allocate %d MSI-X vectors\n", - msix_num_entries); - return ret; - } - return 0; -} - -static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) -{ - pci_free_irq_vectors(pci_dev_info->pci_dev); -} - -static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) -{ - struct adf_etr_bank_data *bank = bank_ptr; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - - csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, - 0); - tasklet_hi_schedule(&bank->resp_handler); - return IRQ_HANDLED; -} - -#ifdef CONFIG_PCI_IOV -void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - unsigned long flags; - - spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); - spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); -} - -void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - unsigned long flags; - - spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr); - spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); -} - -static u32 adf_disable_pending_vf2pf_interrupts(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - u32 pending; - - spin_lock(&accel_dev->pf.vf2pf_ints_lock); - pending = GET_PFVF_OPS(accel_dev)->disable_pending_vf2pf_interrupts(pmisc_addr); - spin_unlock(&accel_dev->pf.vf2pf_ints_lock); - - return pending; -} - -static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) -{ - bool irq_handled = false; - unsigned long vf_mask; - - /* Get the interrupt sources triggered by VFs, except for those already disabled */ - vf_mask = adf_disable_pending_vf2pf_interrupts(accel_dev); - if (vf_mask) { - struct adf_accel_vf_info *vf_info; - int i; - - /* - * Handle VF2PF interrupt unless the VF is malicious and - * is attempting to flood the host OS with VF2PF interrupts. - */ - for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) { - vf_info = accel_dev->pf.vf_info + i; - - if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { - dev_info(&GET_DEV(accel_dev), - "Too many ints from VF%d\n", - vf_info->vf_nr); - continue; - } - - adf_schedule_vf2pf_handler(vf_info); - irq_handled = true; - } - } - return irq_handled; -} -#endif /* CONFIG_PCI_IOV */ - -static bool adf_handle_pm_int(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - - if (hw_data->handle_pm_interrupt && - hw_data->handle_pm_interrupt(accel_dev)) - return true; - - return false; -} - -static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) -{ - struct adf_accel_dev *accel_dev = dev_ptr; - -#ifdef CONFIG_PCI_IOV - /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ - if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev)) - return IRQ_HANDLED; -#endif /* CONFIG_PCI_IOV */ - - if (adf_handle_pm_int(accel_dev)) - return IRQ_HANDLED; - - dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", - accel_dev->accel_id); - - return IRQ_NONE; -} - -static void adf_free_irqs(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; - struct adf_etr_data *etr_data = accel_dev->transport; - int clust_irq = hw_data->num_banks; - int irq, i = 0; - - if (pci_dev_info->msix_entries.num_entries > 1) { - for (i = 0; i < hw_data->num_banks; i++) { - if (irqs[i].enabled) { - irq = pci_irq_vector(pci_dev_info->pci_dev, i); - irq_set_affinity_hint(irq, NULL); - free_irq(irq, &etr_data->banks[i]); - } - } - } - - if (irqs[i].enabled) { - irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); - free_irq(irq, accel_dev); - } -} - -static int adf_request_irqs(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; - struct adf_etr_data *etr_data = accel_dev->transport; - int clust_irq = hw_data->num_banks; - int ret, irq, i = 0; - char *name; - - /* Request msix irq for all banks unless SR-IOV enabled */ - if (!accel_dev->pf.vf_info) { - for (i = 0; i < hw_data->num_banks; i++) { - struct adf_etr_bank_data *bank = &etr_data->banks[i]; - unsigned int cpu, cpus = num_online_cpus(); - - name = irqs[i].name; - snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, - "qat%d-bundle%d", accel_dev->accel_id, i); - irq = pci_irq_vector(pci_dev_info->pci_dev, i); - if (unlikely(irq < 0)) { - dev_err(&GET_DEV(accel_dev), - "Failed to get IRQ number of device vector %d - %s\n", - i, name); - ret = irq; - goto err; - } - ret = request_irq(irq, adf_msix_isr_bundle, 0, - &name[0], bank); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to allocate IRQ %d for %s\n", - irq, name); - goto err; - } - - cpu = ((accel_dev->accel_id * hw_data->num_banks) + - i) % cpus; - irq_set_affinity_hint(irq, get_cpu_mask(cpu)); - irqs[i].enabled = true; - } - } - - /* Request msix irq for AE */ - name = irqs[i].name; - snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, - "qat%d-ae-cluster", accel_dev->accel_id); - irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); - if (unlikely(irq < 0)) { - dev_err(&GET_DEV(accel_dev), - "Failed to get IRQ number of device vector %d - %s\n", - i, name); - ret = irq; - goto err; - } - ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to allocate IRQ %d for %s\n", irq, name); - goto err; - } - irqs[i].enabled = true; - return ret; -err: - adf_free_irqs(accel_dev); - return ret; -} - -static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 msix_num_entries = 1; - struct adf_irq *irqs; - - /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ - if (!accel_dev->pf.vf_info) - msix_num_entries += hw_data->num_banks; - - irqs = kzalloc_node(msix_num_entries * sizeof(*irqs), - GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); - if (!irqs) - return -ENOMEM; - - accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; - accel_dev->accel_pci_dev.msix_entries.irqs = irqs; - return 0; -} - -static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev) -{ - kfree(accel_dev->accel_pci_dev.msix_entries.irqs); - accel_dev->accel_pci_dev.msix_entries.irqs = NULL; -} - -static int adf_setup_bh(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *priv_data = accel_dev->transport; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int i; - - for (i = 0; i < hw_data->num_banks; i++) - tasklet_init(&priv_data->banks[i].resp_handler, - adf_response_handler, - (unsigned long)&priv_data->banks[i]); - return 0; -} - -static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *priv_data = accel_dev->transport; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int i; - - for (i = 0; i < hw_data->num_banks; i++) { - tasklet_disable(&priv_data->banks[i].resp_handler); - tasklet_kill(&priv_data->banks[i].resp_handler); - } -} - -/** - * adf_isr_resource_free() - Free IRQ for acceleration device - * @accel_dev: Pointer to acceleration device. - * - * Function frees interrupts for acceleration device. - */ -void adf_isr_resource_free(struct adf_accel_dev *accel_dev) -{ - adf_free_irqs(accel_dev); - adf_cleanup_bh(accel_dev); - adf_disable_msix(&accel_dev->accel_pci_dev); - adf_isr_free_msix_vectors_data(accel_dev); -} -EXPORT_SYMBOL_GPL(adf_isr_resource_free); - -/** - * adf_isr_resource_alloc() - Allocate IRQ for acceleration device - * @accel_dev: Pointer to acceleration device. - * - * Function allocates interrupts for acceleration device. - * - * Return: 0 on success, error code otherwise. - */ -int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) -{ - int ret; - - ret = adf_isr_alloc_msix_vectors_data(accel_dev); - if (ret) - goto err_out; - - ret = adf_enable_msix(accel_dev); - if (ret) - goto err_free_msix_table; - - ret = adf_setup_bh(accel_dev); - if (ret) - goto err_disable_msix; - - ret = adf_request_irqs(accel_dev); - if (ret) - goto err_cleanup_bh; - - return 0; - -err_cleanup_bh: - adf_cleanup_bh(accel_dev); - -err_disable_msix: - adf_disable_msix(&accel_dev->accel_pci_dev); - -err_free_msix_table: - adf_isr_free_msix_vectors_data(accel_dev); - -err_out: - return ret; -} -EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); - -/** - * adf_init_misc_wq() - Init misc workqueue - * - * Function init workqueue 'qat_misc_wq' for general purpose. - * - * Return: 0 on success, error code otherwise. - */ -int __init adf_init_misc_wq(void) -{ - adf_misc_wq = alloc_workqueue("qat_misc_wq", WQ_MEM_RECLAIM, 0); - - return !adf_misc_wq ? -ENOMEM : 0; -} - -void adf_exit_misc_wq(void) -{ - if (adf_misc_wq) - destroy_workqueue(adf_misc_wq); - - adf_misc_wq = NULL; -} - -bool adf_misc_wq_queue_work(struct work_struct *work) -{ - return queue_work(adf_misc_wq, work); -} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_msg.h deleted file mode 100644 index 204a42438992..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h +++ /dev/null @@ -1,259 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#ifndef ADF_PFVF_MSG_H -#define ADF_PFVF_MSG_H - -#include - -/* - * PF<->VF Gen2 Messaging format - * - * The PF has an array of 32-bit PF2VF registers, one for each VF. The - * PF can access all these registers while each VF can access only the one - * register associated with that particular VF. - * - * The register functionally is split into two parts: - * The bottom half is for PF->VF messages. In particular when the first - * bit of this register (bit 0) gets set an interrupt will be triggered - * in the respective VF. - * The top half is for VF->PF messages. In particular when the first bit - * of this half of register (bit 16) gets set an interrupt will be triggered - * in the PF. - * - * The remaining bits within this register are available to encode messages. - * and implement a collision control mechanism to prevent concurrent use of - * the PF2VF register by both the PF and VF. - * - * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 - * _______________________________________________ - * | | | | | | | | | | | | | | | | | - * +-----------------------------------------------+ - * \___________________________/ \_________/ ^ ^ - * ^ ^ | | - * | | | VF2PF Int - * | | Message Origin - * | Message Type - * Message-specific Data/Reserved - * - * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 - * _______________________________________________ - * | | | | | | | | | | | | | | | | | - * +-----------------------------------------------+ - * \___________________________/ \_________/ ^ ^ - * ^ ^ | | - * | | | PF2VF Int - * | | Message Origin - * | Message Type - * Message-specific Data/Reserved - * - * Message Origin (Should always be 1) - * A legacy out-of-tree QAT driver allowed for a set of messages not supported - * by this driver; these had a Msg Origin of 0 and are ignored by this driver. - * - * When a PF or VF attempts to send a message in the lower or upper 16 bits, - * respectively, the other 16 bits are written to first with a defined - * IN_USE_BY pattern as part of a collision control scheme (see function - * adf_gen2_pfvf_send() in adf_pf2vf_msg.c). - * - * - * PF<->VF Gen4 Messaging format - * - * Similarly to the gen2 messaging format, 32-bit long registers are used for - * communication between PF and VFs. However, each VF and PF share a pair of - * 32-bits register to avoid collisions: one for PV to VF messages and one - * for VF to PF messages. - * - * Both the Interrupt bit and the Message Origin bit retain the same position - * and meaning, although non-system messages are now deprecated and not - * expected. - * - * 31 30 9 8 7 6 5 4 3 2 1 0 - * _______________________________________________ - * | | | . . . | | | | | | | | | | | - * +-----------------------------------------------+ - * \_____________________/ \_______________/ ^ ^ - * ^ ^ | | - * | | | PF/VF Int - * | | Message Origin - * | Message Type - * Message-specific Data/Reserved - * - * For both formats, the message reception is acknowledged by lowering the - * interrupt bit on the register where the message was sent. - */ - -/* PFVF message common bits */ -#define ADF_PFVF_INT BIT(0) -#define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1) - -/* Different generations have different CSR layouts, use this struct - * to abstract these differences away - */ -struct pfvf_message { - u8 type; - u32 data; -}; - -/* PF->VF messages */ -enum pf2vf_msgtype { - ADF_PF2VF_MSGTYPE_RESTARTING = 0x01, - ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02, - ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03, -/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */ - ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10, -}; - -/* VF->PF messages */ -enum vf2pf_msgtype { - ADF_VF2PF_MSGTYPE_INIT = 0x03, - ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04, - ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05, - ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06, - ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07, - ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08, - ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09, -/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */ - ADF_VF2PF_MSGTYPE_RP_RESET = 0x10, -}; - -/* VF/PF compatibility version. */ -enum pfvf_compatibility_version { - /* Support for extended capabilities */ - ADF_PFVF_COMPAT_CAPABILITIES = 0x02, - /* In-use pattern cleared by receiver */ - ADF_PFVF_COMPAT_FAST_ACK = 0x03, - /* Ring to service mapping support for non-standard mappings */ - ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04, - /* Reference to the latest version */ - ADF_PFVF_COMPAT_THIS_VERSION = 0x04, -}; - -/* PF->VF Version Response */ -#define ADF_PF2VF_VERSION_RESP_VERS_MASK GENMASK(7, 0) -#define ADF_PF2VF_VERSION_RESP_RESULT_MASK GENMASK(9, 8) - -enum pf2vf_compat_response { - ADF_PF2VF_VF_COMPATIBLE = 0x01, - ADF_PF2VF_VF_INCOMPATIBLE = 0x02, - ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03, -}; - -enum ring_reset_result { - RPRESET_SUCCESS = 0x00, - RPRESET_NOT_SUPPORTED = 0x01, - RPRESET_INVAL_BANK = 0x02, - RPRESET_TIMEOUT = 0x03, -}; - -#define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0) -#define ADF_VF2PF_RNG_RESET_RSVD_MASK GENMASK(25, 2) - -/* PF->VF Block Responses */ -#define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0) -#define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2) - -enum pf2vf_blkmsg_resp_type { - ADF_PF2VF_BLKMSG_RESP_TYPE_DATA = 0x00, - ADF_PF2VF_BLKMSG_RESP_TYPE_CRC = 0x01, - ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR = 0x02, -}; - -/* PF->VF Block Error Code */ -enum pf2vf_blkmsg_error { - ADF_PF2VF_INVALID_BLOCK_TYPE = 0x00, - ADF_PF2VF_INVALID_BYTE_NUM_REQ = 0x01, - ADF_PF2VF_PAYLOAD_TRUNCATED = 0x02, - ADF_PF2VF_UNSPECIFIED_ERROR = 0x03, -}; - -/* VF->PF Block Requests */ -#define ADF_VF2PF_LARGE_BLOCK_TYPE_MASK GENMASK(1, 0) -#define ADF_VF2PF_LARGE_BLOCK_BYTE_MASK GENMASK(8, 2) -#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK GENMASK(2, 0) -#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK GENMASK(8, 3) -#define ADF_VF2PF_SMALL_BLOCK_TYPE_MASK GENMASK(3, 0) -#define ADF_VF2PF_SMALL_BLOCK_BYTE_MASK GENMASK(8, 4) -#define ADF_VF2PF_BLOCK_CRC_REQ_MASK BIT(9) - -/* PF->VF Block Request Types - * 0..15 - 32 byte message - * 16..23 - 64 byte message - * 24..27 - 128 byte message - */ -enum vf2pf_blkmsg_req_type { - ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02, - ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP = 0x03, -}; - -#define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \ - (FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK)) - -#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX \ - (FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK) + \ - ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1) - -#define ADF_VF2PF_LARGE_BLOCK_TYPE_MAX \ - (FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK) + \ - ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) - -#define ADF_VF2PF_SMALL_BLOCK_BYTE_MAX \ - FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK) - -#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX \ - FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK) - -#define ADF_VF2PF_LARGE_BLOCK_BYTE_MAX \ - FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK) - -struct pfvf_blkmsg_header { - u8 version; - u8 payload_size; -} __packed; - -#define ADF_PFVF_BLKMSG_HEADER_SIZE (sizeof(struct pfvf_blkmsg_header)) -#define ADF_PFVF_BLKMSG_PAYLOAD_SIZE(blkmsg) (sizeof(blkmsg) - \ - ADF_PFVF_BLKMSG_HEADER_SIZE) -#define ADF_PFVF_BLKMSG_MSG_SIZE(blkmsg) (ADF_PFVF_BLKMSG_HEADER_SIZE + \ - (blkmsg)->hdr.payload_size) -#define ADF_PFVF_BLKMSG_MSG_MAX_SIZE 128 - -/* PF->VF Block message header bytes */ -#define ADF_PFVF_BLKMSG_VER_BYTE 0 -#define ADF_PFVF_BLKMSG_LEN_BYTE 1 - -/* PF/VF Capabilities message values */ -enum blkmsg_capabilities_versions { - ADF_PFVF_CAPABILITIES_V1_VERSION = 0x01, - ADF_PFVF_CAPABILITIES_V2_VERSION = 0x02, - ADF_PFVF_CAPABILITIES_V3_VERSION = 0x03, -}; - -struct capabilities_v1 { - struct pfvf_blkmsg_header hdr; - u32 ext_dc_caps; -} __packed; - -struct capabilities_v2 { - struct pfvf_blkmsg_header hdr; - u32 ext_dc_caps; - u32 capabilities; -} __packed; - -struct capabilities_v3 { - struct pfvf_blkmsg_header hdr; - u32 ext_dc_caps; - u32 capabilities; - u32 frequency; -} __packed; - -/* PF/VF Ring to service mapping values */ -enum blkmsg_ring_to_svc_versions { - ADF_PFVF_RING_TO_SVC_VERSION = 0x01, -}; - -struct ring_to_svc_map_v1 { - struct pfvf_blkmsg_header hdr; - u16 map; -} __packed; - -#endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c deleted file mode 100644 index 14c069f0d71a..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include "adf_accel_devices.h" -#include "adf_pfvf_msg.h" -#include "adf_pfvf_pf_msg.h" -#include "adf_pfvf_pf_proto.h" - -void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_vf_info *vf; - struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTING }; - int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); - - for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { - if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg)) - dev_err(&GET_DEV(accel_dev), - "Failed to send restarting msg to VF%d\n", i); - } -} - -int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, - u8 *buffer, u8 compat) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct capabilities_v2 caps_msg; - - caps_msg.ext_dc_caps = hw_data->extended_dc_capabilities; - caps_msg.capabilities = hw_data->accel_capabilities_mask; - - caps_msg.hdr.version = ADF_PFVF_CAPABILITIES_V2_VERSION; - caps_msg.hdr.payload_size = - ADF_PFVF_BLKMSG_PAYLOAD_SIZE(struct capabilities_v2); - - memcpy(buffer, &caps_msg, sizeof(caps_msg)); - - return 0; -} - -int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev, - u8 *buffer, u8 compat) -{ - struct ring_to_svc_map_v1 rts_map_msg; - - rts_map_msg.map = accel_dev->hw_device->ring_to_svc_map; - rts_map_msg.hdr.version = ADF_PFVF_RING_TO_SVC_VERSION; - rts_map_msg.hdr.payload_size = ADF_PFVF_BLKMSG_PAYLOAD_SIZE(rts_map_msg); - - memcpy(buffer, &rts_map_msg, sizeof(rts_map_msg)); - - return 0; -} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h deleted file mode 100644 index e8982d1ac896..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_PFVF_PF_MSG_H -#define ADF_PFVF_PF_MSG_H - -#include "adf_accel_devices.h" - -void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); - -typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev, - u8 *buffer, u8 compat); - -int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, - u8 *buffer, u8 comapt); -int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev, - u8 *buffer, u8 comapt); - -#endif /* ADF_PFVF_PF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c deleted file mode 100644 index 388e58bcbcaf..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c +++ /dev/null @@ -1,348 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_pfvf_msg.h" -#include "adf_pfvf_pf_msg.h" -#include "adf_pfvf_pf_proto.h" -#include "adf_pfvf_utils.h" - -typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte); - -static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = { - NULL, /* no message type defined for value 0 */ - NULL, /* no message type defined for value 1 */ - adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */ - adf_pf_ring_to_svc_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP */ -}; - -/** - * adf_send_pf2vf_msg() - send PF to VF message - * @accel_dev: Pointer to acceleration device - * @vf_nr: VF number to which the message will be sent - * @msg: Message to send - * - * This function allows the PF to send a message to a specific VF. - * - * Return: 0 on success, error code otherwise. - */ -int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg) -{ - struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); - u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr); - - return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset, - &accel_dev->pf.vf_info[vf_nr].pf2vf_lock); -} - -/** - * adf_recv_vf2pf_msg() - receive a VF to PF message - * @accel_dev: Pointer to acceleration device - * @vf_nr: Number of the VF from where the message will be received - * - * This function allows the PF to receive a message from a specific VF. - * - * Return: a valid message on success, zero otherwise. - */ -static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) -{ - struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; - struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); - u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr); - - return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver); -} - -static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type) -{ - if (type >= ARRAY_SIZE(pf2vf_blkmsg_providers)) - return NULL; - - return pf2vf_blkmsg_providers[type]; -} - -/* Byte pf2vf_blkmsg_data_getter_fn callback */ -static u8 adf_pf2vf_blkmsg_get_byte(u8 const *blkmsg, u8 index) -{ - return blkmsg[index]; -} - -/* CRC pf2vf_blkmsg_data_getter_fn callback */ -static u8 adf_pf2vf_blkmsg_get_crc(u8 const *blkmsg, u8 count) -{ - /* count is 0-based, turn it into a length */ - return adf_pfvf_calc_blkmsg_crc(blkmsg, count + 1); -} - -static int adf_pf2vf_blkmsg_get_data(struct adf_accel_vf_info *vf_info, - u8 type, u8 byte, u8 max_size, u8 *data, - pf2vf_blkmsg_data_getter_fn data_getter) -{ - u8 blkmsg[ADF_PFVF_BLKMSG_MSG_MAX_SIZE] = { 0 }; - struct adf_accel_dev *accel_dev = vf_info->accel_dev; - adf_pf2vf_blkmsg_provider provider; - u8 msg_size; - - provider = get_blkmsg_response_provider(type); - - if (unlikely(!provider)) { - pr_err("QAT: No registered provider for message %d\n", type); - *data = ADF_PF2VF_INVALID_BLOCK_TYPE; - return -EINVAL; - } - - if (unlikely((*provider)(accel_dev, blkmsg, vf_info->vf_compat_ver))) { - pr_err("QAT: unknown error from provider for message %d\n", type); - *data = ADF_PF2VF_UNSPECIFIED_ERROR; - return -EINVAL; - } - - msg_size = ADF_PFVF_BLKMSG_HEADER_SIZE + blkmsg[ADF_PFVF_BLKMSG_LEN_BYTE]; - - if (unlikely(msg_size >= max_size)) { - pr_err("QAT: Invalid size %d provided for message type %d\n", - msg_size, type); - *data = ADF_PF2VF_PAYLOAD_TRUNCATED; - return -EINVAL; - } - - if (unlikely(byte >= msg_size)) { - pr_err("QAT: Out-of-bound byte number %d (msg size %d)\n", - byte, msg_size); - *data = ADF_PF2VF_INVALID_BYTE_NUM_REQ; - return -EINVAL; - } - - *data = data_getter(blkmsg, byte); - return 0; -} - -static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, - struct pfvf_message req) -{ - u8 resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR; - struct pfvf_message resp = { 0 }; - u8 resp_data = 0; - u8 blk_type; - u8 blk_byte; - u8 byte_max; - - switch (req.type) { - case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ: - blk_type = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, req.data) - + ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX + 1; - blk_byte = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, req.data); - byte_max = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX; - break; - case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ: - blk_type = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, req.data) - + ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1; - blk_byte = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, req.data); - byte_max = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX; - break; - case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: - blk_type = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, req.data); - blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); - byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; - break; - } - - /* Is this a request for CRC or data? */ - if (FIELD_GET(ADF_VF2PF_BLOCK_CRC_REQ_MASK, req.data)) { - dev_dbg(&GET_DEV(vf_info->accel_dev), - "BlockMsg of type %d for CRC over %d bytes received from VF%d\n", - blk_type, blk_byte + 1, vf_info->vf_nr); - - if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte, - byte_max, &resp_data, - adf_pf2vf_blkmsg_get_crc)) - resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_CRC; - } else { - dev_dbg(&GET_DEV(vf_info->accel_dev), - "BlockMsg of type %d for data byte %d received from VF%d\n", - blk_type, blk_byte, vf_info->vf_nr); - - if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte, - byte_max, &resp_data, - adf_pf2vf_blkmsg_get_byte)) - resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_DATA; - } - - resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; - resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp_type) | - FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp_data); - - return resp; -} - -static struct pfvf_message handle_rp_reset_req(struct adf_accel_dev *accel_dev, u8 vf_nr, - struct pfvf_message req) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct pfvf_message resp = { - .type = ADF_PF2VF_MSGTYPE_RP_RESET_RESP, - .data = RPRESET_SUCCESS - }; - u32 bank_number; - u32 rsvd_field; - - bank_number = FIELD_GET(ADF_VF2PF_RNG_RESET_RP_MASK, req.data); - rsvd_field = FIELD_GET(ADF_VF2PF_RNG_RESET_RSVD_MASK, req.data); - - dev_dbg(&GET_DEV(accel_dev), - "Ring Pair Reset Message received from VF%d for bank 0x%x\n", - vf_nr, bank_number); - - if (!hw_data->ring_pair_reset || rsvd_field) { - dev_dbg(&GET_DEV(accel_dev), - "Ring Pair Reset for VF%d is not supported\n", vf_nr); - resp.data = RPRESET_NOT_SUPPORTED; - goto out; - } - - if (bank_number >= hw_data->num_banks_per_vf) { - dev_err(&GET_DEV(accel_dev), - "Invalid bank number (0x%x) from VF%d for Ring Reset\n", - bank_number, vf_nr); - resp.data = RPRESET_INVAL_BANK; - goto out; - } - - /* Convert the VF provided value to PF bank number */ - bank_number = vf_nr * hw_data->num_banks_per_vf + bank_number; - if (hw_data->ring_pair_reset(accel_dev, bank_number)) { - dev_dbg(&GET_DEV(accel_dev), - "Ring pair reset for VF%d failure\n", vf_nr); - resp.data = RPRESET_TIMEOUT; - goto out; - } - - dev_dbg(&GET_DEV(accel_dev), - "Ring pair reset for VF%d successfully\n", vf_nr); - -out: - return resp; -} - -static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, - struct pfvf_message msg, struct pfvf_message *resp) -{ - struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; - - switch (msg.type) { - case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: - { - u8 vf_compat_ver = msg.data; - u8 compat; - - dev_dbg(&GET_DEV(accel_dev), - "VersionRequest received from VF%d (vers %d) to PF (vers %d)\n", - vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - - if (vf_compat_ver == 0) - compat = ADF_PF2VF_VF_INCOMPATIBLE; - else if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) - compat = ADF_PF2VF_VF_COMPATIBLE; - else - compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - - vf_info->vf_compat_ver = vf_compat_ver; - - resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; - resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, - ADF_PFVF_COMPAT_THIS_VERSION) | - FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat); - } - break; - case ADF_VF2PF_MSGTYPE_VERSION_REQ: - { - u8 compat; - - dev_dbg(&GET_DEV(accel_dev), - "Legacy VersionRequest received from VF%d to PF (vers 1.1)\n", - vf_nr); - - /* legacy driver, VF compat_ver is 0 */ - vf_info->vf_compat_ver = 0; - - /* PF always newer than legacy VF */ - compat = ADF_PF2VF_VF_COMPATIBLE; - - /* Set legacy major and minor version to the latest, 1.1 */ - resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; - resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, 0x11) | - FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat); - } - break; - case ADF_VF2PF_MSGTYPE_INIT: - { - dev_dbg(&GET_DEV(accel_dev), - "Init message received from VF%d\n", vf_nr); - vf_info->init = true; - } - break; - case ADF_VF2PF_MSGTYPE_SHUTDOWN: - { - dev_dbg(&GET_DEV(accel_dev), - "Shutdown message received from VF%d\n", vf_nr); - vf_info->init = false; - } - break; - case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ: - case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ: - case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: - *resp = handle_blkmsg_req(vf_info, msg); - break; - case ADF_VF2PF_MSGTYPE_RP_RESET: - *resp = handle_rp_reset_req(accel_dev, vf_nr, msg); - break; - default: - dev_dbg(&GET_DEV(accel_dev), - "Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n", - vf_nr, msg.type, msg.data); - return -ENOMSG; - } - - return 0; -} - -bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) -{ - struct pfvf_message req; - struct pfvf_message resp = {0}; - - req = adf_recv_vf2pf_msg(accel_dev, vf_nr); - if (!req.type) /* Legacy or no message */ - return true; - - if (adf_handle_vf2pf_msg(accel_dev, vf_nr, req, &resp)) - return false; - - if (resp.type && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) - dev_err(&GET_DEV(accel_dev), - "Failed to send response to VF%d\n", vf_nr); - - return true; -} - -/** - * adf_enable_pf2vf_comms() - Function enables communication from pf to vf - * - * @accel_dev: Pointer to acceleration device virtual function. - * - * This function carries out the necessary steps to setup and start the PFVF - * communication channel, if any. - * - * Return: 0 on success, error code otherwise. - */ -int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) -{ - adf_pfvf_crc_init(); - spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); - - return 0; -} -EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h deleted file mode 100644 index 165d266d023d..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_PFVF_PF_PROTO_H -#define ADF_PFVF_PF_PROTO_H - -#include -#include "adf_accel_devices.h" - -int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg); - -int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); - -#endif /* ADF_PFVF_PF_PROTO_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c b/drivers/crypto/qat/qat_common/adf_pfvf_utils.c deleted file mode 100644 index c5f6d77d4bb8..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2021 Intel Corporation */ -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_pfvf_msg.h" -#include "adf_pfvf_utils.h" - -/* CRC Calculation */ -DECLARE_CRC8_TABLE(pfvf_crc8_table); -#define ADF_PFVF_CRC8_POLYNOMIAL 0x97 - -void adf_pfvf_crc_init(void) -{ - crc8_populate_msb(pfvf_crc8_table, ADF_PFVF_CRC8_POLYNOMIAL); -} - -u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len) -{ - return crc8(pfvf_crc8_table, buf, buf_len, CRC8_INIT_VALUE); -} - -static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg, - u32 value, const struct pfvf_field_format *fmt) -{ - if (unlikely((value & fmt->mask) != value)) { - dev_err(&GET_DEV(accel_dev), - "PFVF message value 0x%X out of range, %u max allowed\n", - value, fmt->mask); - return false; - } - - *csr_msg |= value << fmt->offset; - - return true; -} - -u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, - struct pfvf_message msg, - const struct pfvf_csr_format *fmt) -{ - u32 csr_msg = 0; - - if (!set_value_on_csr_msg(accel_dev, &csr_msg, msg.type, &fmt->type) || - !set_value_on_csr_msg(accel_dev, &csr_msg, msg.data, &fmt->data)) - return 0; - - return csr_msg | ADF_PFVF_MSGORIGIN_SYSTEM; -} - -struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 csr_msg, - const struct pfvf_csr_format *fmt) -{ - struct pfvf_message msg = { 0 }; - - msg.type = (csr_msg >> fmt->type.offset) & fmt->type.mask; - msg.data = (csr_msg >> fmt->data.offset) & fmt->data.mask; - - if (unlikely(!msg.type)) - dev_err(&GET_DEV(accel_dev), - "Invalid PFVF msg with no type received\n"); - - return msg; -} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/qat/qat_common/adf_pfvf_utils.h deleted file mode 100644 index 2be048e2287b..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_PFVF_UTILS_H -#define ADF_PFVF_UTILS_H - -#include -#include "adf_pfvf_msg.h" - -/* How long to wait for far side to acknowledge receipt */ -#define ADF_PFVF_MSG_ACK_DELAY_US 4 -#define ADF_PFVF_MSG_ACK_MAX_DELAY_US (1 * USEC_PER_SEC) - -u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len); -void adf_pfvf_crc_init(void); - -struct pfvf_field_format { - u8 offset; - u32 mask; -}; - -struct pfvf_csr_format { - struct pfvf_field_format type; - struct pfvf_field_format data; -}; - -u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, struct pfvf_message msg, - const struct pfvf_csr_format *fmt); -struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 raw_msg, - const struct pfvf_csr_format *fmt); - -#endif /* ADF_PFVF_UTILS_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c deleted file mode 100644 index 1141258db4b6..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_pfvf_msg.h" -#include "adf_pfvf_vf_msg.h" -#include "adf_pfvf_vf_proto.h" - -/** - * adf_vf2pf_notify_init() - send init msg to PF - * @accel_dev: Pointer to acceleration VF device. - * - * Function sends an init message from the VF to a PF - * - * Return: 0 on success, error code otherwise. - */ -int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) -{ - struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_INIT }; - - if (adf_send_vf2pf_msg(accel_dev, msg)) { - dev_err(&GET_DEV(accel_dev), - "Failed to send Init event to PF\n"); - return -EFAULT; - } - set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - return 0; -} -EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); - -/** - * adf_vf2pf_notify_shutdown() - send shutdown msg to PF - * @accel_dev: Pointer to acceleration VF device. - * - * Function sends a shutdown message from the VF to a PF - * - * Return: void - */ -void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) -{ - struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_SHUTDOWN }; - - if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) - if (adf_send_vf2pf_msg(accel_dev, msg)) - dev_err(&GET_DEV(accel_dev), - "Failed to send Shutdown event to PF\n"); -} -EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); - -int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) -{ - u8 pf_version; - int compat; - int ret; - struct pfvf_message resp; - struct pfvf_message msg = { - .type = ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ, - .data = ADF_PFVF_COMPAT_THIS_VERSION, - }; - - BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); - - ret = adf_send_vf2pf_req(accel_dev, msg, &resp); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to send Compatibility Version Request.\n"); - return ret; - } - - pf_version = FIELD_GET(ADF_PF2VF_VERSION_RESP_VERS_MASK, resp.data); - compat = FIELD_GET(ADF_PF2VF_VERSION_RESP_RESULT_MASK, resp.data); - - /* Response from PF received, check compatibility */ - switch (compat) { - case ADF_PF2VF_VF_COMPATIBLE: - break; - case ADF_PF2VF_VF_COMPAT_UNKNOWN: - /* VF is newer than PF - compatible for now */ - break; - case ADF_PF2VF_VF_INCOMPATIBLE: - dev_err(&GET_DEV(accel_dev), - "PF (vers %d) and VF (vers %d) are not compatible\n", - pf_version, ADF_PFVF_COMPAT_THIS_VERSION); - return -EINVAL; - default: - dev_err(&GET_DEV(accel_dev), - "Invalid response from PF; assume not compatible\n"); - return -EINVAL; - } - - accel_dev->vf.pf_compat_ver = pf_version; - return 0; -} - -int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct capabilities_v3 cap_msg = { 0 }; - unsigned int len = sizeof(cap_msg); - - if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_CAPABILITIES) - /* The PF is too old to support the extended capabilities */ - return 0; - - if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY, - (u8 *)&cap_msg, &len)) { - dev_err(&GET_DEV(accel_dev), - "QAT: Failed to get block message response\n"); - return -EFAULT; - } - - switch (cap_msg.hdr.version) { - default: - /* Newer version received, handle only the know parts */ - fallthrough; - case ADF_PFVF_CAPABILITIES_V3_VERSION: - if (likely(len >= sizeof(struct capabilities_v3))) - hw_data->clock_frequency = cap_msg.frequency; - else - dev_info(&GET_DEV(accel_dev), "Could not get frequency"); - fallthrough; - case ADF_PFVF_CAPABILITIES_V2_VERSION: - if (likely(len >= sizeof(struct capabilities_v2))) - hw_data->accel_capabilities_mask = cap_msg.capabilities; - else - dev_info(&GET_DEV(accel_dev), "Could not get capabilities"); - fallthrough; - case ADF_PFVF_CAPABILITIES_V1_VERSION: - if (likely(len >= sizeof(struct capabilities_v1))) { - hw_data->extended_dc_capabilities = cap_msg.ext_dc_caps; - } else { - dev_err(&GET_DEV(accel_dev), - "Capabilities message truncated to %d bytes\n", len); - return -EFAULT; - } - } - - return 0; -} - -int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev) -{ - struct ring_to_svc_map_v1 rts_map_msg = { 0 }; - unsigned int len = sizeof(rts_map_msg); - - if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_RING_TO_SVC_MAP) - /* Use already set default mappings */ - return 0; - - if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP, - (u8 *)&rts_map_msg, &len)) { - dev_err(&GET_DEV(accel_dev), - "QAT: Failed to get block message response\n"); - return -EFAULT; - } - - if (unlikely(len < sizeof(struct ring_to_svc_map_v1))) { - dev_err(&GET_DEV(accel_dev), - "RING_TO_SVC message truncated to %d bytes\n", len); - return -EFAULT; - } - - /* Only v1 at present */ - accel_dev->hw_device->ring_to_svc_map = rts_map_msg.map; - return 0; -} diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h deleted file mode 100644 index 71bc0e3f1d93..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_PFVF_VF_MSG_H -#define ADF_PFVF_VF_MSG_H - -#if defined(CONFIG_PCI_IOV) -int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); -void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); -int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev); -int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev); -int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev); -#else -static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) -{ - return 0; -} - -static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) -{ -} -#endif - -#endif /* ADF_PFVF_VF_MSG_H */ diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c deleted file mode 100644 index 1015155b6374..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_pfvf_msg.h" -#include "adf_pfvf_utils.h" -#include "adf_pfvf_vf_msg.h" -#include "adf_pfvf_vf_proto.h" - -#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10 -#define ADF_PFVF_MSG_ACK_DELAY 2 -#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 - -/* How often to retry if there is no response */ -#define ADF_PFVF_MSG_RESP_RETRIES 5 -#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ - ADF_PFVF_MSG_ACK_MAX_RETRY + \ - ADF_PFVF_MSG_COLLISION_DETECT_DELAY) - -/** - * adf_send_vf2pf_msg() - send VF to PF message - * @accel_dev: Pointer to acceleration device - * @msg: Message to send - * - * This function allows the VF to send a message to the PF. - * - * Return: 0 on success, error code otherwise. - */ -int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg) -{ - struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); - u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(0); - - return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset, - &accel_dev->vf.vf2pf_lock); -} - -/** - * adf_recv_pf2vf_msg() - receive a PF to VF message - * @accel_dev: Pointer to acceleration device - * - * This function allows the VF to receive a message from the PF. - * - * Return: a valid message on success, zero otherwise. - */ -static struct pfvf_message adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) -{ - struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); - u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0); - - return pfvf_ops->recv_msg(accel_dev, pfvf_offset, accel_dev->vf.pf_compat_ver); -} - -/** - * adf_send_vf2pf_req() - send VF2PF request message - * @accel_dev: Pointer to acceleration device. - * @msg: Request message to send - * @resp: Returned PF response - * - * This function sends a message that requires a response from the VF to the PF - * and waits for a reply. - * - * Return: 0 on success, error code otherwise. - */ -int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, - struct pfvf_message *resp) -{ - unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); - unsigned int retries = ADF_PFVF_MSG_RESP_RETRIES; - int ret; - - reinit_completion(&accel_dev->vf.msg_received); - - /* Send request from VF to PF */ - do { - ret = adf_send_vf2pf_msg(accel_dev, msg); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to send request msg to PF\n"); - return ret; - } - - /* Wait for response, if it times out retry */ - ret = wait_for_completion_timeout(&accel_dev->vf.msg_received, - timeout); - if (ret) { - if (likely(resp)) - *resp = accel_dev->vf.response; - - /* Once copied, set to an invalid value */ - accel_dev->vf.response.type = 0; - - return 0; - } - - dev_err(&GET_DEV(accel_dev), "PFVF response message timeout\n"); - } while (--retries); - - return -EIO; -} - -static int adf_vf2pf_blkmsg_data_req(struct adf_accel_dev *accel_dev, bool crc, - u8 *type, u8 *data) -{ - struct pfvf_message req = { 0 }; - struct pfvf_message resp = { 0 }; - u8 blk_type; - u8 blk_byte; - u8 msg_type; - u8 max_data; - int err; - - /* Convert the block type to {small, medium, large} size category */ - if (*type <= ADF_VF2PF_SMALL_BLOCK_TYPE_MAX) { - msg_type = ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ; - blk_type = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, *type); - blk_byte = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, *data); - max_data = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; - } else if (*type <= ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) { - msg_type = ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ; - blk_type = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, - *type - ADF_VF2PF_SMALL_BLOCK_TYPE_MAX); - blk_byte = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, *data); - max_data = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX; - } else if (*type <= ADF_VF2PF_LARGE_BLOCK_TYPE_MAX) { - msg_type = ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ; - blk_type = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, - *type - ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX); - blk_byte = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, *data); - max_data = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX; - } else { - dev_err(&GET_DEV(accel_dev), "Invalid message type %u\n", *type); - return -EINVAL; - } - - /* Sanity check */ - if (*data > max_data) { - dev_err(&GET_DEV(accel_dev), - "Invalid byte %s %u for message type %u\n", - crc ? "count" : "index", *data, *type); - return -EINVAL; - } - - /* Build the block message */ - req.type = msg_type; - req.data = blk_type | blk_byte | FIELD_PREP(ADF_VF2PF_BLOCK_CRC_REQ_MASK, crc); - - err = adf_send_vf2pf_req(accel_dev, req, &resp); - if (err) - return err; - - *type = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp.data); - *data = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp.data); - - return 0; -} - -static int adf_vf2pf_blkmsg_get_byte(struct adf_accel_dev *accel_dev, u8 type, - u8 index, u8 *data) -{ - int ret; - - ret = adf_vf2pf_blkmsg_data_req(accel_dev, false, &type, &index); - if (ret < 0) - return ret; - - if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_DATA)) { - dev_err(&GET_DEV(accel_dev), - "Unexpected BLKMSG response type %u, byte 0x%x\n", - type, index); - return -EFAULT; - } - - *data = index; - return 0; -} - -static int adf_vf2pf_blkmsg_get_crc(struct adf_accel_dev *accel_dev, u8 type, - u8 bytes, u8 *crc) -{ - int ret; - - /* The count of bytes refers to a length, however shift it to a 0-based - * count to avoid overflows. Thus, a request for 0 bytes is technically - * valid. - */ - --bytes; - - ret = adf_vf2pf_blkmsg_data_req(accel_dev, true, &type, &bytes); - if (ret < 0) - return ret; - - if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_CRC)) { - dev_err(&GET_DEV(accel_dev), - "Unexpected CRC BLKMSG response type %u, crc 0x%x\n", - type, bytes); - return -EFAULT; - } - - *crc = bytes; - return 0; -} - -/** - * adf_send_vf2pf_blkmsg_req() - retrieve block message - * @accel_dev: Pointer to acceleration VF device. - * @type: The block message type, see adf_pfvf_msg.h for allowed values - * @buffer: input buffer where to place the received data - * @buffer_len: buffer length as input, the amount of written bytes on output - * - * Request a message of type 'type' over the block message transport. - * This function will send the required amount block message requests and - * return the overall content back to the caller through the provided buffer. - * The buffer should be large enough to contain the requested message type, - * otherwise the response will be truncated. - * - * Return: 0 on success, error code otherwise. - */ -int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type, - u8 *buffer, unsigned int *buffer_len) -{ - unsigned int index; - unsigned int msg_len; - int ret; - u8 remote_crc; - u8 local_crc; - - if (unlikely(type > ADF_VF2PF_LARGE_BLOCK_TYPE_MAX)) { - dev_err(&GET_DEV(accel_dev), "Invalid block message type %d\n", - type); - return -EINVAL; - } - - if (unlikely(*buffer_len < ADF_PFVF_BLKMSG_HEADER_SIZE)) { - dev_err(&GET_DEV(accel_dev), - "Buffer size too small for a block message\n"); - return -EINVAL; - } - - ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, - ADF_PFVF_BLKMSG_VER_BYTE, - &buffer[ADF_PFVF_BLKMSG_VER_BYTE]); - if (unlikely(ret)) - return ret; - - if (unlikely(!buffer[ADF_PFVF_BLKMSG_VER_BYTE])) { - dev_err(&GET_DEV(accel_dev), - "Invalid version 0 received for block request %u", type); - return -EFAULT; - } - - ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, - ADF_PFVF_BLKMSG_LEN_BYTE, - &buffer[ADF_PFVF_BLKMSG_LEN_BYTE]); - if (unlikely(ret)) - return ret; - - if (unlikely(!buffer[ADF_PFVF_BLKMSG_LEN_BYTE])) { - dev_err(&GET_DEV(accel_dev), - "Invalid size 0 received for block request %u", type); - return -EFAULT; - } - - /* We need to pick the minimum since there is no way to request a - * specific version. As a consequence any scenario is possible: - * - PF has a newer (longer) version which doesn't fit in the buffer - * - VF expects a newer (longer) version, so we must not ask for - * bytes in excess - * - PF and VF share the same version, no problem - */ - msg_len = ADF_PFVF_BLKMSG_HEADER_SIZE + buffer[ADF_PFVF_BLKMSG_LEN_BYTE]; - msg_len = min(*buffer_len, msg_len); - - /* Get the payload */ - for (index = ADF_PFVF_BLKMSG_HEADER_SIZE; index < msg_len; index++) { - ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, index, - &buffer[index]); - if (unlikely(ret)) - return ret; - } - - ret = adf_vf2pf_blkmsg_get_crc(accel_dev, type, msg_len, &remote_crc); - if (unlikely(ret)) - return ret; - - local_crc = adf_pfvf_calc_blkmsg_crc(buffer, msg_len); - if (unlikely(local_crc != remote_crc)) { - dev_err(&GET_DEV(accel_dev), - "CRC error on msg type %d. Local %02X, remote %02X\n", - type, local_crc, remote_crc); - return -EIO; - } - - *buffer_len = msg_len; - return 0; -} - -static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, - struct pfvf_message msg) -{ - switch (msg.type) { - case ADF_PF2VF_MSGTYPE_RESTARTING: - dev_dbg(&GET_DEV(accel_dev), "Restarting message received from PF\n"); - - adf_pf2vf_handle_pf_restarting(accel_dev); - return false; - case ADF_PF2VF_MSGTYPE_VERSION_RESP: - case ADF_PF2VF_MSGTYPE_BLKMSG_RESP: - case ADF_PF2VF_MSGTYPE_RP_RESET_RESP: - dev_dbg(&GET_DEV(accel_dev), - "Response Message received from PF (type 0x%.4x, data 0x%.4x)\n", - msg.type, msg.data); - accel_dev->vf.response = msg; - complete(&accel_dev->vf.msg_received); - return true; - default: - dev_err(&GET_DEV(accel_dev), - "Unknown message from PF (type 0x%.4x, data: 0x%.4x)\n", - msg.type, msg.data); - } - - return false; -} - -bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) -{ - struct pfvf_message msg; - - msg = adf_recv_pf2vf_msg(accel_dev); - if (msg.type) /* Invalid or no message */ - return adf_handle_pf2vf_msg(accel_dev, msg); - - /* No replies for PF->VF messages at present */ - - return true; -} - -/** - * adf_enable_vf2pf_comms() - Function enables communication from vf to pf - * - * @accel_dev: Pointer to acceleration device virtual function. - * - * Return: 0 on success, error code otherwise. - */ -int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) -{ - int ret; - - adf_pfvf_crc_init(); - adf_enable_pf2vf_interrupts(accel_dev); - - ret = adf_vf2pf_request_version(accel_dev); - if (ret) - return ret; - - ret = adf_vf2pf_get_capabilities(accel_dev); - if (ret) - return ret; - - ret = adf_vf2pf_get_ring_to_svc(accel_dev); - - return ret; -} -EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h deleted file mode 100644 index f6ee9b38c0e1..000000000000 --- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2021 Intel Corporation */ -#ifndef ADF_PFVF_VF_PROTO_H -#define ADF_PFVF_VF_PROTO_H - -#include -#include "adf_accel_devices.h" - -int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg); -int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, - struct pfvf_message *resp); -int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type, - u8 *buffer, unsigned int *buffer_len); - -int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); - -#endif /* ADF_PFVF_VF_PROTO_H */ diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c deleted file mode 100644 index f44025bb6f99..000000000000 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ /dev/null @@ -1,215 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include -#include -#include "adf_common_drv.h" -#include "adf_cfg.h" -#include "adf_pfvf_pf_msg.h" - -#define ADF_VF2PF_RATELIMIT_INTERVAL 8 -#define ADF_VF2PF_RATELIMIT_BURST 130 - -static struct workqueue_struct *pf2vf_resp_wq; - -struct adf_pf2vf_resp { - struct work_struct pf2vf_resp_work; - struct adf_accel_vf_info *vf_info; -}; - -static void adf_iov_send_resp(struct work_struct *work) -{ - struct adf_pf2vf_resp *pf2vf_resp = - container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work); - struct adf_accel_vf_info *vf_info = pf2vf_resp->vf_info; - struct adf_accel_dev *accel_dev = vf_info->accel_dev; - u32 vf_nr = vf_info->vf_nr; - bool ret; - - ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr); - if (ret) - /* re-enable interrupt on PF from this VF */ - adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr); - - kfree(pf2vf_resp); -} - -void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info) -{ - struct adf_pf2vf_resp *pf2vf_resp; - - pf2vf_resp = kzalloc(sizeof(*pf2vf_resp), GFP_ATOMIC); - if (!pf2vf_resp) - return; - - pf2vf_resp->vf_info = vf_info; - INIT_WORK(&pf2vf_resp->pf2vf_resp_work, adf_iov_send_resp); - queue_work(pf2vf_resp_wq, &pf2vf_resp->pf2vf_resp_work); -} - -static int adf_enable_sriov(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - int totalvfs = pci_sriov_get_totalvfs(pdev); - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_accel_vf_info *vf_info; - int i; - - for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; - i++, vf_info++) { - /* This ptr will be populated when VFs will be created */ - vf_info->accel_dev = accel_dev; - vf_info->vf_nr = i; - vf_info->vf_compat_ver = 0; - - mutex_init(&vf_info->pf2vf_lock); - ratelimit_state_init(&vf_info->vf2pf_ratelimit, - ADF_VF2PF_RATELIMIT_INTERVAL, - ADF_VF2PF_RATELIMIT_BURST); - } - - /* Set Valid bits in AE Thread to PCIe Function Mapping */ - if (hw_data->configure_iov_threads) - hw_data->configure_iov_threads(accel_dev, true); - - /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); - - /* - * Due to the hardware design, when SR-IOV and the ring arbiter - * are enabled all the VFs supported in hardware must be enabled in - * order for all the hardware resources (i.e. bundles) to be usable. - * When SR-IOV is enabled, each of the VFs will own one bundle. - */ - return pci_enable_sriov(pdev, totalvfs); -} - -/** - * adf_disable_sriov() - Disable SRIOV for the device - * @accel_dev: Pointer to accel device. - * - * Function disables SRIOV for the accel device. - * - * Return: 0 on success, error code otherwise. - */ -void adf_disable_sriov(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); - struct adf_accel_vf_info *vf; - int i; - - if (!accel_dev->pf.vf_info) - return; - - adf_pf2vf_notify_restarting(accel_dev); - pci_disable_sriov(accel_to_pci_dev(accel_dev)); - - /* Disable VF to PF interrupts */ - adf_disable_all_vf2pf_interrupts(accel_dev); - - /* Clear Valid bits in AE Thread to PCIe Function Mapping */ - if (hw_data->configure_iov_threads) - hw_data->configure_iov_threads(accel_dev, false); - - for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) - mutex_destroy(&vf->pf2vf_lock); - - kfree(accel_dev->pf.vf_info); - accel_dev->pf.vf_info = NULL; -} -EXPORT_SYMBOL_GPL(adf_disable_sriov); - -/** - * adf_sriov_configure() - Enable SRIOV for the device - * @pdev: Pointer to PCI device. - * @numvfs: Number of virtual functions (VFs) to enable. - * - * Note that the @numvfs parameter is ignored and all VFs supported by the - * device are enabled due to the design of the hardware. - * - * Function enables SRIOV for the PCI device. - * - * Return: number of VFs enabled on success, error code otherwise. - */ -int adf_sriov_configure(struct pci_dev *pdev, int numvfs) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - int totalvfs = pci_sriov_get_totalvfs(pdev); - unsigned long val; - int ret; - - if (!accel_dev) { - dev_err(&pdev->dev, "Failed to find accel_dev\n"); - return -EFAULT; - } - - if (!device_iommu_mapped(&pdev->dev)) - dev_warn(&pdev->dev, "IOMMU should be enabled for SR-IOV to work correctly\n"); - - if (accel_dev->pf.vf_info) { - dev_info(&pdev->dev, "Already enabled for this device\n"); - return -EINVAL; - } - - if (adf_dev_started(accel_dev)) { - if (adf_devmgr_in_reset(accel_dev) || - adf_dev_in_use(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Device busy\n"); - return -EBUSY; - } - - ret = adf_dev_down(accel_dev, true); - if (ret) - return ret; - } - - if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) - return -EFAULT; - val = 0; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, (void *)&val, ADF_DEC)) - return -EFAULT; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - return ret; - - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - - /* Allocate memory for VF info structs */ - accel_dev->pf.vf_info = kcalloc(totalvfs, - sizeof(struct adf_accel_vf_info), - GFP_KERNEL); - if (!accel_dev->pf.vf_info) - return -ENOMEM; - - if (adf_dev_up(accel_dev, false)) { - dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", - accel_dev->accel_id); - return -EFAULT; - } - - ret = adf_enable_sriov(accel_dev); - if (ret) - return ret; - - return numvfs; -} -EXPORT_SYMBOL_GPL(adf_sriov_configure); - -int __init adf_init_pf_wq(void) -{ - /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", WQ_MEM_RECLAIM, 0); - - return !pf2vf_resp_wq ? -ENOMEM : 0; -} - -void adf_exit_pf_wq(void) -{ - if (pf2vf_resp_wq) { - destroy_workqueue(pf2vf_resp_wq); - pf2vf_resp_wq = NULL; - } -} diff --git a/drivers/crypto/qat/qat_common/adf_sysfs.c b/drivers/crypto/qat/qat_common/adf_sysfs.c deleted file mode 100644 index 3eb6611ab1b1..000000000000 --- a/drivers/crypto/qat/qat_common/adf_sysfs.c +++ /dev/null @@ -1,174 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2022 Intel Corporation */ -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_cfg.h" -#include "adf_common_drv.h" - -static const char * const state_operations[] = { - [DEV_DOWN] = "down", - [DEV_UP] = "up", -}; - -static ssize_t state_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct adf_accel_dev *accel_dev; - char *state; - - accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); - if (!accel_dev) - return -EINVAL; - - state = adf_dev_started(accel_dev) ? "up" : "down"; - return sysfs_emit(buf, "%s\n", state); -} - -static ssize_t state_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct adf_accel_dev *accel_dev; - u32 accel_id; - int ret; - - accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); - if (!accel_dev) - return -EINVAL; - - accel_id = accel_dev->accel_id; - - if (adf_devmgr_in_reset(accel_dev) || adf_dev_in_use(accel_dev)) { - dev_info(dev, "Device qat_dev%d is busy\n", accel_id); - return -EBUSY; - } - - ret = sysfs_match_string(state_operations, buf); - if (ret < 0) - return ret; - - switch (ret) { - case DEV_DOWN: - dev_info(dev, "Stopping device qat_dev%d\n", accel_id); - - ret = adf_dev_down(accel_dev, true); - if (ret < 0) - return -EINVAL; - - break; - case DEV_UP: - dev_info(dev, "Starting device qat_dev%d\n", accel_id); - - ret = adf_dev_up(accel_dev, true); - if (ret < 0) { - dev_err(dev, "Failed to start device qat_dev%d\n", - accel_id); - adf_dev_down(accel_dev, true); - return ret; - } - break; - default: - return -EINVAL; - } - - return count; -} - -static const char * const services_operations[] = { - ADF_CFG_CY, - ADF_CFG_DC, -}; - -static ssize_t cfg_services_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - struct adf_accel_dev *accel_dev; - int ret; - - accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); - if (!accel_dev) - return -EINVAL; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - if (ret) - return ret; - - return sysfs_emit(buf, "%s\n", services); -} - -static int adf_sysfs_update_dev_config(struct adf_accel_dev *accel_dev, - const char *services) -{ - return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services, - ADF_STR); -} - -static ssize_t cfg_services_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct adf_hw_device_data *hw_data; - struct adf_accel_dev *accel_dev; - int ret; - - ret = sysfs_match_string(services_operations, buf); - if (ret < 0) - return ret; - - accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); - if (!accel_dev) - return -EINVAL; - - if (adf_dev_started(accel_dev)) { - dev_info(dev, "Device qat_dev%d must be down to reconfigure the service.\n", - accel_dev->accel_id); - return -EINVAL; - } - - ret = adf_sysfs_update_dev_config(accel_dev, services_operations[ret]); - if (ret < 0) - return ret; - - hw_data = GET_HW_DATA(accel_dev); - - /* Update capabilities mask after change in configuration. - * A call to this function is required as capabilities are, at the - * moment, tied to configuration - */ - hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); - if (!hw_data->accel_capabilities_mask) - return -EINVAL; - - return count; -} - -static DEVICE_ATTR_RW(state); -static DEVICE_ATTR_RW(cfg_services); - -static struct attribute *qat_attrs[] = { - &dev_attr_state.attr, - &dev_attr_cfg_services.attr, - NULL, -}; - -static struct attribute_group qat_group = { - .attrs = qat_attrs, - .name = "qat", -}; - -int adf_sysfs_init(struct adf_accel_dev *accel_dev) -{ - int ret; - - ret = devm_device_add_group(&GET_DEV(accel_dev), &qat_group); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "Failed to create qat attribute group: %d\n", ret); - } - - return ret; -} -EXPORT_SYMBOL_GPL(adf_sysfs_init); diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c deleted file mode 100644 index 630d0483c4e0..000000000000 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ /dev/null @@ -1,577 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include "adf_accel_devices.h" -#include "adf_transport_internal.h" -#include "adf_transport_access_macros.h" -#include "adf_cfg.h" -#include "adf_common_drv.h" - -#define ADF_MAX_RING_THRESHOLD 80 -#define ADF_PERCENT(tot, percent) (((tot) * (percent)) / 100) - -static inline u32 adf_modulo(u32 data, u32 shift) -{ - u32 div = data >> shift; - u32 mult = div << shift; - - return data - mult; -} - -static inline int adf_check_ring_alignment(u64 addr, u64 size) -{ - if (((size - 1) & addr) != 0) - return -EFAULT; - return 0; -} - -static int adf_verify_ring_size(u32 msg_size, u32 msg_num) -{ - int i = ADF_MIN_RING_SIZE; - - for (; i <= ADF_MAX_RING_SIZE; i++) - if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) - return i; - - return ADF_DEFAULT_RING_SIZE; -} - -static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring) -{ - spin_lock(&bank->lock); - if (bank->ring_mask & (1 << ring)) { - spin_unlock(&bank->lock); - return -EFAULT; - } - bank->ring_mask |= (1 << ring); - spin_unlock(&bank->lock); - return 0; -} - -static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) -{ - spin_lock(&bank->lock); - bank->ring_mask &= ~(1 << ring); - spin_unlock(&bank->lock); -} - -static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) -{ - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - - spin_lock_bh(&bank->lock); - bank->irq_mask |= (1 << ring); - spin_unlock_bh(&bank->lock); - csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, - bank->irq_mask); - csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number, - bank->irq_coalesc_timer); -} - -static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) -{ - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - - spin_lock_bh(&bank->lock); - bank->irq_mask &= ~(1 << ring); - spin_unlock_bh(&bank->lock); - csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, - bank->irq_mask); -} - -bool adf_ring_nearly_full(struct adf_etr_ring_data *ring) -{ - return atomic_read(ring->inflights) > ring->threshold; -} - -int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) -{ - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); - - if (atomic_add_return(1, ring->inflights) > - ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { - atomic_dec(ring->inflights); - return -EAGAIN; - } - spin_lock_bh(&ring->lock); - memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg, - ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); - - ring->tail = adf_modulo(ring->tail + - ADF_MSG_SIZE_TO_BYTES(ring->msg_size), - ADF_RING_SIZE_MODULO(ring->ring_size)); - csr_ops->write_csr_ring_tail(ring->bank->csr_addr, - ring->bank->bank_number, ring->ring_number, - ring->tail); - spin_unlock_bh(&ring->lock); - - return 0; -} - -static int adf_handle_response(struct adf_etr_ring_data *ring) -{ - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); - u32 msg_counter = 0; - u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); - - while (*msg != ADF_RING_EMPTY_SIG) { - ring->callback((u32 *)msg); - atomic_dec(ring->inflights); - *msg = ADF_RING_EMPTY_SIG; - ring->head = adf_modulo(ring->head + - ADF_MSG_SIZE_TO_BYTES(ring->msg_size), - ADF_RING_SIZE_MODULO(ring->ring_size)); - msg_counter++; - msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); - } - if (msg_counter > 0) { - csr_ops->write_csr_ring_head(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, ring->head); - } - return 0; -} - -static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) -{ - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); - u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); - - csr_ops->write_csr_ring_config(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, ring_config); - -} - -static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) -{ - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); - u32 ring_config = - BUILD_RESP_RING_CONFIG(ring->ring_size, - ADF_RING_NEAR_WATERMARK_512, - ADF_RING_NEAR_WATERMARK_0); - - csr_ops->write_csr_ring_config(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, ring_config); -} - -static int adf_init_ring(struct adf_etr_ring_data *ring) -{ - struct adf_etr_bank_data *bank = ring->bank; - struct adf_accel_dev *accel_dev = bank->accel_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); - u64 ring_base; - u32 ring_size_bytes = - ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); - - ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); - ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev), - ring_size_bytes, &ring->dma_addr, - GFP_KERNEL); - if (!ring->base_addr) - return -ENOMEM; - - memset(ring->base_addr, 0x7F, ring_size_bytes); - /* The base_addr has to be aligned to the size of the buffer */ - if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { - dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n"); - dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, - ring->base_addr, ring->dma_addr); - ring->base_addr = NULL; - return -EFAULT; - } - - if (hw_data->tx_rings_mask & (1 << ring->ring_number)) - adf_configure_tx_ring(ring); - - else - adf_configure_rx_ring(ring); - - ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr, - ring->ring_size); - - csr_ops->write_csr_ring_base(ring->bank->csr_addr, - ring->bank->bank_number, ring->ring_number, - ring_base); - spin_lock_init(&ring->lock); - return 0; -} - -static void adf_cleanup_ring(struct adf_etr_ring_data *ring) -{ - u32 ring_size_bytes = - ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); - ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); - - if (ring->base_addr) { - memset(ring->base_addr, 0x7F, ring_size_bytes); - dma_free_coherent(&GET_DEV(ring->bank->accel_dev), - ring_size_bytes, ring->base_addr, - ring->dma_addr); - } -} - -int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, - u32 bank_num, u32 num_msgs, - u32 msg_size, const char *ring_name, - adf_callback_fn callback, int poll_mode, - struct adf_etr_ring_data **ring_ptr) -{ - struct adf_etr_data *transport_data = accel_dev->transport; - u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); - struct adf_etr_bank_data *bank; - struct adf_etr_ring_data *ring; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - int max_inflights; - u32 ring_num; - int ret; - - if (bank_num >= GET_MAX_BANKS(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Invalid bank number\n"); - return -EFAULT; - } - if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { - dev_err(&GET_DEV(accel_dev), "Invalid msg size\n"); - return -EFAULT; - } - if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), - ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { - dev_err(&GET_DEV(accel_dev), - "Invalid ring size for given msg size\n"); - return -EFAULT; - } - if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { - dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n", - section, ring_name); - return -EFAULT; - } - if (kstrtouint(val, 10, &ring_num)) { - dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); - return -EFAULT; - } - if (ring_num >= num_rings_per_bank) { - dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); - return -EFAULT; - } - - ring_num = array_index_nospec(ring_num, num_rings_per_bank); - bank = &transport_data->banks[bank_num]; - if (adf_reserve_ring(bank, ring_num)) { - dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n", - ring_num, ring_name); - return -EFAULT; - } - ring = &bank->rings[ring_num]; - ring->ring_number = ring_num; - ring->bank = bank; - ring->callback = callback; - ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size); - ring->ring_size = adf_verify_ring_size(msg_size, num_msgs); - ring->head = 0; - ring->tail = 0; - max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size); - ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD); - atomic_set(ring->inflights, 0); - ret = adf_init_ring(ring); - if (ret) - goto err; - - /* Enable HW arbitration for the given ring */ - adf_update_ring_arb(ring); - - if (adf_ring_debugfs_add(ring, ring_name)) { - dev_err(&GET_DEV(accel_dev), - "Couldn't add ring debugfs entry\n"); - ret = -EFAULT; - goto err; - } - - /* Enable interrupts if needed */ - if (callback && (!poll_mode)) - adf_enable_ring_irq(bank, ring->ring_number); - *ring_ptr = ring; - return 0; -err: - adf_cleanup_ring(ring); - adf_unreserve_ring(bank, ring_num); - adf_update_ring_arb(ring); - return ret; -} - -void adf_remove_ring(struct adf_etr_ring_data *ring) -{ - struct adf_etr_bank_data *bank = ring->bank; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - - /* Disable interrupts for the given ring */ - adf_disable_ring_irq(bank, ring->ring_number); - - /* Clear PCI config space */ - - csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); - csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); - adf_ring_debugfs_rm(ring); - adf_unreserve_ring(bank, ring->ring_number); - /* Disable HW arbitration for the given ring */ - adf_update_ring_arb(ring); - adf_cleanup_ring(ring); -} - -static void adf_ring_response_handler(struct adf_etr_bank_data *bank) -{ - struct adf_accel_dev *accel_dev = bank->accel_dev; - u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); - unsigned long empty_rings; - int i; - - empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr, - bank->bank_number); - empty_rings = ~empty_rings & bank->irq_mask; - - for_each_set_bit(i, &empty_rings, num_rings_per_bank) - adf_handle_response(&bank->rings[i]); -} - -void adf_response_handler(uintptr_t bank_addr) -{ - struct adf_etr_bank_data *bank = (void *)bank_addr; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - - /* Handle all the responses and reenable IRQs */ - adf_ring_response_handler(bank); - - csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, - bank->irq_mask); -} - -static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, - const char *section, const char *format, - u32 key, u32 *value) -{ - char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - - snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key); - - if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf)) - return -EFAULT; - - if (kstrtouint(val_buf, 10, value)) - return -EFAULT; - return 0; -} - -static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank, - const char *section, - u32 bank_num_in_accel) -{ - if (adf_get_cfg_int(bank->accel_dev, section, - ADF_ETRMGR_COALESCE_TIMER_FORMAT, - bank_num_in_accel, &bank->irq_coalesc_timer)) - bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; - - if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer || - ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer) - bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; -} - -static int adf_init_bank(struct adf_accel_dev *accel_dev, - struct adf_etr_bank_data *bank, - u32 bank_num, void __iomem *csr_addr) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u8 num_rings_per_bank = hw_data->num_rings_per_bank; - struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; - u32 irq_mask = BIT(num_rings_per_bank) - 1; - struct adf_etr_ring_data *ring; - struct adf_etr_ring_data *tx_ring; - u32 i, coalesc_enabled = 0; - unsigned long ring_mask; - int size; - - memset(bank, 0, sizeof(*bank)); - bank->bank_number = bank_num; - bank->csr_addr = csr_addr; - bank->accel_dev = accel_dev; - spin_lock_init(&bank->lock); - - /* Allocate the rings in the bank */ - size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); - bank->rings = kzalloc_node(size, GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!bank->rings) - return -ENOMEM; - - /* Enable IRQ coalescing always. This will allow to use - * the optimised flag and coalesc register. - * If it is disabled in the config file just use min time value */ - if ((adf_get_cfg_int(accel_dev, "Accelerator0", - ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num, - &coalesc_enabled) == 0) && coalesc_enabled) - adf_get_coalesc_timer(bank, "Accelerator0", bank_num); - else - bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; - - for (i = 0; i < num_rings_per_bank; i++) { - csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); - csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); - - ring = &bank->rings[i]; - if (hw_data->tx_rings_mask & (1 << i)) { - ring->inflights = - kzalloc_node(sizeof(atomic_t), - GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!ring->inflights) - goto err; - } else { - if (i < hw_data->tx_rx_gap) { - dev_err(&GET_DEV(accel_dev), - "Invalid tx rings mask config\n"); - goto err; - } - tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; - ring->inflights = tx_ring->inflights; - } - } - if (adf_bank_debugfs_add(bank)) { - dev_err(&GET_DEV(accel_dev), - "Failed to add bank debugfs entry\n"); - goto err; - } - - csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); - csr_ops->write_csr_int_srcsel(csr_addr, bank_num); - - return 0; -err: - ring_mask = hw_data->tx_rings_mask; - for_each_set_bit(i, &ring_mask, num_rings_per_bank) { - ring = &bank->rings[i]; - kfree(ring->inflights); - ring->inflights = NULL; - } - kfree(bank->rings); - return -ENOMEM; -} - -/** - * adf_init_etr_data() - Initialize transport rings for acceleration device - * @accel_dev: Pointer to acceleration device. - * - * Function is the initializes the communications channels (rings) to the - * acceleration device accel_dev. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ -int adf_init_etr_data(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *etr_data; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *csr_addr; - u32 size; - u32 num_banks = 0; - int i, ret; - - etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!etr_data) - return -ENOMEM; - - num_banks = GET_MAX_BANKS(accel_dev); - size = num_banks * sizeof(struct adf_etr_bank_data); - etr_data->banks = kzalloc_node(size, GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!etr_data->banks) { - ret = -ENOMEM; - goto err_bank; - } - - accel_dev->transport = etr_data; - i = hw_data->get_etr_bar_id(hw_data); - csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; - - /* accel_dev->debugfs_dir should always be non-NULL here */ - etr_data->debug = debugfs_create_dir("transport", - accel_dev->debugfs_dir); - - for (i = 0; i < num_banks; i++) { - ret = adf_init_bank(accel_dev, &etr_data->banks[i], i, - csr_addr); - if (ret) - goto err_bank_all; - } - - return 0; - -err_bank_all: - debugfs_remove(etr_data->debug); - kfree(etr_data->banks); -err_bank: - kfree(etr_data); - accel_dev->transport = NULL; - return ret; -} -EXPORT_SYMBOL_GPL(adf_init_etr_data); - -static void cleanup_bank(struct adf_etr_bank_data *bank) -{ - struct adf_accel_dev *accel_dev = bank->accel_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u8 num_rings_per_bank = hw_data->num_rings_per_bank; - u32 i; - - for (i = 0; i < num_rings_per_bank; i++) { - struct adf_etr_ring_data *ring = &bank->rings[i]; - - if (bank->ring_mask & (1 << i)) - adf_cleanup_ring(ring); - - if (hw_data->tx_rings_mask & (1 << i)) - kfree(ring->inflights); - } - kfree(bank->rings); - adf_bank_debugfs_rm(bank); - memset(bank, 0, sizeof(*bank)); -} - -static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *etr_data = accel_dev->transport; - u32 i, num_banks = GET_MAX_BANKS(accel_dev); - - for (i = 0; i < num_banks; i++) - cleanup_bank(&etr_data->banks[i]); -} - -/** - * adf_cleanup_etr_data() - Clear transport rings for acceleration device - * @accel_dev: Pointer to acceleration device. - * - * Function is the clears the communications channels (rings) of the - * acceleration device accel_dev. - * To be used by QAT device specific drivers. - * - * Return: void - */ -void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *etr_data = accel_dev->transport; - - if (etr_data) { - adf_cleanup_etr_handles(accel_dev); - debugfs_remove(etr_data->debug); - kfree(etr_data->banks->rings); - kfree(etr_data->banks); - kfree(etr_data); - accel_dev->transport = NULL; - } -} -EXPORT_SYMBOL_GPL(adf_cleanup_etr_data); diff --git a/drivers/crypto/qat/qat_common/adf_transport.h b/drivers/crypto/qat/qat_common/adf_transport.h deleted file mode 100644 index e6ef6f9b7691..000000000000 --- a/drivers/crypto/qat/qat_common/adf_transport.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_TRANSPORT_H -#define ADF_TRANSPORT_H - -#include "adf_accel_devices.h" - -struct adf_etr_ring_data; - -typedef void (*adf_callback_fn)(void *resp_msg); - -int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, - u32 bank_num, u32 num_mgs, u32 msg_size, - const char *ring_name, adf_callback_fn callback, - int poll_mode, struct adf_etr_ring_data **ring_ptr); - -bool adf_ring_nearly_full(struct adf_etr_ring_data *ring); -int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg); -void adf_remove_ring(struct adf_etr_ring_data *ring); -#endif diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h deleted file mode 100644 index d3667dbd9826..000000000000 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_TRANSPORT_ACCESS_MACROS_H -#define ADF_TRANSPORT_ACCESS_MACROS_H - -#include "adf_accel_devices.h" -#define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A -#define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 -#define ADF_COALESCING_MIN_TIME 0x1FF -#define ADF_COALESCING_MAX_TIME 0xFFFFF -#define ADF_COALESCING_DEF_TIME 0x27FF -#define ADF_RING_NEAR_WATERMARK_512 0x08 -#define ADF_RING_NEAR_WATERMARK_0 0x00 -#define ADF_RING_EMPTY_SIG 0x7F7F7F7F - -/* Valid internal ring size values */ -#define ADF_RING_SIZE_128 0x01 -#define ADF_RING_SIZE_256 0x02 -#define ADF_RING_SIZE_512 0x03 -#define ADF_RING_SIZE_4K 0x06 -#define ADF_RING_SIZE_16K 0x08 -#define ADF_RING_SIZE_4M 0x10 -#define ADF_MIN_RING_SIZE ADF_RING_SIZE_128 -#define ADF_MAX_RING_SIZE ADF_RING_SIZE_4M -#define ADF_DEFAULT_RING_SIZE ADF_RING_SIZE_16K - -/* Valid internal msg size values */ -#define ADF_MSG_SIZE_32 0x01 -#define ADF_MSG_SIZE_64 0x02 -#define ADF_MSG_SIZE_128 0x04 -#define ADF_MIN_MSG_SIZE ADF_MSG_SIZE_32 -#define ADF_MAX_MSG_SIZE ADF_MSG_SIZE_128 - -/* Size to bytes conversion macros for ring and msg size values */ -#define ADF_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5) -#define ADF_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5) -#define ADF_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7) -#define ADF_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) - -/* Minimum ring buffer size for memory allocation */ -#define ADF_RING_SIZE_BYTES_MIN(SIZE) \ - ((SIZE < ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K)) ? \ - ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K) : SIZE) -#define ADF_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) -#define ADF_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \ - SIZE) & ~0x4) -/* Max outstanding requests */ -#define ADF_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \ - ((((1 << (RING_SIZE - 1)) << 3) >> ADF_SIZE_TO_POW(MSG_SIZE)) - 1) -#define BUILD_RING_CONFIG(size) \ - ((ADF_RING_NEAR_WATERMARK_0 << ADF_RING_CONFIG_NEAR_FULL_WM) \ - | (ADF_RING_NEAR_WATERMARK_0 << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ - | size) -#define BUILD_RESP_RING_CONFIG(size, watermark_nf, watermark_ne) \ - ((watermark_nf << ADF_RING_CONFIG_NEAR_FULL_WM) \ - | (watermark_ne << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ - | size) -#endif diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c deleted file mode 100644 index 08bca1c506c0..000000000000 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_transport_internal.h" -#include "adf_transport_access_macros.h" - -static DEFINE_MUTEX(ring_read_lock); -static DEFINE_MUTEX(bank_read_lock); - -static void *adf_ring_start(struct seq_file *sfile, loff_t *pos) -{ - struct adf_etr_ring_data *ring = sfile->private; - - mutex_lock(&ring_read_lock); - if (*pos == 0) - return SEQ_START_TOKEN; - - if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / - ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) - return NULL; - - return ring->base_addr + - (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); -} - -static void *adf_ring_next(struct seq_file *sfile, void *v, loff_t *pos) -{ - struct adf_etr_ring_data *ring = sfile->private; - - if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / - ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) - return NULL; - - return ring->base_addr + - (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); -} - -static int adf_ring_show(struct seq_file *sfile, void *v) -{ - struct adf_etr_ring_data *ring = sfile->private; - struct adf_etr_bank_data *bank = ring->bank; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - void __iomem *csr = ring->bank->csr_addr; - - if (v == SEQ_START_TOKEN) { - int head, tail, empty; - - head = csr_ops->read_csr_ring_head(csr, bank->bank_number, - ring->ring_number); - tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, - ring->ring_number); - empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); - - seq_puts(sfile, "------- Ring configuration -------\n"); - seq_printf(sfile, "ring name: %s\n", - ring->ring_debug->ring_name); - seq_printf(sfile, "ring num %d, bank num %d\n", - ring->ring_number, ring->bank->bank_number); - seq_printf(sfile, "head %x, tail %x, empty: %d\n", - head, tail, (empty & 1 << ring->ring_number) - >> ring->ring_number); - seq_printf(sfile, "ring size %lld, msg size %d\n", - (long long)ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), - ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); - seq_puts(sfile, "----------- Ring data ------------\n"); - return 0; - } - seq_hex_dump(sfile, "", DUMP_PREFIX_ADDRESS, 32, 4, - v, ADF_MSG_SIZE_TO_BYTES(ring->msg_size), false); - return 0; -} - -static void adf_ring_stop(struct seq_file *sfile, void *v) -{ - mutex_unlock(&ring_read_lock); -} - -static const struct seq_operations adf_ring_debug_sops = { - .start = adf_ring_start, - .next = adf_ring_next, - .stop = adf_ring_stop, - .show = adf_ring_show -}; - -DEFINE_SEQ_ATTRIBUTE(adf_ring_debug); - -int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) -{ - struct adf_etr_ring_debug_entry *ring_debug; - char entry_name[8]; - - ring_debug = kzalloc(sizeof(*ring_debug), GFP_KERNEL); - if (!ring_debug) - return -ENOMEM; - - strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); - snprintf(entry_name, sizeof(entry_name), "ring_%02d", - ring->ring_number); - - ring_debug->debug = debugfs_create_file(entry_name, S_IRUSR, - ring->bank->bank_debug_dir, - ring, &adf_ring_debug_fops); - ring->ring_debug = ring_debug; - return 0; -} - -void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring) -{ - if (ring->ring_debug) { - debugfs_remove(ring->ring_debug->debug); - kfree(ring->ring_debug); - ring->ring_debug = NULL; - } -} - -static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) -{ - struct adf_etr_bank_data *bank = sfile->private; - u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); - - mutex_lock(&bank_read_lock); - if (*pos == 0) - return SEQ_START_TOKEN; - - if (*pos >= num_rings_per_bank) - return NULL; - - return pos; -} - -static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) -{ - struct adf_etr_bank_data *bank = sfile->private; - u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); - - if (++(*pos) >= num_rings_per_bank) - return NULL; - - return pos; -} - -static int adf_bank_show(struct seq_file *sfile, void *v) -{ - struct adf_etr_bank_data *bank = sfile->private; - struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - - if (v == SEQ_START_TOKEN) { - seq_printf(sfile, "------- Bank %d configuration -------\n", - bank->bank_number); - } else { - int ring_id = *((int *)v) - 1; - struct adf_etr_ring_data *ring = &bank->rings[ring_id]; - void __iomem *csr = bank->csr_addr; - int head, tail, empty; - - if (!(bank->ring_mask & 1 << ring_id)) - return 0; - - head = csr_ops->read_csr_ring_head(csr, bank->bank_number, - ring->ring_number); - tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, - ring->ring_number); - empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); - - seq_printf(sfile, - "ring num %02d, head %04x, tail %04x, empty: %d\n", - ring->ring_number, head, tail, - (empty & 1 << ring->ring_number) >> - ring->ring_number); - } - return 0; -} - -static void adf_bank_stop(struct seq_file *sfile, void *v) -{ - mutex_unlock(&bank_read_lock); -} - -static const struct seq_operations adf_bank_debug_sops = { - .start = adf_bank_start, - .next = adf_bank_next, - .stop = adf_bank_stop, - .show = adf_bank_show -}; - -DEFINE_SEQ_ATTRIBUTE(adf_bank_debug); - -int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) -{ - struct adf_accel_dev *accel_dev = bank->accel_dev; - struct dentry *parent = accel_dev->transport->debug; - char name[8]; - - snprintf(name, sizeof(name), "bank_%02d", bank->bank_number); - bank->bank_debug_dir = debugfs_create_dir(name, parent); - bank->bank_debug_cfg = debugfs_create_file("config", S_IRUSR, - bank->bank_debug_dir, bank, - &adf_bank_debug_fops); - return 0; -} - -void adf_bank_debugfs_rm(struct adf_etr_bank_data *bank) -{ - debugfs_remove(bank->bank_debug_cfg); - debugfs_remove(bank->bank_debug_dir); -} diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h deleted file mode 100644 index 8b2c92ba7ca1..000000000000 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ /dev/null @@ -1,73 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_TRANSPORT_INTRN_H -#define ADF_TRANSPORT_INTRN_H - -#include -#include -#include "adf_transport.h" - -struct adf_etr_ring_debug_entry { - char ring_name[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - struct dentry *debug; -}; - -struct adf_etr_ring_data { - void *base_addr; - atomic_t *inflights; - adf_callback_fn callback; - struct adf_etr_bank_data *bank; - dma_addr_t dma_addr; - struct adf_etr_ring_debug_entry *ring_debug; - spinlock_t lock; /* protects ring data struct */ - u16 head; - u16 tail; - u32 threshold; - u8 ring_number; - u8 ring_size; - u8 msg_size; -}; - -struct adf_etr_bank_data { - struct adf_etr_ring_data *rings; - struct tasklet_struct resp_handler; - void __iomem *csr_addr; - u32 irq_coalesc_timer; - u32 bank_number; - u16 ring_mask; - u16 irq_mask; - spinlock_t lock; /* protects bank data struct */ - struct adf_accel_dev *accel_dev; - struct dentry *bank_debug_dir; - struct dentry *bank_debug_cfg; -}; - -struct adf_etr_data { - struct adf_etr_bank_data *banks; - struct dentry *debug; -}; - -void adf_response_handler(uintptr_t bank_addr); -#ifdef CONFIG_DEBUG_FS -#include -int adf_bank_debugfs_add(struct adf_etr_bank_data *bank); -void adf_bank_debugfs_rm(struct adf_etr_bank_data *bank); -int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name); -void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring); -#else -static inline int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) -{ - return 0; -} - -#define adf_bank_debugfs_rm(bank) do {} while (0) - -static inline int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, - const char *name) -{ - return 0; -} - -#define adf_ring_debugfs_rm(ring) do {} while (0) -#endif -#endif diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c deleted file mode 100644 index b05c3957a160..000000000000 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ /dev/null @@ -1,313 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_cfg.h" -#include "adf_cfg_strings.h" -#include "adf_cfg_common.h" -#include "adf_transport_access_macros.h" -#include "adf_transport_internal.h" - -#define ADF_VINTSOU_OFFSET 0x204 -#define ADF_VINTMSK_OFFSET 0x208 -#define ADF_VINTSOU_BUN BIT(0) -#define ADF_VINTSOU_PF2VF BIT(1) - -static struct workqueue_struct *adf_vf_stop_wq; - -struct adf_vf_stop_data { - struct adf_accel_dev *accel_dev; - struct work_struct work; -}; - -void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - - ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x0); -} - -void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) -{ - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - - ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x2); -} -EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); - -static int adf_enable_msi(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; - int stat = pci_alloc_irq_vectors(pci_dev_info->pci_dev, 1, 1, - PCI_IRQ_MSI); - if (unlikely(stat < 0)) { - dev_err(&GET_DEV(accel_dev), - "Failed to enable MSI interrupt: %d\n", stat); - return stat; - } - - return 0; -} - -static void adf_disable_msi(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - - pci_free_irq_vectors(pdev); -} - -static void adf_dev_stop_async(struct work_struct *work) -{ - struct adf_vf_stop_data *stop_data = - container_of(work, struct adf_vf_stop_data, work); - struct adf_accel_dev *accel_dev = stop_data->accel_dev; - - adf_dev_restarting_notify(accel_dev); - adf_dev_down(accel_dev, false); - - /* Re-enable PF2VF interrupts */ - adf_enable_pf2vf_interrupts(accel_dev); - kfree(stop_data); -} - -int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev) -{ - struct adf_vf_stop_data *stop_data; - - clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); - stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC); - if (!stop_data) { - dev_err(&GET_DEV(accel_dev), - "Couldn't schedule stop for vf_%d\n", - accel_dev->accel_id); - return -ENOMEM; - } - stop_data->accel_dev = accel_dev; - INIT_WORK(&stop_data->work, adf_dev_stop_async); - queue_work(adf_vf_stop_wq, &stop_data->work); - - return 0; -} - -static void adf_pf2vf_bh_handler(void *data) -{ - struct adf_accel_dev *accel_dev = data; - bool ret; - - ret = adf_recv_and_handle_pf2vf_msg(accel_dev); - if (ret) - /* Re-enable PF2VF interrupts */ - adf_enable_pf2vf_interrupts(accel_dev); - - return; - -} - -static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) -{ - tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet, - (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev); - - mutex_init(&accel_dev->vf.vf2pf_lock); - return 0; -} - -static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) -{ - tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet); - tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet); - mutex_destroy(&accel_dev->vf.vf2pf_lock); -} - -static irqreturn_t adf_isr(int irq, void *privdata) -{ - struct adf_accel_dev *accel_dev = privdata; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_bar_addr = pmisc->virt_addr; - bool handled = false; - u32 v_int, v_mask; - - /* Read VF INT source CSR to determine the source of VF interrupt */ - v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET); - - /* Read VF INT mask CSR to determine which sources are masked */ - v_mask = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTMSK_OFFSET); - - /* - * Recompute v_int ignoring sources that are masked. This is to - * avoid rescheduling the tasklet for interrupts already handled - */ - v_int &= ~v_mask; - - /* Check for PF2VF interrupt */ - if (v_int & ADF_VINTSOU_PF2VF) { - /* Disable PF to VF interrupt */ - adf_disable_pf2vf_interrupts(accel_dev); - - /* Schedule tasklet to handle interrupt BH */ - tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet); - handled = true; - } - - /* Check bundle interrupt */ - if (v_int & ADF_VINTSOU_BUN) { - struct adf_etr_data *etr_data = accel_dev->transport; - struct adf_etr_bank_data *bank = &etr_data->banks[0]; - - /* Disable Flag and Coalesce Ring Interrupts */ - csr_ops->write_csr_int_flag_and_col(bank->csr_addr, - bank->bank_number, 0); - tasklet_hi_schedule(&bank->resp_handler); - handled = true; - } - - return handled ? IRQ_HANDLED : IRQ_NONE; -} - -static int adf_request_msi_irq(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - unsigned int cpu; - int ret; - - snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME, - "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); - ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name, - (void *)accel_dev); - if (ret) { - dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n", - accel_dev->vf.irq_name); - return ret; - } - cpu = accel_dev->accel_id % num_online_cpus(); - irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu)); - accel_dev->vf.irq_enabled = true; - - return ret; -} - -static int adf_setup_bh(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *priv_data = accel_dev->transport; - - tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler, - (unsigned long)priv_data->banks); - return 0; -} - -static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) -{ - struct adf_etr_data *priv_data = accel_dev->transport; - - tasklet_disable(&priv_data->banks[0].resp_handler); - tasklet_kill(&priv_data->banks[0].resp_handler); -} - -/** - * adf_vf_isr_resource_free() - Free IRQ for acceleration device - * @accel_dev: Pointer to acceleration device. - * - * Function frees interrupts for acceleration device virtual function. - */ -void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - - if (accel_dev->vf.irq_enabled) { - irq_set_affinity_hint(pdev->irq, NULL); - free_irq(pdev->irq, accel_dev); - } - adf_cleanup_bh(accel_dev); - adf_cleanup_pf2vf_bh(accel_dev); - adf_disable_msi(accel_dev); -} -EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free); - -/** - * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device - * @accel_dev: Pointer to acceleration device. - * - * Function allocates interrupts for acceleration device virtual function. - * - * Return: 0 on success, error code otherwise. - */ -int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) -{ - if (adf_enable_msi(accel_dev)) - goto err_out; - - if (adf_setup_pf2vf_bh(accel_dev)) - goto err_disable_msi; - - if (adf_setup_bh(accel_dev)) - goto err_cleanup_pf2vf_bh; - - if (adf_request_msi_irq(accel_dev)) - goto err_cleanup_bh; - - return 0; - -err_cleanup_bh: - adf_cleanup_bh(accel_dev); - -err_cleanup_pf2vf_bh: - adf_cleanup_pf2vf_bh(accel_dev); - -err_disable_msi: - adf_disable_msi(accel_dev); - -err_out: - return -EFAULT; -} -EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); - -/** - * adf_flush_vf_wq() - Flush workqueue for VF - * @accel_dev: Pointer to acceleration device. - * - * Function disables the PF/VF interrupts on the VF so that no new messages - * are received and flushes the workqueue 'adf_vf_stop_wq'. - * - * Return: void. - */ -void adf_flush_vf_wq(struct adf_accel_dev *accel_dev) -{ - adf_disable_pf2vf_interrupts(accel_dev); - - flush_workqueue(adf_vf_stop_wq); -} -EXPORT_SYMBOL_GPL(adf_flush_vf_wq); - -/** - * adf_init_vf_wq() - Init workqueue for VF - * - * Function init workqueue 'adf_vf_stop_wq' for VF. - * - * Return: 0 on success, error code otherwise. - */ -int __init adf_init_vf_wq(void) -{ - adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); - - return !adf_vf_stop_wq ? -EFAULT : 0; -} - -void adf_exit_vf_wq(void) -{ - if (adf_vf_stop_wq) - destroy_workqueue(adf_vf_stop_wq); - - adf_vf_stop_wq = NULL; -} diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw.h b/drivers/crypto/qat/qat_common/icp_qat_fw.h deleted file mode 100644 index c141160421e1..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_fw.h +++ /dev/null @@ -1,298 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef _ICP_QAT_FW_H_ -#define _ICP_QAT_FW_H_ -#include -#include "icp_qat_hw.h" - -#define QAT_FIELD_SET(flags, val, bitpos, mask) \ -{ (flags) = (((flags) & (~((mask) << (bitpos)))) | \ - (((val) & (mask)) << (bitpos))) ; } - -#define QAT_FIELD_GET(flags, bitpos, mask) \ - (((flags) >> (bitpos)) & (mask)) - -#define ICP_QAT_FW_REQ_DEFAULT_SZ 128 -#define ICP_QAT_FW_RESP_DEFAULT_SZ 32 -#define ICP_QAT_FW_COMN_ONE_BYTE_SHIFT 8 -#define ICP_QAT_FW_COMN_SINGLE_BYTE_MASK 0xFF -#define ICP_QAT_FW_NUM_LONGWORDS_1 1 -#define ICP_QAT_FW_NUM_LONGWORDS_2 2 -#define ICP_QAT_FW_NUM_LONGWORDS_3 3 -#define ICP_QAT_FW_NUM_LONGWORDS_4 4 -#define ICP_QAT_FW_NUM_LONGWORDS_5 5 -#define ICP_QAT_FW_NUM_LONGWORDS_6 6 -#define ICP_QAT_FW_NUM_LONGWORDS_7 7 -#define ICP_QAT_FW_NUM_LONGWORDS_10 10 -#define ICP_QAT_FW_NUM_LONGWORDS_13 13 -#define ICP_QAT_FW_NULL_REQ_SERV_ID 1 - -enum icp_qat_fw_comn_resp_serv_id { - ICP_QAT_FW_COMN_RESP_SERV_NULL, - ICP_QAT_FW_COMN_RESP_SERV_CPM_FW, - ICP_QAT_FW_COMN_RESP_SERV_DELIMITER -}; - -enum icp_qat_fw_comn_request_id { - ICP_QAT_FW_COMN_REQ_NULL = 0, - ICP_QAT_FW_COMN_REQ_CPM_FW_PKE = 3, - ICP_QAT_FW_COMN_REQ_CPM_FW_LA = 4, - ICP_QAT_FW_COMN_REQ_CPM_FW_DMA = 7, - ICP_QAT_FW_COMN_REQ_CPM_FW_COMP = 9, - ICP_QAT_FW_COMN_REQ_DELIMITER -}; - -struct icp_qat_fw_comn_req_hdr_cd_pars { - union { - struct { - __u64 content_desc_addr; - __u16 content_desc_resrvd1; - __u8 content_desc_params_sz; - __u8 content_desc_hdr_resrvd2; - __u32 content_desc_resrvd3; - } s; - struct { - __u32 serv_specif_fields[4]; - } s1; - } u; -}; - -struct icp_qat_fw_comn_req_mid { - __u64 opaque_data; - __u64 src_data_addr; - __u64 dest_data_addr; - __u32 src_length; - __u32 dst_length; -}; - -struct icp_qat_fw_comn_req_cd_ctrl { - __u32 content_desc_ctrl_lw[ICP_QAT_FW_NUM_LONGWORDS_5]; -}; - -struct icp_qat_fw_comn_req_hdr { - __u8 resrvd1; - __u8 service_cmd_id; - __u8 service_type; - __u8 hdr_flags; - __u16 serv_specif_flags; - __u16 comn_req_flags; -}; - -struct icp_qat_fw_comn_req_rqpars { - __u32 serv_specif_rqpars_lw[ICP_QAT_FW_NUM_LONGWORDS_13]; -}; - -struct icp_qat_fw_comn_req { - struct icp_qat_fw_comn_req_hdr comn_hdr; - struct icp_qat_fw_comn_req_hdr_cd_pars cd_pars; - struct icp_qat_fw_comn_req_mid comn_mid; - struct icp_qat_fw_comn_req_rqpars serv_specif_rqpars; - struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; -}; - -struct icp_qat_fw_comn_error { - __u8 xlat_err_code; - __u8 cmp_err_code; -}; - -struct icp_qat_fw_comn_resp_hdr { - __u8 resrvd1; - __u8 service_id; - __u8 response_type; - __u8 hdr_flags; - struct icp_qat_fw_comn_error comn_error; - __u8 comn_status; - __u8 cmd_id; -}; - -struct icp_qat_fw_comn_resp { - struct icp_qat_fw_comn_resp_hdr comn_hdr; - __u64 opaque_data; - __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_4]; -}; - -#define ICP_QAT_FW_COMN_REQ_FLAG_SET 1 -#define ICP_QAT_FW_COMN_REQ_FLAG_CLR 0 -#define ICP_QAT_FW_COMN_VALID_FLAG_BITPOS 7 -#define ICP_QAT_FW_COMN_VALID_FLAG_MASK 0x1 -#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK 0x7F -#define ICP_QAT_FW_COMN_CNV_FLAG_BITPOS 6 -#define ICP_QAT_FW_COMN_CNV_FLAG_MASK 0x1 -#define ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS 5 -#define ICP_QAT_FW_COMN_CNVNR_FLAG_MASK 0x1 - -#define ICP_QAT_FW_COMN_OV_SRV_TYPE_GET(icp_qat_fw_comn_req_hdr_t) \ - icp_qat_fw_comn_req_hdr_t.service_type - -#define ICP_QAT_FW_COMN_OV_SRV_TYPE_SET(icp_qat_fw_comn_req_hdr_t, val) \ - icp_qat_fw_comn_req_hdr_t.service_type = val - -#define ICP_QAT_FW_COMN_OV_SRV_CMD_ID_GET(icp_qat_fw_comn_req_hdr_t) \ - icp_qat_fw_comn_req_hdr_t.service_cmd_id - -#define ICP_QAT_FW_COMN_OV_SRV_CMD_ID_SET(icp_qat_fw_comn_req_hdr_t, val) \ - icp_qat_fw_comn_req_hdr_t.service_cmd_id = val - -#define ICP_QAT_FW_COMN_HDR_VALID_FLAG_GET(hdr_t) \ - ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_t.hdr_flags) - -#define ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_GET(hdr_flags) \ - QAT_FIELD_GET(hdr_flags, \ - ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_CNVNR_FLAG_MASK) - -#define ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_SET(hdr_t, val) \ - QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ - ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_CNVNR_FLAG_MASK) - -#define ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(hdr_flags) \ - QAT_FIELD_GET(hdr_flags, \ - ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_CNV_FLAG_MASK) - -#define ICP_QAT_FW_COMN_HDR_CNV_FLAG_SET(hdr_t, val) \ - QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ - ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_CNV_FLAG_MASK) - -#define ICP_QAT_FW_COMN_HDR_VALID_FLAG_SET(hdr_t, val) \ - ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val) - -#define ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_flags) \ - QAT_FIELD_GET(hdr_flags, \ - ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_VALID_FLAG_MASK) - -#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_GET(hdr_flags) \ - (hdr_flags & ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK) - -#define ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val) \ - QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ - ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_VALID_FLAG_MASK) - -#define ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(valid) \ - (((valid) & ICP_QAT_FW_COMN_VALID_FLAG_MASK) << \ - ICP_QAT_FW_COMN_VALID_FLAG_BITPOS) - -#define QAT_COMN_PTR_TYPE_BITPOS 0 -#define QAT_COMN_PTR_TYPE_MASK 0x1 -#define QAT_COMN_CD_FLD_TYPE_BITPOS 1 -#define QAT_COMN_CD_FLD_TYPE_MASK 0x1 -#define QAT_COMN_PTR_TYPE_FLAT 0x0 -#define QAT_COMN_PTR_TYPE_SGL 0x1 -#define QAT_COMN_CD_FLD_TYPE_64BIT_ADR 0x0 -#define QAT_COMN_CD_FLD_TYPE_16BYTE_DATA 0x1 - -#define ICP_QAT_FW_COMN_FLAGS_BUILD(cdt, ptr) \ - ((((cdt) & QAT_COMN_CD_FLD_TYPE_MASK) << QAT_COMN_CD_FLD_TYPE_BITPOS) \ - | (((ptr) & QAT_COMN_PTR_TYPE_MASK) << QAT_COMN_PTR_TYPE_BITPOS)) - -#define ICP_QAT_FW_COMN_PTR_TYPE_GET(flags) \ - QAT_FIELD_GET(flags, QAT_COMN_PTR_TYPE_BITPOS, QAT_COMN_PTR_TYPE_MASK) - -#define ICP_QAT_FW_COMN_CD_FLD_TYPE_GET(flags) \ - QAT_FIELD_GET(flags, QAT_COMN_CD_FLD_TYPE_BITPOS, \ - QAT_COMN_CD_FLD_TYPE_MASK) - -#define ICP_QAT_FW_COMN_PTR_TYPE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_COMN_PTR_TYPE_BITPOS, \ - QAT_COMN_PTR_TYPE_MASK) - -#define ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_COMN_CD_FLD_TYPE_BITPOS, \ - QAT_COMN_CD_FLD_TYPE_MASK) - -#define ICP_QAT_FW_COMN_NEXT_ID_BITPOS 4 -#define ICP_QAT_FW_COMN_NEXT_ID_MASK 0xF0 -#define ICP_QAT_FW_COMN_CURR_ID_BITPOS 0 -#define ICP_QAT_FW_COMN_CURR_ID_MASK 0x0F - -#define ICP_QAT_FW_COMN_NEXT_ID_GET(cd_ctrl_hdr_t) \ - ((((cd_ctrl_hdr_t)->next_curr_id) & ICP_QAT_FW_COMN_NEXT_ID_MASK) \ - >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) - -#define ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ - { ((cd_ctrl_hdr_t)->next_curr_id) = ((((cd_ctrl_hdr_t)->next_curr_id) \ - & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ - ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ - & ICP_QAT_FW_COMN_NEXT_ID_MASK)); } - -#define ICP_QAT_FW_COMN_CURR_ID_GET(cd_ctrl_hdr_t) \ - (((cd_ctrl_hdr_t)->next_curr_id) & ICP_QAT_FW_COMN_CURR_ID_MASK) - -#define ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl_hdr_t, val) \ - { ((cd_ctrl_hdr_t)->next_curr_id) = ((((cd_ctrl_hdr_t)->next_curr_id) \ - & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ - ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)); } - -#define QAT_COMN_RESP_CRYPTO_STATUS_BITPOS 7 -#define QAT_COMN_RESP_CRYPTO_STATUS_MASK 0x1 -#define QAT_COMN_RESP_PKE_STATUS_BITPOS 6 -#define QAT_COMN_RESP_PKE_STATUS_MASK 0x1 -#define QAT_COMN_RESP_CMP_STATUS_BITPOS 5 -#define QAT_COMN_RESP_CMP_STATUS_MASK 0x1 -#define QAT_COMN_RESP_XLAT_STATUS_BITPOS 4 -#define QAT_COMN_RESP_XLAT_STATUS_MASK 0x1 -#define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS 3 -#define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK 0x1 - -#define ICP_QAT_FW_COMN_RESP_STATUS_BUILD(crypto, comp, xlat, eolb) \ - ((((crypto) & QAT_COMN_RESP_CRYPTO_STATUS_MASK) << \ - QAT_COMN_RESP_CRYPTO_STATUS_BITPOS) | \ - (((comp) & QAT_COMN_RESP_CMP_STATUS_MASK) << \ - QAT_COMN_RESP_CMP_STATUS_BITPOS) | \ - (((xlat) & QAT_COMN_RESP_XLAT_STATUS_MASK) << \ - QAT_COMN_RESP_XLAT_STATUS_BITPOS) | \ - (((eolb) & QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) << \ - QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS)) - -#define ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(status) \ - QAT_FIELD_GET(status, QAT_COMN_RESP_CRYPTO_STATUS_BITPOS, \ - QAT_COMN_RESP_CRYPTO_STATUS_MASK) - -#define ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(status) \ - QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_STATUS_BITPOS, \ - QAT_COMN_RESP_CMP_STATUS_MASK) - -#define ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(status) \ - QAT_FIELD_GET(status, QAT_COMN_RESP_XLAT_STATUS_BITPOS, \ - QAT_COMN_RESP_XLAT_STATUS_MASK) - -#define ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(status) \ - QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS, \ - QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) - -#define ICP_QAT_FW_COMN_STATUS_FLAG_OK 0 -#define ICP_QAT_FW_COMN_STATUS_FLAG_ERROR 1 -#define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_CLR 0 -#define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET 1 -#define ERR_CODE_NO_ERROR 0 -#define ERR_CODE_INVALID_BLOCK_TYPE -1 -#define ERR_CODE_NO_MATCH_ONES_COMP -2 -#define ERR_CODE_TOO_MANY_LEN_OR_DIS -3 -#define ERR_CODE_INCOMPLETE_LEN -4 -#define ERR_CODE_RPT_LEN_NO_FIRST_LEN -5 -#define ERR_CODE_RPT_GT_SPEC_LEN -6 -#define ERR_CODE_INV_LIT_LEN_CODE_LEN -7 -#define ERR_CODE_INV_DIS_CODE_LEN -8 -#define ERR_CODE_INV_LIT_LEN_DIS_IN_BLK -9 -#define ERR_CODE_DIS_TOO_FAR_BACK -10 -#define ERR_CODE_OVERFLOW_ERROR -11 -#define ERR_CODE_SOFT_ERROR -12 -#define ERR_CODE_FATAL_ERROR -13 -#define ERR_CODE_SSM_ERROR -14 -#define ERR_CODE_ENDPOINT_ERROR -15 - -enum icp_qat_fw_slice { - ICP_QAT_FW_SLICE_NULL = 0, - ICP_QAT_FW_SLICE_CIPHER = 1, - ICP_QAT_FW_SLICE_AUTH = 2, - ICP_QAT_FW_SLICE_DRAM_RD = 3, - ICP_QAT_FW_SLICE_DRAM_WR = 4, - ICP_QAT_FW_SLICE_COMP = 5, - ICP_QAT_FW_SLICE_XLAT = 6, - ICP_QAT_FW_SLICE_DELIMITER -}; -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_comp.h b/drivers/crypto/qat/qat_common/icp_qat_fw_comp.h deleted file mode 100644 index a03d43fef2b3..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_comp.h +++ /dev/null @@ -1,404 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef _ICP_QAT_FW_COMP_H_ -#define _ICP_QAT_FW_COMP_H_ -#include "icp_qat_fw.h" - -enum icp_qat_fw_comp_cmd_id { - ICP_QAT_FW_COMP_CMD_STATIC = 0, - ICP_QAT_FW_COMP_CMD_DYNAMIC = 1, - ICP_QAT_FW_COMP_CMD_DECOMPRESS = 2, - ICP_QAT_FW_COMP_CMD_DELIMITER -}; - -enum icp_qat_fw_comp_20_cmd_id { - ICP_QAT_FW_COMP_20_CMD_LZ4_COMPRESS = 3, - ICP_QAT_FW_COMP_20_CMD_LZ4_DECOMPRESS = 4, - ICP_QAT_FW_COMP_20_CMD_LZ4S_COMPRESS = 5, - ICP_QAT_FW_COMP_20_CMD_LZ4S_DECOMPRESS = 6, - ICP_QAT_FW_COMP_20_CMD_XP10_COMPRESS = 7, - ICP_QAT_FW_COMP_20_CMD_XP10_DECOMPRESS = 8, - ICP_QAT_FW_COMP_20_CMD_RESERVED_9 = 9, - ICP_QAT_FW_COMP_23_CMD_ZSTD_COMPRESS = 10, - ICP_QAT_FW_COMP_23_CMD_ZSTD_DECOMPRESS = 11, - ICP_QAT_FW_COMP_20_CMD_DELIMITER -}; - -#define ICP_QAT_FW_COMP_STATELESS_SESSION 0 -#define ICP_QAT_FW_COMP_STATEFUL_SESSION 1 -#define ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST 0 -#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST 1 -#define ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST 0 -#define ICP_QAT_FW_COMP_ENH_AUTO_SELECT_BEST 1 -#define ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST 0 -#define ICP_QAT_FW_COMP_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST 1 -#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_USED_AS_INTMD_BUF 1 -#define ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF 0 -#define ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS 2 -#define ICP_QAT_FW_COMP_SESSION_TYPE_MASK 0x1 -#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS 3 -#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK 0x1 -#define ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS 4 -#define ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK 0x1 -#define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS 5 -#define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK 0x1 -#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS 7 -#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK 0x1 - -#define ICP_QAT_FW_COMP_FLAGS_BUILD(sesstype, autoselect, enhanced_asb, \ - ret_uncomp, secure_ram) \ - ((((sesstype) & ICP_QAT_FW_COMP_SESSION_TYPE_MASK) << \ - ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS) | \ - (((autoselect) & ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK) << \ - ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS) | \ - (((enhanced_asb) & ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK) << \ - ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS) | \ - (((ret_uncomp) & ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK) << \ - ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS) | \ - (((secure_ram) & ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) << \ - ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS)) - -#define ICP_QAT_FW_COMP_SESSION_TYPE_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS, \ - ICP_QAT_FW_COMP_SESSION_TYPE_MASK) - -#define ICP_QAT_FW_COMP_SESSION_TYPE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS, \ - ICP_QAT_FW_COMP_SESSION_TYPE_MASK) - -#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS, \ - ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK) - -#define ICP_QAT_FW_COMP_EN_ASB_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS, \ - ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK) - -#define ICP_QAT_FW_COMP_RET_UNCOMP_GET(flags) \ - QAT_FIELD_GET(flags, \ - ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS, \ - ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK) - -#define ICP_QAT_FW_COMP_SECURE_RAM_USE_GET(flags) \ - QAT_FIELD_GET(flags, \ - ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS, \ - ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) - -struct icp_qat_fw_comp_req_hdr_cd_pars { - union { - struct { - __u64 content_desc_addr; - __u16 content_desc_resrvd1; - __u8 content_desc_params_sz; - __u8 content_desc_hdr_resrvd2; - __u32 content_desc_resrvd3; - } s; - struct { - __u32 comp_slice_cfg_word[ICP_QAT_FW_NUM_LONGWORDS_2]; - __u32 content_desc_resrvd4; - } sl; - } u; -}; - -struct icp_qat_fw_comp_req_params { - __u32 comp_len; - __u32 out_buffer_sz; - union { - struct { - __u32 initial_crc32; - __u32 initial_adler; - } legacy; - __u64 crc_data_addr; - } crc; - __u32 req_par_flags; - __u32 rsrvd; -}; - -#define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop, eop, bfinal, cnv, cnvnr, \ - cnvdfx, crc, xxhash_acc, \ - cnv_error_type, append_crc, \ - drop_data) \ - ((((sop) & ICP_QAT_FW_COMP_SOP_MASK) << \ - ICP_QAT_FW_COMP_SOP_BITPOS) | \ - (((eop) & ICP_QAT_FW_COMP_EOP_MASK) << \ - ICP_QAT_FW_COMP_EOP_BITPOS) | \ - (((bfinal) & ICP_QAT_FW_COMP_BFINAL_MASK) \ - << ICP_QAT_FW_COMP_BFINAL_BITPOS) | \ - (((cnv) & ICP_QAT_FW_COMP_CNV_MASK) << \ - ICP_QAT_FW_COMP_CNV_BITPOS) | \ - (((cnvnr) & ICP_QAT_FW_COMP_CNVNR_MASK) \ - << ICP_QAT_FW_COMP_CNVNR_BITPOS) | \ - (((cnvdfx) & ICP_QAT_FW_COMP_CNV_DFX_MASK) \ - << ICP_QAT_FW_COMP_CNV_DFX_BITPOS) | \ - (((crc) & ICP_QAT_FW_COMP_CRC_MODE_MASK) \ - << ICP_QAT_FW_COMP_CRC_MODE_BITPOS) | \ - (((xxhash_acc) & ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) \ - << ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS) | \ - (((cnv_error_type) & ICP_QAT_FW_COMP_CNV_ERROR_MASK) \ - << ICP_QAT_FW_COMP_CNV_ERROR_BITPOS) | \ - (((append_crc) & ICP_QAT_FW_COMP_APPEND_CRC_MASK) \ - << ICP_QAT_FW_COMP_APPEND_CRC_BITPOS) | \ - (((drop_data) & ICP_QAT_FW_COMP_DROP_DATA_MASK) \ - << ICP_QAT_FW_COMP_DROP_DATA_BITPOS)) - -#define ICP_QAT_FW_COMP_NOT_SOP 0 -#define ICP_QAT_FW_COMP_SOP 1 -#define ICP_QAT_FW_COMP_NOT_EOP 0 -#define ICP_QAT_FW_COMP_EOP 1 -#define ICP_QAT_FW_COMP_NOT_BFINAL 0 -#define ICP_QAT_FW_COMP_BFINAL 1 -#define ICP_QAT_FW_COMP_NO_CNV 0 -#define ICP_QAT_FW_COMP_CNV 1 -#define ICP_QAT_FW_COMP_NO_CNV_RECOVERY 0 -#define ICP_QAT_FW_COMP_CNV_RECOVERY 1 -#define ICP_QAT_FW_COMP_NO_CNV_DFX 0 -#define ICP_QAT_FW_COMP_CNV_DFX 1 -#define ICP_QAT_FW_COMP_CRC_MODE_LEGACY 0 -#define ICP_QAT_FW_COMP_CRC_MODE_E2E 1 -#define ICP_QAT_FW_COMP_NO_XXHASH_ACC 0 -#define ICP_QAT_FW_COMP_XXHASH_ACC 1 -#define ICP_QAT_FW_COMP_APPEND_CRC 1 -#define ICP_QAT_FW_COMP_NO_APPEND_CRC 0 -#define ICP_QAT_FW_COMP_DROP_DATA 1 -#define ICP_QAT_FW_COMP_NO_DROP_DATA 0 -#define ICP_QAT_FW_COMP_SOP_BITPOS 0 -#define ICP_QAT_FW_COMP_SOP_MASK 0x1 -#define ICP_QAT_FW_COMP_EOP_BITPOS 1 -#define ICP_QAT_FW_COMP_EOP_MASK 0x1 -#define ICP_QAT_FW_COMP_BFINAL_BITPOS 6 -#define ICP_QAT_FW_COMP_BFINAL_MASK 0x1 -#define ICP_QAT_FW_COMP_CNV_BITPOS 16 -#define ICP_QAT_FW_COMP_CNV_MASK 0x1 -#define ICP_QAT_FW_COMP_CNVNR_BITPOS 17 -#define ICP_QAT_FW_COMP_CNVNR_MASK 0x1 -#define ICP_QAT_FW_COMP_CNV_DFX_BITPOS 18 -#define ICP_QAT_FW_COMP_CNV_DFX_MASK 0x1 -#define ICP_QAT_FW_COMP_CRC_MODE_BITPOS 19 -#define ICP_QAT_FW_COMP_CRC_MODE_MASK 0x1 -#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS 20 -#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK 0x1 -#define ICP_QAT_FW_COMP_CNV_ERROR_BITPOS 21 -#define ICP_QAT_FW_COMP_CNV_ERROR_MASK 0b111 -#define ICP_QAT_FW_COMP_CNV_ERROR_NONE 0b000 -#define ICP_QAT_FW_COMP_CNV_ERROR_CHECKSUM 0b001 -#define ICP_QAT_FW_COMP_CNV_ERROR_DCPR_OBC_DIFF 0b010 -#define ICP_QAT_FW_COMP_CNV_ERROR_DCPR 0b011 -#define ICP_QAT_FW_COMP_CNV_ERROR_XLT 0b100 -#define ICP_QAT_FW_COMP_CNV_ERROR_DCPR_IBC_DIFF 0b101 -#define ICP_QAT_FW_COMP_APPEND_CRC_BITPOS 24 -#define ICP_QAT_FW_COMP_APPEND_CRC_MASK 0x1 -#define ICP_QAT_FW_COMP_DROP_DATA_BITPOS 25 -#define ICP_QAT_FW_COMP_DROP_DATA_MASK 0x1 - -#define ICP_QAT_FW_COMP_SOP_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_SOP_BITPOS, \ - ICP_QAT_FW_COMP_SOP_MASK) - -#define ICP_QAT_FW_COMP_SOP_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_SOP_BITPOS, \ - ICP_QAT_FW_COMP_SOP_MASK) - -#define ICP_QAT_FW_COMP_EOP_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_EOP_BITPOS, \ - ICP_QAT_FW_COMP_EOP_MASK) - -#define ICP_QAT_FW_COMP_EOP_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_EOP_BITPOS, \ - ICP_QAT_FW_COMP_EOP_MASK) - -#define ICP_QAT_FW_COMP_BFINAL_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_BFINAL_BITPOS, \ - ICP_QAT_FW_COMP_BFINAL_MASK) - -#define ICP_QAT_FW_COMP_BFINAL_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_BFINAL_BITPOS, \ - ICP_QAT_FW_COMP_BFINAL_MASK) - -#define ICP_QAT_FW_COMP_CNV_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNV_BITPOS, \ - ICP_QAT_FW_COMP_CNV_MASK) - -#define ICP_QAT_FW_COMP_CNVNR_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNVNR_BITPOS, \ - ICP_QAT_FW_COMP_CNVNR_MASK) - -#define ICP_QAT_FW_COMP_CNV_DFX_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNV_DFX_BITPOS, \ - ICP_QAT_FW_COMP_CNV_DFX_MASK) - -#define ICP_QAT_FW_COMP_CNV_DFX_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_CNV_DFX_BITPOS, \ - ICP_QAT_FW_COMP_CNV_DFX_MASK) - -#define ICP_QAT_FW_COMP_CRC_MODE_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CRC_MODE_BITPOS, \ - ICP_QAT_FW_COMP_CRC_MODE_MASK) - -#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \ - ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) - -#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \ - ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) - -#define ICP_QAT_FW_COMP_CNV_ERROR_TYPE_GET(flags) \ - QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNV_ERROR_BITPOS, \ - ICP_QAT_FW_COMP_CNV_ERROR_MASK) - -#define ICP_QAT_FW_COMP_CNV_ERROR_TYPE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_CNV_ERROR_BITPOS, \ - ICP_QAT_FW_COMP_CNV_ERROR_MASK) - -struct icp_qat_fw_xlt_req_params { - __u64 inter_buff_ptr; -}; - -struct icp_qat_fw_comp_cd_hdr { - __u16 ram_bank_flags; - __u8 comp_cfg_offset; - __u8 next_curr_id; - __u32 resrvd; - __u64 comp_state_addr; - __u64 ram_banks_addr; -}; - -#define COMP_CPR_INITIAL_CRC 0 -#define COMP_CPR_INITIAL_ADLER 1 - -struct icp_qat_fw_xlt_cd_hdr { - __u16 resrvd1; - __u8 resrvd2; - __u8 next_curr_id; - __u32 resrvd3; -}; - -struct icp_qat_fw_comp_req { - struct icp_qat_fw_comn_req_hdr comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars cd_pars; - struct icp_qat_fw_comn_req_mid comn_mid; - struct icp_qat_fw_comp_req_params comp_pars; - union { - struct icp_qat_fw_xlt_req_params xlt_pars; - __u32 resrvd1[ICP_QAT_FW_NUM_LONGWORDS_2]; - } u1; - __u32 resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; - struct icp_qat_fw_comp_cd_hdr comp_cd_ctrl; - union { - struct icp_qat_fw_xlt_cd_hdr xlt_cd_ctrl; - __u32 resrvd3[ICP_QAT_FW_NUM_LONGWORDS_2]; - } u2; -}; - -struct icp_qat_fw_resp_comp_pars { - __u32 input_byte_counter; - __u32 output_byte_counter; - union { - struct { - __u32 curr_crc32; - __u32 curr_adler_32; - } legacy; - __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_2]; - } crc; -}; - -struct icp_qat_fw_comp_state { - __u32 rd8_counter; - __u32 status_flags; - __u32 in_counter; - __u32 out_counter; - __u64 intermediate_state; - __u32 lobc; - __u32 replaybc; - __u64 pcrc64_poly; - __u32 crc32; - __u32 adler_xxhash32; - __u64 pcrc64_xorout; - __u32 out_buf_size; - __u32 in_buf_size; - __u64 in_pcrc64; - __u64 out_pcrc64; - __u32 lobs; - __u32 libc; - __u64 reserved; - __u32 xxhash_state[4]; - __u32 cleartext[4]; -}; - -struct icp_qat_fw_comp_resp { - struct icp_qat_fw_comn_resp_hdr comn_resp; - __u64 opaque_data; - struct icp_qat_fw_resp_comp_pars comp_resp_pars; -}; - -#define QAT_FW_COMP_BANK_FLAG_MASK 0x1 -#define QAT_FW_COMP_BANK_I_BITPOS 8 -#define QAT_FW_COMP_BANK_H_BITPOS 7 -#define QAT_FW_COMP_BANK_G_BITPOS 6 -#define QAT_FW_COMP_BANK_F_BITPOS 5 -#define QAT_FW_COMP_BANK_E_BITPOS 4 -#define QAT_FW_COMP_BANK_D_BITPOS 3 -#define QAT_FW_COMP_BANK_C_BITPOS 2 -#define QAT_FW_COMP_BANK_B_BITPOS 1 -#define QAT_FW_COMP_BANK_A_BITPOS 0 - -enum icp_qat_fw_comp_bank_enabled { - ICP_QAT_FW_COMP_BANK_DISABLED = 0, - ICP_QAT_FW_COMP_BANK_ENABLED = 1, - ICP_QAT_FW_COMP_BANK_DELIMITER = 2 -}; - -#define ICP_QAT_FW_COMP_RAM_FLAGS_BUILD(bank_i_enable, bank_h_enable, \ - bank_g_enable, bank_f_enable, \ - bank_e_enable, bank_d_enable, \ - bank_c_enable, bank_b_enable, \ - bank_a_enable) \ - ((((bank_i_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_I_BITPOS) | \ - (((bank_h_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_H_BITPOS) | \ - (((bank_g_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_G_BITPOS) | \ - (((bank_f_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_F_BITPOS) | \ - (((bank_e_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_E_BITPOS) | \ - (((bank_d_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_D_BITPOS) | \ - (((bank_c_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_C_BITPOS) | \ - (((bank_b_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_B_BITPOS) | \ - (((bank_a_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ - QAT_FW_COMP_BANK_A_BITPOS)) - -struct icp_qat_fw_comp_crc_data_struct { - __u32 crc32; - union { - __u32 adler; - __u32 xxhash; - } adler_xxhash_u; - __u32 cpr_in_crc_lo; - __u32 cpr_in_crc_hi; - __u32 cpr_out_crc_lo; - __u32 cpr_out_crc_hi; - __u32 xlt_in_crc_lo; - __u32 xlt_in_crc_hi; - __u32 xlt_out_crc_lo; - __u32 xlt_out_crc_hi; - __u32 prog_crc_poly_lo; - __u32 prog_crc_poly_hi; - __u32 xor_out_lo; - __u32 xor_out_hi; - __u32 append_crc_lo; - __u32 append_crc_hi; -}; - -struct xxhash_acc_state_buff { - __u32 in_counter; - __u32 out_counter; - __u32 xxhash_state[4]; - __u32 clear_txt[4]; -}; - -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h deleted file mode 100644 index 56cb827f93ea..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef _ICP_QAT_FW_INIT_ADMIN_H_ -#define _ICP_QAT_FW_INIT_ADMIN_H_ - -#include "icp_qat_fw.h" - -enum icp_qat_fw_init_admin_cmd_id { - ICP_QAT_FW_INIT_AE = 0, - ICP_QAT_FW_TRNG_ENABLE = 1, - ICP_QAT_FW_TRNG_DISABLE = 2, - ICP_QAT_FW_CONSTANTS_CFG = 3, - ICP_QAT_FW_STATUS_GET = 4, - ICP_QAT_FW_COUNTERS_GET = 5, - ICP_QAT_FW_LOOPBACK = 6, - ICP_QAT_FW_HEARTBEAT_SYNC = 7, - ICP_QAT_FW_HEARTBEAT_GET = 8, - ICP_QAT_FW_COMP_CAPABILITY_GET = 9, - ICP_QAT_FW_PM_STATE_CONFIG = 128, -}; - -enum icp_qat_fw_init_admin_resp_status { - ICP_QAT_FW_INIT_RESP_STATUS_SUCCESS = 0, - ICP_QAT_FW_INIT_RESP_STATUS_FAIL -}; - -struct icp_qat_fw_init_admin_req { - __u16 init_cfg_sz; - __u8 resrvd1; - __u8 cmd_id; - __u32 resrvd2; - __u64 opaque_data; - __u64 init_cfg_ptr; - - union { - struct { - __u16 ibuf_size_in_kb; - __u16 resrvd3; - }; - __u32 idle_filter; - }; - - __u32 resrvd4; -} __packed; - -struct icp_qat_fw_init_admin_resp { - __u8 flags; - __u8 resrvd1; - __u8 status; - __u8 cmd_id; - union { - __u32 resrvd2; - struct { - __u16 version_minor_num; - __u16 version_major_num; - }; - __u32 extended_features; - }; - __u64 opaque_data; - union { - __u32 resrvd3[ICP_QAT_FW_NUM_LONGWORDS_4]; - struct { - __u32 version_patch_num; - __u8 context_id; - __u8 ae_id; - __u16 resrvd4; - __u64 resrvd5; - }; - struct { - __u64 req_rec_count; - __u64 resp_sent_count; - }; - struct { - __u16 compression_algos; - __u16 checksum_algos; - __u32 deflate_capabilities; - __u32 resrvd6; - __u32 lzs_capabilities; - }; - struct { - __u32 cipher_algos; - __u32 hash_algos; - __u16 keygen_algos; - __u16 other; - __u16 public_key_algos; - __u16 prime_algos; - }; - struct { - __u64 timestamp; - __u64 resrvd7; - }; - struct { - __u32 successful_count; - __u32 unsuccessful_count; - __u64 resrvd8; - }; - }; -} __packed; - -#define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 -#define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0 -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK 0x1 -#define ICP_QAT_FW_COMN_STATUS_RESRVD_FLD_MASK 0xFE -#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_GET(hdr_t) \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(hdr_t.flags) - -#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_SET(hdr_t, val) \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_SET(hdr_t, val) - -#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS, \ - ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK) -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h deleted file mode 100644 index 28fa17f14be4..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h +++ /dev/null @@ -1,367 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef _ICP_QAT_FW_LA_H_ -#define _ICP_QAT_FW_LA_H_ -#include "icp_qat_fw.h" - -enum icp_qat_fw_la_cmd_id { - ICP_QAT_FW_LA_CMD_CIPHER = 0, - ICP_QAT_FW_LA_CMD_AUTH = 1, - ICP_QAT_FW_LA_CMD_CIPHER_HASH = 2, - ICP_QAT_FW_LA_CMD_HASH_CIPHER = 3, - ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM = 4, - ICP_QAT_FW_LA_CMD_TRNG_TEST = 5, - ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE = 6, - ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE = 7, - ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE = 8, - ICP_QAT_FW_LA_CMD_MGF1 = 9, - ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP = 10, - ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP = 11, - ICP_QAT_FW_LA_CMD_DELIMITER = 12 -}; - -#define ICP_QAT_FW_LA_ICV_VER_STATUS_PASS ICP_QAT_FW_COMN_STATUS_FLAG_OK -#define ICP_QAT_FW_LA_ICV_VER_STATUS_FAIL ICP_QAT_FW_COMN_STATUS_FLAG_ERROR -#define ICP_QAT_FW_LA_TRNG_STATUS_PASS ICP_QAT_FW_COMN_STATUS_FLAG_OK -#define ICP_QAT_FW_LA_TRNG_STATUS_FAIL ICP_QAT_FW_COMN_STATUS_FLAG_ERROR - -struct icp_qat_fw_la_bulk_req { - struct icp_qat_fw_comn_req_hdr comn_hdr; - struct icp_qat_fw_comn_req_hdr_cd_pars cd_pars; - struct icp_qat_fw_comn_req_mid comn_mid; - struct icp_qat_fw_comn_req_rqpars serv_specif_rqpars; - struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; -}; - -#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1 -#define QAT_LA_SLICE_TYPE_BITPOS 14 -#define QAT_LA_SLICE_TYPE_MASK 0x3 -#define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 -#define ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS 0 -#define QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS 12 -#define ICP_QAT_FW_LA_ZUC_3G_PROTO 1 -#define QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK 0x1 -#define QAT_LA_GCM_IV_LEN_FLAG_BITPOS 11 -#define QAT_LA_GCM_IV_LEN_FLAG_MASK 0x1 -#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER 1 -#define ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER 0 -#define QAT_LA_DIGEST_IN_BUFFER_BITPOS 10 -#define QAT_LA_DIGEST_IN_BUFFER_MASK 0x1 -#define ICP_QAT_FW_LA_SNOW_3G_PROTO 4 -#define ICP_QAT_FW_LA_GCM_PROTO 2 -#define ICP_QAT_FW_LA_CCM_PROTO 1 -#define ICP_QAT_FW_LA_NO_PROTO 0 -#define QAT_LA_PROTO_BITPOS 7 -#define QAT_LA_PROTO_MASK 0x7 -#define ICP_QAT_FW_LA_CMP_AUTH_RES 1 -#define ICP_QAT_FW_LA_NO_CMP_AUTH_RES 0 -#define QAT_LA_CMP_AUTH_RES_BITPOS 6 -#define QAT_LA_CMP_AUTH_RES_MASK 0x1 -#define ICP_QAT_FW_LA_RET_AUTH_RES 1 -#define ICP_QAT_FW_LA_NO_RET_AUTH_RES 0 -#define QAT_LA_RET_AUTH_RES_BITPOS 5 -#define QAT_LA_RET_AUTH_RES_MASK 0x1 -#define ICP_QAT_FW_LA_UPDATE_STATE 1 -#define ICP_QAT_FW_LA_NO_UPDATE_STATE 0 -#define QAT_LA_UPDATE_STATE_BITPOS 4 -#define QAT_LA_UPDATE_STATE_MASK 0x1 -#define ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP 0 -#define ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP 1 -#define QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS 3 -#define QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK 0x1 -#define ICP_QAT_FW_CIPH_IV_64BIT_PTR 0 -#define ICP_QAT_FW_CIPH_IV_16BYTE_DATA 1 -#define QAT_LA_CIPH_IV_FLD_BITPOS 2 -#define QAT_LA_CIPH_IV_FLD_MASK 0x1 -#define ICP_QAT_FW_LA_PARTIAL_NONE 0 -#define ICP_QAT_FW_LA_PARTIAL_START 1 -#define ICP_QAT_FW_LA_PARTIAL_MID 3 -#define ICP_QAT_FW_LA_PARTIAL_END 2 -#define QAT_LA_PARTIAL_BITPOS 0 -#define QAT_LA_PARTIAL_MASK 0x3 -#define ICP_QAT_FW_LA_FLAGS_BUILD(zuc_proto, gcm_iv_len, auth_rslt, proto, \ - cmp_auth, ret_auth, update_state, \ - ciph_iv, ciphcfg, partial) \ - (((zuc_proto & QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) << \ - QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS) | \ - ((gcm_iv_len & QAT_LA_GCM_IV_LEN_FLAG_MASK) << \ - QAT_LA_GCM_IV_LEN_FLAG_BITPOS) | \ - ((auth_rslt & QAT_LA_DIGEST_IN_BUFFER_MASK) << \ - QAT_LA_DIGEST_IN_BUFFER_BITPOS) | \ - ((proto & QAT_LA_PROTO_MASK) << \ - QAT_LA_PROTO_BITPOS) | \ - ((cmp_auth & QAT_LA_CMP_AUTH_RES_MASK) << \ - QAT_LA_CMP_AUTH_RES_BITPOS) | \ - ((ret_auth & QAT_LA_RET_AUTH_RES_MASK) << \ - QAT_LA_RET_AUTH_RES_BITPOS) | \ - ((update_state & QAT_LA_UPDATE_STATE_MASK) << \ - QAT_LA_UPDATE_STATE_BITPOS) | \ - ((ciph_iv & QAT_LA_CIPH_IV_FLD_MASK) << \ - QAT_LA_CIPH_IV_FLD_BITPOS) | \ - ((ciphcfg & QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) << \ - QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS) | \ - ((partial & QAT_LA_PARTIAL_MASK) << \ - QAT_LA_PARTIAL_BITPOS)) - -#define ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_CIPH_IV_FLD_BITPOS, \ - QAT_LA_CIPH_IV_FLD_MASK) - -#define ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ - QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) - -#define ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ - QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) - -#define ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_GCM_IV_LEN_FLAG_BITPOS, \ - QAT_LA_GCM_IV_LEN_FLAG_MASK) - -#define ICP_QAT_FW_LA_PROTO_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_PROTO_BITPOS, QAT_LA_PROTO_MASK) - -#define ICP_QAT_FW_LA_CMP_AUTH_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_CMP_AUTH_RES_BITPOS, \ - QAT_LA_CMP_AUTH_RES_MASK) - -#define ICP_QAT_FW_LA_RET_AUTH_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_RET_AUTH_RES_BITPOS, \ - QAT_LA_RET_AUTH_RES_MASK) - -#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_DIGEST_IN_BUFFER_BITPOS, \ - QAT_LA_DIGEST_IN_BUFFER_MASK) - -#define ICP_QAT_FW_LA_UPDATE_STATE_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_UPDATE_STATE_BITPOS, \ - QAT_LA_UPDATE_STATE_MASK) - -#define ICP_QAT_FW_LA_PARTIAL_GET(flags) \ - QAT_FIELD_GET(flags, QAT_LA_PARTIAL_BITPOS, \ - QAT_LA_PARTIAL_MASK) - -#define ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_CIPH_IV_FLD_BITPOS, \ - QAT_LA_CIPH_IV_FLD_MASK) - -#define ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ - QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) - -#define ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ - QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) - -#define ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_GCM_IV_LEN_FLAG_BITPOS, \ - QAT_LA_GCM_IV_LEN_FLAG_MASK) - -#define ICP_QAT_FW_LA_PROTO_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_PROTO_BITPOS, \ - QAT_LA_PROTO_MASK) - -#define ICP_QAT_FW_LA_CMP_AUTH_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_CMP_AUTH_RES_BITPOS, \ - QAT_LA_CMP_AUTH_RES_MASK) - -#define ICP_QAT_FW_LA_RET_AUTH_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_RET_AUTH_RES_BITPOS, \ - QAT_LA_RET_AUTH_RES_MASK) - -#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_DIGEST_IN_BUFFER_BITPOS, \ - QAT_LA_DIGEST_IN_BUFFER_MASK) - -#define ICP_QAT_FW_LA_UPDATE_STATE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_UPDATE_STATE_BITPOS, \ - QAT_LA_UPDATE_STATE_MASK) - -#define ICP_QAT_FW_LA_PARTIAL_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_PARTIAL_BITPOS, \ - QAT_LA_PARTIAL_MASK) - -#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \ - QAT_FIELD_SET(flags, val, QAT_LA_SLICE_TYPE_BITPOS, \ - QAT_LA_SLICE_TYPE_MASK) - -struct icp_qat_fw_cipher_req_hdr_cd_pars { - union { - struct { - __u64 content_desc_addr; - __u16 content_desc_resrvd1; - __u8 content_desc_params_sz; - __u8 content_desc_hdr_resrvd2; - __u32 content_desc_resrvd3; - } s; - struct { - __u32 cipher_key_array[ICP_QAT_FW_NUM_LONGWORDS_4]; - } s1; - } u; -}; - -struct icp_qat_fw_cipher_auth_req_hdr_cd_pars { - union { - struct { - __u64 content_desc_addr; - __u16 content_desc_resrvd1; - __u8 content_desc_params_sz; - __u8 content_desc_hdr_resrvd2; - __u32 content_desc_resrvd3; - } s; - struct { - __u32 cipher_key_array[ICP_QAT_FW_NUM_LONGWORDS_4]; - } sl; - } u; -}; - -struct icp_qat_fw_cipher_cd_ctrl_hdr { - __u8 cipher_state_sz; - __u8 cipher_key_sz; - __u8 cipher_cfg_offset; - __u8 next_curr_id; - __u8 cipher_padding_sz; - __u8 resrvd1; - __u16 resrvd2; - __u32 resrvd3[ICP_QAT_FW_NUM_LONGWORDS_3]; -}; - -struct icp_qat_fw_auth_cd_ctrl_hdr { - __u32 resrvd1; - __u8 resrvd2; - __u8 hash_flags; - __u8 hash_cfg_offset; - __u8 next_curr_id; - __u8 resrvd3; - __u8 outer_prefix_sz; - __u8 final_sz; - __u8 inner_res_sz; - __u8 resrvd4; - __u8 inner_state1_sz; - __u8 inner_state2_offset; - __u8 inner_state2_sz; - __u8 outer_config_offset; - __u8 outer_state1_sz; - __u8 outer_res_sz; - __u8 outer_prefix_offset; -}; - -struct icp_qat_fw_cipher_auth_cd_ctrl_hdr { - __u8 cipher_state_sz; - __u8 cipher_key_sz; - __u8 cipher_cfg_offset; - __u8 next_curr_id_cipher; - __u8 cipher_padding_sz; - __u8 hash_flags; - __u8 hash_cfg_offset; - __u8 next_curr_id_auth; - __u8 resrvd1; - __u8 outer_prefix_sz; - __u8 final_sz; - __u8 inner_res_sz; - __u8 resrvd2; - __u8 inner_state1_sz; - __u8 inner_state2_offset; - __u8 inner_state2_sz; - __u8 outer_config_offset; - __u8 outer_state1_sz; - __u8 outer_res_sz; - __u8 outer_prefix_offset; -}; - -#define ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED 1 -#define ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED 0 -#define ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX 240 -#define ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET \ - (sizeof(struct icp_qat_fw_la_cipher_req_params_t)) -#define ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET (0) - -struct icp_qat_fw_la_cipher_req_params { - __u32 cipher_offset; - __u32 cipher_length; - union { - __u32 cipher_IV_array[ICP_QAT_FW_NUM_LONGWORDS_4]; - struct { - __u64 cipher_IV_ptr; - __u64 resrvd1; - } s; - } u; -}; - -struct icp_qat_fw_la_auth_req_params { - __u32 auth_off; - __u32 auth_len; - union { - __u64 auth_partial_st_prefix; - __u64 aad_adr; - } u1; - __u64 auth_res_addr; - union { - __u8 inner_prefix_sz; - __u8 aad_sz; - } u2; - __u8 resrvd1; - __u8 hash_state_sz; - __u8 auth_res_sz; -} __packed; - -struct icp_qat_fw_la_auth_req_params_resrvd_flds { - __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_6]; - union { - __u8 inner_prefix_sz; - __u8 aad_sz; - } u2; - __u8 resrvd1; - __u16 resrvd2; -}; - -struct icp_qat_fw_la_resp { - struct icp_qat_fw_comn_resp_hdr comn_resp; - __u64 opaque_data; - __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_4]; -}; - -#define ICP_QAT_FW_CIPHER_NEXT_ID_GET(cd_ctrl_hdr_t) \ - ((((cd_ctrl_hdr_t)->next_curr_id_cipher) & \ - ICP_QAT_FW_COMN_NEXT_ID_MASK) >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) - -#define ICP_QAT_FW_CIPHER_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ -{ (cd_ctrl_hdr_t)->next_curr_id_cipher = \ - ((((cd_ctrl_hdr_t)->next_curr_id_cipher) \ - & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ - ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ - & ICP_QAT_FW_COMN_NEXT_ID_MASK)) } - -#define ICP_QAT_FW_CIPHER_CURR_ID_GET(cd_ctrl_hdr_t) \ - (((cd_ctrl_hdr_t)->next_curr_id_cipher) \ - & ICP_QAT_FW_COMN_CURR_ID_MASK) - -#define ICP_QAT_FW_CIPHER_CURR_ID_SET(cd_ctrl_hdr_t, val) \ -{ (cd_ctrl_hdr_t)->next_curr_id_cipher = \ - ((((cd_ctrl_hdr_t)->next_curr_id_cipher) \ - & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ - ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)) } - -#define ICP_QAT_FW_AUTH_NEXT_ID_GET(cd_ctrl_hdr_t) \ - ((((cd_ctrl_hdr_t)->next_curr_id_auth) & ICP_QAT_FW_COMN_NEXT_ID_MASK) \ - >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) - -#define ICP_QAT_FW_AUTH_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ -{ (cd_ctrl_hdr_t)->next_curr_id_auth = \ - ((((cd_ctrl_hdr_t)->next_curr_id_auth) \ - & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ - ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ - & ICP_QAT_FW_COMN_NEXT_ID_MASK)) } - -#define ICP_QAT_FW_AUTH_CURR_ID_GET(cd_ctrl_hdr_t) \ - (((cd_ctrl_hdr_t)->next_curr_id_auth) \ - & ICP_QAT_FW_COMN_CURR_ID_MASK) - -#define ICP_QAT_FW_AUTH_CURR_ID_SET(cd_ctrl_hdr_t, val) \ -{ (cd_ctrl_hdr_t)->next_curr_id_auth = \ - ((((cd_ctrl_hdr_t)->next_curr_id_auth) \ - & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ - ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)) } - -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h deleted file mode 100644 index 7eb5daef4f88..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef __ICP_QAT_FW_LOADER_HANDLE_H__ -#define __ICP_QAT_FW_LOADER_HANDLE_H__ -#include "icp_qat_uclo.h" - -struct icp_qat_fw_loader_ae_data { - unsigned int state; - unsigned int ustore_size; - unsigned int free_addr; - unsigned int free_size; - unsigned int live_ctx_mask; -}; - -struct icp_qat_fw_loader_hal_handle { - struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; - unsigned int ae_mask; - unsigned int admin_ae_mask; - unsigned int slice_mask; - unsigned int revision_id; - unsigned int ae_max_num; - unsigned int upc_mask; - unsigned int max_ustore; -}; - -struct icp_qat_fw_loader_chip_info { - int mmp_sram_size; - bool nn; - bool lm2lm3; - u32 lm_size; - u32 icp_rst_csr; - u32 icp_rst_mask; - u32 glb_clk_enable_csr; - u32 misc_ctl_csr; - u32 wakeup_event_val; - bool fw_auth; - bool css_3k; - bool tgroup_share_ustore; - u32 fcu_ctl_csr; - u32 fcu_sts_csr; - u32 fcu_dram_addr_hi; - u32 fcu_dram_addr_lo; - u32 fcu_loaded_ae_csr; - u8 fcu_loaded_ae_pos; -}; - -struct icp_qat_fw_loader_handle { - struct icp_qat_fw_loader_hal_handle *hal_handle; - struct icp_qat_fw_loader_chip_info *chip_info; - struct pci_dev *pci_dev; - void *obj_handle; - void *sobj_handle; - void *mobj_handle; - unsigned int cfg_ae_mask; - void __iomem *hal_sram_addr_v; - void __iomem *hal_cap_g_ctl_csr_addr_v; - void __iomem *hal_cap_ae_xfer_csr_addr_v; - void __iomem *hal_cap_ae_local_csr_addr_v; - void __iomem *hal_ep_csr_addr_v; -}; - -struct icp_firml_dram_desc { - void __iomem *dram_base_addr; - void *dram_base_addr_v; - dma_addr_t dram_bus_addr; - u64 dram_size; -}; -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h b/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h deleted file mode 100644 index 9dddae0009fc..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef _ICP_QAT_FW_PKE_ -#define _ICP_QAT_FW_PKE_ - -#include "icp_qat_fw.h" - -struct icp_qat_fw_req_hdr_pke_cd_pars { - __u64 content_desc_addr; - __u32 content_desc_resrvd; - __u32 func_id; -}; - -struct icp_qat_fw_req_pke_mid { - __u64 opaque; - __u64 src_data_addr; - __u64 dest_data_addr; -}; - -struct icp_qat_fw_req_pke_hdr { - __u8 resrvd1; - __u8 resrvd2; - __u8 service_type; - __u8 hdr_flags; - __u16 comn_req_flags; - __u16 resrvd4; - struct icp_qat_fw_req_hdr_pke_cd_pars cd_pars; -}; - -struct icp_qat_fw_pke_request { - struct icp_qat_fw_req_pke_hdr pke_hdr; - struct icp_qat_fw_req_pke_mid pke_mid; - __u8 output_param_count; - __u8 input_param_count; - __u16 resrvd1; - __u32 resrvd2; - __u64 next_req_adr; -}; - -struct icp_qat_fw_resp_pke_hdr { - __u8 resrvd1; - __u8 resrvd2; - __u8 response_type; - __u8 hdr_flags; - __u16 comn_resp_flags; - __u16 resrvd4; -}; - -struct icp_qat_fw_pke_resp { - struct icp_qat_fw_resp_pke_hdr pke_resp_hdr; - __u64 opaque; - __u64 src_data_addr; - __u64 dest_data_addr; -}; - -#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS 7 -#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK 0x1 -#define ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(status_word) \ - QAT_FIELD_GET(((status_word >> ICP_QAT_FW_COMN_ONE_BYTE_SHIFT) & \ - ICP_QAT_FW_COMN_SINGLE_BYTE_MASK), \ - QAT_COMN_RESP_PKE_STATUS_BITPOS, \ - QAT_COMN_RESP_PKE_STATUS_MASK) - -#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(hdr_t, val) \ - QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ - ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS, \ - ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK) -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h deleted file mode 100644 index 20b2ee1fc65a..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ /dev/null @@ -1,143 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef __ICP_QAT_HAL_H -#define __ICP_QAT_HAL_H -#include "icp_qat_fw_loader_handle.h" - -enum hal_global_csr { - MISC_CONTROL = 0xA04, - ICP_RESET = 0xA0c, - ICP_GLOBAL_CLK_ENABLE = 0xA50 -}; - -enum { - MISC_CONTROL_C4XXX = 0xAA0, - ICP_RESET_CPP0 = 0x938, - ICP_RESET_CPP1 = 0x93c, - ICP_GLOBAL_CLK_ENABLE_CPP0 = 0x964, - ICP_GLOBAL_CLK_ENABLE_CPP1 = 0x968 -}; - -enum hal_ae_csr { - USTORE_ADDRESS = 0x000, - USTORE_DATA_LOWER = 0x004, - USTORE_DATA_UPPER = 0x008, - ALU_OUT = 0x010, - CTX_ARB_CNTL = 0x014, - CTX_ENABLES = 0x018, - CC_ENABLE = 0x01c, - CSR_CTX_POINTER = 0x020, - CTX_STS_INDIRECT = 0x040, - ACTIVE_CTX_STATUS = 0x044, - CTX_SIG_EVENTS_INDIRECT = 0x048, - CTX_SIG_EVENTS_ACTIVE = 0x04c, - CTX_WAKEUP_EVENTS_INDIRECT = 0x050, - LM_ADDR_0_INDIRECT = 0x060, - LM_ADDR_1_INDIRECT = 0x068, - LM_ADDR_2_INDIRECT = 0x0cc, - LM_ADDR_3_INDIRECT = 0x0d4, - INDIRECT_LM_ADDR_0_BYTE_INDEX = 0x0e0, - INDIRECT_LM_ADDR_1_BYTE_INDEX = 0x0e8, - INDIRECT_LM_ADDR_2_BYTE_INDEX = 0x10c, - INDIRECT_LM_ADDR_3_BYTE_INDEX = 0x114, - INDIRECT_T_INDEX = 0x0f8, - INDIRECT_T_INDEX_BYTE_INDEX = 0x0fc, - FUTURE_COUNT_SIGNAL_INDIRECT = 0x078, - TIMESTAMP_LOW = 0x0c0, - TIMESTAMP_HIGH = 0x0c4, - PROFILE_COUNT = 0x144, - SIGNATURE_ENABLE = 0x150, - AE_MISC_CONTROL = 0x160, - LOCAL_CSR_STATUS = 0x180, -}; - -enum fcu_csr { - FCU_CONTROL = 0x8c0, - FCU_STATUS = 0x8c4, - FCU_STATUS1 = 0x8c8, - FCU_DRAM_ADDR_LO = 0x8cc, - FCU_DRAM_ADDR_HI = 0x8d0, - FCU_RAMBASE_ADDR_HI = 0x8d4, - FCU_RAMBASE_ADDR_LO = 0x8d8 -}; - -enum fcu_csr_4xxx { - FCU_CONTROL_4XXX = 0x1000, - FCU_STATUS_4XXX = 0x1004, - FCU_ME_BROADCAST_MASK_TYPE = 0x1008, - FCU_AE_LOADED_4XXX = 0x1010, - FCU_DRAM_ADDR_LO_4XXX = 0x1014, - FCU_DRAM_ADDR_HI_4XXX = 0x1018, -}; - -enum fcu_cmd { - FCU_CTRL_CMD_NOOP = 0, - FCU_CTRL_CMD_AUTH = 1, - FCU_CTRL_CMD_LOAD = 2, - FCU_CTRL_CMD_START = 3 -}; - -enum fcu_sts { - FCU_STS_NO_STS = 0, - FCU_STS_VERI_DONE = 1, - FCU_STS_LOAD_DONE = 2, - FCU_STS_VERI_FAIL = 3, - FCU_STS_LOAD_FAIL = 4, - FCU_STS_BUSY = 5 -}; - -#define ALL_AE_MASK 0xFFFFFFFF -#define UA_ECS (0x1 << 31) -#define ACS_ABO_BITPOS 31 -#define ACS_ACNO 0x7 -#define CE_ENABLE_BITPOS 0x8 -#define CE_LMADDR_0_GLOBAL_BITPOS 16 -#define CE_LMADDR_1_GLOBAL_BITPOS 17 -#define CE_LMADDR_2_GLOBAL_BITPOS 22 -#define CE_LMADDR_3_GLOBAL_BITPOS 23 -#define CE_T_INDEX_GLOBAL_BITPOS 21 -#define CE_NN_MODE_BITPOS 20 -#define CE_REG_PAR_ERR_BITPOS 25 -#define CE_BREAKPOINT_BITPOS 27 -#define CE_CNTL_STORE_PARITY_ERROR_BITPOS 29 -#define CE_INUSE_CONTEXTS_BITPOS 31 -#define CE_NN_MODE (0x1 << CE_NN_MODE_BITPOS) -#define CE_INUSE_CONTEXTS (0x1 << CE_INUSE_CONTEXTS_BITPOS) -#define XCWE_VOLUNTARY (0x1) -#define LCS_STATUS (0x1) -#define MMC_SHARE_CS_BITPOS 2 -#define WAKEUP_EVENT 0x10000 -#define FCU_CTRL_BROADCAST_POS 0x4 -#define FCU_CTRL_AE_POS 0x8 -#define FCU_AUTH_STS_MASK 0x7 -#define FCU_STS_DONE_POS 0x9 -#define FCU_STS_AUTHFWLD_POS 0X8 -#define FCU_LOADED_AE_POS 0x16 -#define FW_AUTH_WAIT_PERIOD 10 -#define FW_AUTH_MAX_RETRY 300 -#define ICP_QAT_AE_OFFSET 0x20000 -#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) -#define LOCAL_TO_XFER_REG_OFFSET 0x800 -#define ICP_QAT_EP_OFFSET 0x3a000 -#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */ -#define ICP_QAT_AE_OFFSET_4XXX 0x600000 -#define ICP_QAT_CAP_OFFSET_4XXX 0x640000 -#define SET_CAP_CSR(handle, csr, val) \ - ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) -#define GET_CAP_CSR(handle, csr) \ - ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) -#define AE_CSR(handle, ae) \ - ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) -#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) -#define SET_AE_CSR(handle, ae, csr, val) \ - ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) -#define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) -#define AE_XFER(handle, ae) \ - ((char __iomem *)(handle)->hal_cap_ae_xfer_csr_addr_v + ((ae) << 12)) -#define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ - (((reg) & 0xff) << 2)) -#define SET_AE_XFER(handle, ae, reg, val) \ - ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) -#define SRAM_WRITE(handle, addr, val) \ - ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val) -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h deleted file mode 100644 index 4042739bb6fa..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ /dev/null @@ -1,376 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef _ICP_QAT_HW_H_ -#define _ICP_QAT_HW_H_ - -enum icp_qat_hw_ae_id { - ICP_QAT_HW_AE_0 = 0, - ICP_QAT_HW_AE_1 = 1, - ICP_QAT_HW_AE_2 = 2, - ICP_QAT_HW_AE_3 = 3, - ICP_QAT_HW_AE_4 = 4, - ICP_QAT_HW_AE_5 = 5, - ICP_QAT_HW_AE_6 = 6, - ICP_QAT_HW_AE_7 = 7, - ICP_QAT_HW_AE_8 = 8, - ICP_QAT_HW_AE_9 = 9, - ICP_QAT_HW_AE_10 = 10, - ICP_QAT_HW_AE_11 = 11, - ICP_QAT_HW_AE_DELIMITER = 12 -}; - -enum icp_qat_hw_qat_id { - ICP_QAT_HW_QAT_0 = 0, - ICP_QAT_HW_QAT_1 = 1, - ICP_QAT_HW_QAT_2 = 2, - ICP_QAT_HW_QAT_3 = 3, - ICP_QAT_HW_QAT_4 = 4, - ICP_QAT_HW_QAT_5 = 5, - ICP_QAT_HW_QAT_DELIMITER = 6 -}; - -enum icp_qat_hw_auth_algo { - ICP_QAT_HW_AUTH_ALGO_NULL = 0, - ICP_QAT_HW_AUTH_ALGO_SHA1 = 1, - ICP_QAT_HW_AUTH_ALGO_MD5 = 2, - ICP_QAT_HW_AUTH_ALGO_SHA224 = 3, - ICP_QAT_HW_AUTH_ALGO_SHA256 = 4, - ICP_QAT_HW_AUTH_ALGO_SHA384 = 5, - ICP_QAT_HW_AUTH_ALGO_SHA512 = 6, - ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, - ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, - ICP_QAT_HW_AUTH_ALGO_AES_F9 = 9, - ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, - ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, - ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, - ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, - ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 = 14, - ICP_QAT_HW_AUTH_RESERVED_1 = 15, - ICP_QAT_HW_AUTH_RESERVED_2 = 16, - ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, - ICP_QAT_HW_AUTH_RESERVED_3 = 18, - ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, - ICP_QAT_HW_AUTH_ALGO_DELIMITER = 20 -}; - -enum icp_qat_hw_auth_mode { - ICP_QAT_HW_AUTH_MODE0 = 0, - ICP_QAT_HW_AUTH_MODE1 = 1, - ICP_QAT_HW_AUTH_MODE2 = 2, - ICP_QAT_HW_AUTH_MODE_DELIMITER = 3 -}; - -struct icp_qat_hw_auth_config { - __u32 config; - __u32 reserved; -}; - -struct icp_qat_hw_ucs_cipher_config { - __u32 val; - __u32 reserved[3]; -}; - -enum icp_qat_slice_mask { - ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), - ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), - ICP_ACCEL_MASK_PKE_SLICE = BIT(2), - ICP_ACCEL_MASK_COMPRESS_SLICE = BIT(3), - ICP_ACCEL_MASK_LZS_SLICE = BIT(4), - ICP_ACCEL_MASK_EIA3_SLICE = BIT(5), - ICP_ACCEL_MASK_SHA3_SLICE = BIT(6), -}; - -enum icp_qat_capabilities_mask { - ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = BIT(0), - ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = BIT(1), - ICP_ACCEL_CAPABILITIES_CIPHER = BIT(2), - ICP_ACCEL_CAPABILITIES_AUTHENTICATION = BIT(3), - ICP_ACCEL_CAPABILITIES_RESERVED_1 = BIT(4), - ICP_ACCEL_CAPABILITIES_COMPRESSION = BIT(5), - ICP_ACCEL_CAPABILITIES_LZS_COMPRESSION = BIT(6), - ICP_ACCEL_CAPABILITIES_RAND = BIT(7), - ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), - ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), - /* Bits 10-11 are currently reserved */ - ICP_ACCEL_CAPABILITIES_HKDF = BIT(12), - ICP_ACCEL_CAPABILITIES_ECEDMONT = BIT(13), - /* Bit 14 is currently reserved */ - ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), - ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), - ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), - /* Bits 18-21 are currently reserved */ - ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY = BIT(22), - ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23), - ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), - ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION = BIT(25), - ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) -}; - -#define QAT_AUTH_MODE_BITPOS 4 -#define QAT_AUTH_MODE_MASK 0xF -#define QAT_AUTH_ALGO_BITPOS 0 -#define QAT_AUTH_ALGO_MASK 0xF -#define QAT_AUTH_CMP_BITPOS 8 -#define QAT_AUTH_CMP_MASK 0x7F -#define QAT_AUTH_SHA3_PADDING_BITPOS 16 -#define QAT_AUTH_SHA3_PADDING_MASK 0x1 -#define QAT_AUTH_ALGO_SHA3_BITPOS 22 -#define QAT_AUTH_ALGO_SHA3_MASK 0x3 -#define ICP_QAT_HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len) \ - (((mode & QAT_AUTH_MODE_MASK) << QAT_AUTH_MODE_BITPOS) | \ - ((algo & QAT_AUTH_ALGO_MASK) << QAT_AUTH_ALGO_BITPOS) | \ - (((algo >> 4) & QAT_AUTH_ALGO_SHA3_MASK) << \ - QAT_AUTH_ALGO_SHA3_BITPOS) | \ - (((((algo == ICP_QAT_HW_AUTH_ALGO_SHA3_256) || \ - (algo == ICP_QAT_HW_AUTH_ALGO_SHA3_512)) ? 1 : 0) \ - & QAT_AUTH_SHA3_PADDING_MASK) << QAT_AUTH_SHA3_PADDING_BITPOS) | \ - ((cmp_len & QAT_AUTH_CMP_MASK) << QAT_AUTH_CMP_BITPOS)) - -struct icp_qat_hw_auth_counter { - __be32 counter; - __u32 reserved; -}; - -#define QAT_AUTH_COUNT_MASK 0xFFFFFFFF -#define QAT_AUTH_COUNT_BITPOS 0 -#define ICP_QAT_HW_AUTH_COUNT_BUILD(val) \ - (((val) & QAT_AUTH_COUNT_MASK) << QAT_AUTH_COUNT_BITPOS) - -struct icp_qat_hw_auth_setup { - struct icp_qat_hw_auth_config auth_config; - struct icp_qat_hw_auth_counter auth_counter; -}; - -#define QAT_HW_DEFAULT_ALIGNMENT 8 -#define QAT_HW_ROUND_UP(val, n) (((val) + ((n) - 1)) & (~(n - 1))) -#define ICP_QAT_HW_NULL_STATE1_SZ 32 -#define ICP_QAT_HW_MD5_STATE1_SZ 16 -#define ICP_QAT_HW_SHA1_STATE1_SZ 20 -#define ICP_QAT_HW_SHA224_STATE1_SZ 32 -#define ICP_QAT_HW_SHA256_STATE1_SZ 32 -#define ICP_QAT_HW_SHA3_256_STATE1_SZ 32 -#define ICP_QAT_HW_SHA384_STATE1_SZ 64 -#define ICP_QAT_HW_SHA512_STATE1_SZ 64 -#define ICP_QAT_HW_SHA3_512_STATE1_SZ 64 -#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28 -#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48 -#define ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ 16 -#define ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ 16 -#define ICP_QAT_HW_AES_F9_STATE1_SZ 32 -#define ICP_QAT_HW_KASUMI_F9_STATE1_SZ 16 -#define ICP_QAT_HW_GALOIS_128_STATE1_SZ 16 -#define ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ 8 -#define ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ 8 -#define ICP_QAT_HW_NULL_STATE2_SZ 32 -#define ICP_QAT_HW_MD5_STATE2_SZ 16 -#define ICP_QAT_HW_SHA1_STATE2_SZ 20 -#define ICP_QAT_HW_SHA224_STATE2_SZ 32 -#define ICP_QAT_HW_SHA256_STATE2_SZ 32 -#define ICP_QAT_HW_SHA3_256_STATE2_SZ 0 -#define ICP_QAT_HW_SHA384_STATE2_SZ 64 -#define ICP_QAT_HW_SHA512_STATE2_SZ 64 -#define ICP_QAT_HW_SHA3_512_STATE2_SZ 0 -#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0 -#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0 -#define ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ 16 -#define ICP_QAT_HW_AES_CBC_MAC_KEY_SZ 16 -#define ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ 16 -#define ICP_QAT_HW_F9_IK_SZ 16 -#define ICP_QAT_HW_F9_FK_SZ 16 -#define ICP_QAT_HW_KASUMI_F9_STATE2_SZ (ICP_QAT_HW_F9_IK_SZ + \ - ICP_QAT_HW_F9_FK_SZ) -#define ICP_QAT_HW_AES_F9_STATE2_SZ ICP_QAT_HW_KASUMI_F9_STATE2_SZ -#define ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ 24 -#define ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ 32 -#define ICP_QAT_HW_GALOIS_H_SZ 16 -#define ICP_QAT_HW_GALOIS_LEN_A_SZ 8 -#define ICP_QAT_HW_GALOIS_E_CTR0_SZ 16 - -struct icp_qat_hw_auth_sha512 { - struct icp_qat_hw_auth_setup inner_setup; - __u8 state1[ICP_QAT_HW_SHA512_STATE1_SZ]; - struct icp_qat_hw_auth_setup outer_setup; - __u8 state2[ICP_QAT_HW_SHA512_STATE2_SZ]; -}; - -struct icp_qat_hw_auth_algo_blk { - struct icp_qat_hw_auth_sha512 sha; -}; - -#define ICP_QAT_HW_GALOIS_LEN_A_BITPOS 0 -#define ICP_QAT_HW_GALOIS_LEN_A_MASK 0xFFFFFFFF - -enum icp_qat_hw_cipher_algo { - ICP_QAT_HW_CIPHER_ALGO_NULL = 0, - ICP_QAT_HW_CIPHER_ALGO_DES = 1, - ICP_QAT_HW_CIPHER_ALGO_3DES = 2, - ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, - ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, - ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, - ICP_QAT_HW_CIPHER_ALGO_ARC4 = 6, - ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, - ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, - ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, - ICP_QAT_HW_CIPHER_DELIMITER = 10 -}; - -enum icp_qat_hw_cipher_mode { - ICP_QAT_HW_CIPHER_ECB_MODE = 0, - ICP_QAT_HW_CIPHER_CBC_MODE = 1, - ICP_QAT_HW_CIPHER_CTR_MODE = 2, - ICP_QAT_HW_CIPHER_F8_MODE = 3, - ICP_QAT_HW_CIPHER_XTS_MODE = 6, - ICP_QAT_HW_CIPHER_MODE_DELIMITER = 7 -}; - -struct icp_qat_hw_cipher_config { - __u32 val; - __u32 reserved; -}; - -enum icp_qat_hw_cipher_dir { - ICP_QAT_HW_CIPHER_ENCRYPT = 0, - ICP_QAT_HW_CIPHER_DECRYPT = 1, -}; - -enum icp_qat_hw_cipher_convert { - ICP_QAT_HW_CIPHER_NO_CONVERT = 0, - ICP_QAT_HW_CIPHER_KEY_CONVERT = 1, -}; - -#define QAT_CIPHER_MODE_BITPOS 4 -#define QAT_CIPHER_MODE_MASK 0xF -#define QAT_CIPHER_ALGO_BITPOS 0 -#define QAT_CIPHER_ALGO_MASK 0xF -#define QAT_CIPHER_CONVERT_BITPOS 9 -#define QAT_CIPHER_CONVERT_MASK 0x1 -#define QAT_CIPHER_DIR_BITPOS 8 -#define QAT_CIPHER_DIR_MASK 0x1 -#define QAT_CIPHER_MODE_F8_KEY_SZ_MULT 2 -#define QAT_CIPHER_MODE_XTS_KEY_SZ_MULT 2 -#define ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, algo, convert, dir) \ - (((mode & QAT_CIPHER_MODE_MASK) << QAT_CIPHER_MODE_BITPOS) | \ - ((algo & QAT_CIPHER_ALGO_MASK) << QAT_CIPHER_ALGO_BITPOS) | \ - ((convert & QAT_CIPHER_CONVERT_MASK) << QAT_CIPHER_CONVERT_BITPOS) | \ - ((dir & QAT_CIPHER_DIR_MASK) << QAT_CIPHER_DIR_BITPOS)) -#define ICP_QAT_HW_DES_BLK_SZ 8 -#define ICP_QAT_HW_3DES_BLK_SZ 8 -#define ICP_QAT_HW_NULL_BLK_SZ 8 -#define ICP_QAT_HW_AES_BLK_SZ 16 -#define ICP_QAT_HW_KASUMI_BLK_SZ 8 -#define ICP_QAT_HW_SNOW_3G_BLK_SZ 8 -#define ICP_QAT_HW_ZUC_3G_BLK_SZ 8 -#define ICP_QAT_HW_NULL_KEY_SZ 256 -#define ICP_QAT_HW_DES_KEY_SZ 8 -#define ICP_QAT_HW_3DES_KEY_SZ 24 -#define ICP_QAT_HW_AES_128_KEY_SZ 16 -#define ICP_QAT_HW_AES_192_KEY_SZ 24 -#define ICP_QAT_HW_AES_256_KEY_SZ 32 -#define ICP_QAT_HW_AES_128_F8_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ - QAT_CIPHER_MODE_F8_KEY_SZ_MULT) -#define ICP_QAT_HW_AES_192_F8_KEY_SZ (ICP_QAT_HW_AES_192_KEY_SZ * \ - QAT_CIPHER_MODE_F8_KEY_SZ_MULT) -#define ICP_QAT_HW_AES_256_F8_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ - QAT_CIPHER_MODE_F8_KEY_SZ_MULT) -#define ICP_QAT_HW_AES_128_XTS_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ - QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) -#define ICP_QAT_HW_AES_256_XTS_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ - QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) -#define ICP_QAT_HW_KASUMI_KEY_SZ 16 -#define ICP_QAT_HW_KASUMI_F8_KEY_SZ (ICP_QAT_HW_KASUMI_KEY_SZ * \ - QAT_CIPHER_MODE_F8_KEY_SZ_MULT) -#define ICP_QAT_HW_AES_128_XTS_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ - QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) -#define ICP_QAT_HW_AES_256_XTS_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ - QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) -#define ICP_QAT_HW_ARC4_KEY_SZ 256 -#define ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ 16 -#define ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ 16 -#define ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ 16 -#define ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ 16 -#define ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR 2 -#define INIT_SHRAM_CONSTANTS_TABLE_SZ 1024 - -struct icp_qat_hw_cipher_aes256_f8 { - struct icp_qat_hw_cipher_config cipher_config; - __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; -}; - -struct icp_qat_hw_ucs_cipher_aes256_f8 { - struct icp_qat_hw_ucs_cipher_config cipher_config; - __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; -}; - -struct icp_qat_hw_cipher_algo_blk { - union { - struct icp_qat_hw_cipher_aes256_f8 aes; - struct icp_qat_hw_ucs_cipher_aes256_f8 ucs_aes; - }; -} __aligned(64); - -enum icp_qat_hw_compression_direction { - ICP_QAT_HW_COMPRESSION_DIR_COMPRESS = 0, - ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS = 1, - ICP_QAT_HW_COMPRESSION_DIR_DELIMITER = 2 -}; - -enum icp_qat_hw_compression_delayed_match { - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED = 0, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED = 1, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DELIMITER = 2 -}; - -enum icp_qat_hw_compression_algo { - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE = 0, - ICP_QAT_HW_COMPRESSION_ALGO_LZS = 1, - ICP_QAT_HW_COMPRESSION_ALGO_DELIMITER = 2 -}; - -enum icp_qat_hw_compression_depth { - ICP_QAT_HW_COMPRESSION_DEPTH_1 = 0, - ICP_QAT_HW_COMPRESSION_DEPTH_4 = 1, - ICP_QAT_HW_COMPRESSION_DEPTH_8 = 2, - ICP_QAT_HW_COMPRESSION_DEPTH_16 = 3, - ICP_QAT_HW_COMPRESSION_DEPTH_128 = 4, - ICP_QAT_HW_COMPRESSION_DEPTH_DELIMITER = 5 -}; - -enum icp_qat_hw_compression_file_type { - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0 = 0, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_1 = 1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_2 = 2, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_3 = 3, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_4 = 4, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_DELIMITER = 5 -}; - -struct icp_qat_hw_compression_config { - __u32 lower_val; - __u32 upper_val; -}; - -#define QAT_COMPRESSION_DIR_BITPOS 4 -#define QAT_COMPRESSION_DIR_MASK 0x7 -#define QAT_COMPRESSION_DELAYED_MATCH_BITPOS 16 -#define QAT_COMPRESSION_DELAYED_MATCH_MASK 0x1 -#define QAT_COMPRESSION_ALGO_BITPOS 31 -#define QAT_COMPRESSION_ALGO_MASK 0x1 -#define QAT_COMPRESSION_DEPTH_BITPOS 28 -#define QAT_COMPRESSION_DEPTH_MASK 0x7 -#define QAT_COMPRESSION_FILE_TYPE_BITPOS 24 -#define QAT_COMPRESSION_FILE_TYPE_MASK 0xF - -#define ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(dir, delayed, \ - algo, depth, filetype) \ - ((((dir) & QAT_COMPRESSION_DIR_MASK) << \ - QAT_COMPRESSION_DIR_BITPOS) | \ - (((delayed) & QAT_COMPRESSION_DELAYED_MATCH_MASK) << \ - QAT_COMPRESSION_DELAYED_MATCH_BITPOS) | \ - (((algo) & QAT_COMPRESSION_ALGO_MASK) << \ - QAT_COMPRESSION_ALGO_BITPOS) | \ - (((depth) & QAT_COMPRESSION_DEPTH_MASK) << \ - QAT_COMPRESSION_DEPTH_BITPOS) | \ - (((filetype) & QAT_COMPRESSION_FILE_TYPE_MASK) << \ - QAT_COMPRESSION_FILE_TYPE_BITPOS)) - -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h b/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h deleted file mode 100644 index 7ea8962272f2..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h +++ /dev/null @@ -1,164 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef _ICP_QAT_HW_20_COMP_H_ -#define _ICP_QAT_HW_20_COMP_H_ - -#include "icp_qat_hw_20_comp_defs.h" -#include "icp_qat_fw.h" - -struct icp_qat_hw_comp_20_config_csr_lower { - enum icp_qat_hw_comp_20_extended_delay_match_mode edmm; - enum icp_qat_hw_comp_20_hw_comp_format algo; - enum icp_qat_hw_comp_20_search_depth sd; - enum icp_qat_hw_comp_20_hbs_control hbs; - enum icp_qat_hw_comp_20_abd abd; - enum icp_qat_hw_comp_20_lllbd_ctrl lllbd; - enum icp_qat_hw_comp_20_min_match_control mmctrl; - enum icp_qat_hw_comp_20_skip_hash_collision hash_col; - enum icp_qat_hw_comp_20_skip_hash_update hash_update; - enum icp_qat_hw_comp_20_byte_skip skip_ctrl; -}; - -static inline __u32 -ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(struct icp_qat_hw_comp_20_config_csr_lower csr) -{ - u32 val32 = 0; - - QAT_FIELD_SET(val32, csr.algo, - ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK); - QAT_FIELD_SET(val32, csr.sd, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK); - QAT_FIELD_SET(val32, csr.edmm, - ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK); - QAT_FIELD_SET(val32, csr.hbs, - ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.lllbd, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK); - QAT_FIELD_SET(val32, csr.mmctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.hash_col, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK); - QAT_FIELD_SET(val32, csr.hash_update, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK); - QAT_FIELD_SET(val32, csr.skip_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK); - QAT_FIELD_SET(val32, csr.abd, ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK); - - return __builtin_bswap32(val32); -} - -struct icp_qat_hw_comp_20_config_csr_upper { - enum icp_qat_hw_comp_20_scb_control scb_ctrl; - enum icp_qat_hw_comp_20_rmb_control rmb_ctrl; - enum icp_qat_hw_comp_20_som_control som_ctrl; - enum icp_qat_hw_comp_20_skip_hash_rd_control skip_hash_ctrl; - enum icp_qat_hw_comp_20_scb_unload_control scb_unload_ctrl; - enum icp_qat_hw_comp_20_disable_token_fusion_control disable_token_fusion_ctrl; - enum icp_qat_hw_comp_20_lbms lbms; - enum icp_qat_hw_comp_20_scb_mode_reset_mask scb_mode_reset; - __u16 lazy; - __u16 nice; -}; - -static inline __u32 -ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(struct icp_qat_hw_comp_20_config_csr_upper csr) -{ - u32 val32 = 0; - - QAT_FIELD_SET(val32, csr.scb_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.rmb_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.som_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.skip_hash_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.scb_unload_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.disable_token_fusion_ctrl, - ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.lbms, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_MASK); - QAT_FIELD_SET(val32, csr.scb_mode_reset, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); - QAT_FIELD_SET(val32, csr.lazy, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK); - QAT_FIELD_SET(val32, csr.nice, - ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK); - - return __builtin_bswap32(val32); -} - -struct icp_qat_hw_decomp_20_config_csr_lower { - enum icp_qat_hw_decomp_20_hbs_control hbs; - enum icp_qat_hw_decomp_20_lbms lbms; - enum icp_qat_hw_decomp_20_hw_comp_format algo; - enum icp_qat_hw_decomp_20_min_match_control mmctrl; - enum icp_qat_hw_decomp_20_lz4_block_checksum_present lbc; -}; - -static inline __u32 -ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(struct icp_qat_hw_decomp_20_config_csr_lower csr) -{ - u32 val32 = 0; - - QAT_FIELD_SET(val32, csr.hbs, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.lbms, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_MASK); - QAT_FIELD_SET(val32, csr.algo, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK); - QAT_FIELD_SET(val32, csr.mmctrl, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.lbc, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK); - - return __builtin_bswap32(val32); -} - -struct icp_qat_hw_decomp_20_config_csr_upper { - enum icp_qat_hw_decomp_20_speculative_decoder_control sdc; - enum icp_qat_hw_decomp_20_mini_cam_control mcc; -}; - -static inline __u32 -ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_UPPER(struct icp_qat_hw_decomp_20_config_csr_upper csr) -{ - u32 val32 = 0; - - QAT_FIELD_SET(val32, csr.sdc, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK); - QAT_FIELD_SET(val32, csr.mcc, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK); - - return __builtin_bswap32(val32); -} - -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp_defs.h b/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp_defs.h deleted file mode 100644 index 208d4554283b..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp_defs.h +++ /dev/null @@ -1,300 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef _ICP_QAT_HW_20_COMP_DEFS_H -#define _ICP_QAT_HW_20_COMP_DEFS_H - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS 31 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK 0x1 - -enum icp_qat_hw_comp_20_scb_control { - ICP_QAT_HW_COMP_20_SCB_CONTROL_ENABLE = 0x0, - ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS 30 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK 0x1 - -enum icp_qat_hw_comp_20_rmb_control { - ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL = 0x0, - ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_FC_ONLY = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS 28 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK 0x3 - -enum icp_qat_hw_comp_20_som_control { - ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE = 0x0, - ICP_QAT_HW_COMP_20_SOM_CONTROL_REPLAY_MODE = 0x1, - ICP_QAT_HW_COMP_20_SOM_CONTROL_INPUT_CRC = 0x2, - ICP_QAT_HW_COMP_20_SOM_CONTROL_RESERVED_MODE = 0x3, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK 0x1 - -enum icp_qat_hw_comp_20_skip_hash_rd_control { - ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0, - ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS 26 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK 0x1 - -enum icp_qat_hw_comp_20_scb_unload_control { - ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD = 0x0, - ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_NO_UNLOAD = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS 21 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK 0x1 - -enum icp_qat_hw_comp_20_disable_token_fusion_control { - ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE = 0x0, - ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_DISABLE = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_BITPOS 19 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_MASK 0x3 - -enum icp_qat_hw_comp_20_lbms { - ICP_QAT_HW_COMP_20_LBMS_LBMS_64KB = 0x0, - ICP_QAT_HW_COMP_20_LBMS_LBMS_256KB = 0x1, - ICP_QAT_HW_COMP_20_LBMS_LBMS_1MB = 0x2, - ICP_QAT_HW_COMP_20_LBMS_LBMS_4MB = 0x3, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_LBMS_LBMS_64KB - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK 0x1 - -enum icp_qat_hw_comp_20_scb_mode_reset_mask { - ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS = 0x0, - ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS_AND_HISTORY = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS 9 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK 0x1ff -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL 258 - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS 0 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK 0x1ff -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL 259 - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7 - -enum icp_qat_hw_comp_20_hbs_control { - ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, - ICP_QAT_HW_COMP_23_HBS_CONTROL_HBS_IS_64KB = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS 13 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK 0x1 - -enum icp_qat_hw_comp_20_abd { - ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED = 0x0, - ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS 12 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK 0x1 - -enum icp_qat_hw_comp_20_lllbd_ctrl { - ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED = 0x0, - ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK 0xf - -enum icp_qat_hw_comp_20_search_depth { - ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 = 0x1, - ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6 = 0x3, - ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9 = 0x4, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS 5 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK 0x7 - -enum icp_qat_hw_comp_20_hw_comp_format { - ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77 = 0x0, - ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE = 0x1, - ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4 = 0x2, - ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4S = 0x3, - ICP_QAT_HW_COMP_23_HW_COMP_FORMAT_ZSTD = 0x4, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK 0x1 - -enum icp_qat_hw_comp_20_min_match_control { - ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_3B = 0x0, - ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_4B = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_3B - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK 0x1 - -enum icp_qat_hw_comp_20_skip_hash_collision { - ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW = 0x0, - ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK 0x1 - -enum icp_qat_hw_comp_20_skip_hash_update { - ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW = 0x0, - ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS 1 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK 0x1 - -enum icp_qat_hw_comp_20_byte_skip { - ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN = 0x0, - ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS 0 -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK 0x1 - -enum icp_qat_hw_comp_20_extended_delay_match_mode { - ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED = 0x0, - ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED = 0x1, -}; - -#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_DEFAULT_VAL \ - ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS 31 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK 0x1 - -enum icp_qat_hw_decomp_20_speculative_decoder_control { - ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE = 0x0, - ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_DISABLE = 0x1, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS 30 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK 0x1 - -enum icp_qat_hw_decomp_20_mini_cam_control { - ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE = 0x0, - ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_DISABLE = 0x1, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7 - -enum icp_qat_hw_decomp_20_hbs_control { - ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_BITPOS 8 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_MASK 0x3 - -enum icp_qat_hw_decomp_20_lbms { - ICP_QAT_HW_DECOMP_20_LBMS_LBMS_64KB = 0x0, - ICP_QAT_HW_DECOMP_20_LBMS_LBMS_256KB = 0x1, - ICP_QAT_HW_DECOMP_20_LBMS_LBMS_1MB = 0x2, - ICP_QAT_HW_DECOMP_20_LBMS_LBMS_4MB = 0x3, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_LBMS_LBMS_64KB - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS 5 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK 0x7 - -enum icp_qat_hw_decomp_20_hw_comp_format { - ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE = 0x1, - ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4 = 0x2, - ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4S = 0x3, - ICP_QAT_HW_DECOMP_23_HW_DECOMP_FORMAT_ZSTD = 0x4, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK 0x1 - -enum icp_qat_hw_decomp_20_min_match_control { - ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_3B = 0x0, - ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_4B = 0x1, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_3B - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS 3 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK 0x1 - -enum icp_qat_hw_decomp_20_lz4_block_checksum_present { - ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_ABSENT = 0x0, - ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_PRESENT = 0x1, -}; - -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_ABSENT - -#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h deleted file mode 100644 index 69482abdb8b9..000000000000 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ /dev/null @@ -1,585 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef __ICP_QAT_UCLO_H__ -#define __ICP_QAT_UCLO_H__ - -#define ICP_QAT_AC_895XCC_DEV_TYPE 0x00400000 -#define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 -#define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 -#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 -#define ICP_QAT_UCLO_MAX_AE 12 -#define ICP_QAT_UCLO_MAX_CTX 8 -#define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) -#define ICP_QAT_UCLO_MAX_USTORE 0x4000 -#define ICP_QAT_UCLO_MAX_XFER_REG 128 -#define ICP_QAT_UCLO_MAX_GPR_REG 128 -#define ICP_QAT_UCLO_MAX_LMEM_REG 1024 -#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280 -#define ICP_QAT_UCLO_AE_ALL_CTX 0xff -#define ICP_QAT_UOF_OBJID_LEN 8 -#define ICP_QAT_UOF_FID 0xc6c2 -#define ICP_QAT_UOF_MAJVER 0x4 -#define ICP_QAT_UOF_MINVER 0x11 -#define ICP_QAT_UOF_OBJS "UOF_OBJS" -#define ICP_QAT_UOF_STRT "UOF_STRT" -#define ICP_QAT_UOF_IMAG "UOF_IMAG" -#define ICP_QAT_UOF_IMEM "UOF_IMEM" -#define ICP_QAT_UOF_LOCAL_SCOPE 1 -#define ICP_QAT_UOF_INIT_EXPR 0 -#define ICP_QAT_UOF_INIT_REG 1 -#define ICP_QAT_UOF_INIT_REG_CTX 2 -#define ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP 3 -#define ICP_QAT_SUOF_OBJ_ID_LEN 8 -#define ICP_QAT_SUOF_FID 0x53554f46 -#define ICP_QAT_SUOF_MAJVER 0x0 -#define ICP_QAT_SUOF_MINVER 0x1 -#define ICP_QAT_SUOF_OBJ_NAME_LEN 128 -#define ICP_QAT_MOF_OBJ_ID_LEN 8 -#define ICP_QAT_MOF_OBJ_CHUNKID_LEN 8 -#define ICP_QAT_MOF_FID 0x00666f6d -#define ICP_QAT_MOF_MAJVER 0x0 -#define ICP_QAT_MOF_MINVER 0x1 -#define ICP_QAT_MOF_SYM_OBJS "SYM_OBJS" -#define ICP_QAT_SUOF_OBJS "SUF_OBJS" -#define ICP_QAT_SUOF_IMAG "SUF_IMAG" -#define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) -#define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) - -#define DSS_FWSK_MODULUS_LEN 384 /* RSA3K */ -#define DSS_FWSK_EXPONENT_LEN 4 -#define DSS_FWSK_PADDING_LEN 380 -#define DSS_SIGNATURE_LEN 384 /* RSA3K */ - -#define CSS_FWSK_MODULUS_LEN 256 /* RSA2K */ -#define CSS_FWSK_EXPONENT_LEN 4 -#define CSS_FWSK_PADDING_LEN 252 -#define CSS_SIGNATURE_LEN 256 /* RSA2K */ - -#define ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) ((handle)->chip_info->css_3k ? \ - DSS_FWSK_MODULUS_LEN : \ - CSS_FWSK_MODULUS_LEN) - -#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) ((handle)->chip_info->css_3k ? \ - DSS_FWSK_EXPONENT_LEN : \ - CSS_FWSK_EXPONENT_LEN) - -#define ICP_QAT_CSS_FWSK_PAD_LEN(handle) ((handle)->chip_info->css_3k ? \ - DSS_FWSK_PADDING_LEN : \ - CSS_FWSK_PADDING_LEN) - -#define ICP_QAT_CSS_FWSK_PUB_LEN(handle) (ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ - ICP_QAT_CSS_FWSK_PAD_LEN(handle)) - -#define ICP_QAT_CSS_SIGNATURE_LEN(handle) ((handle)->chip_info->css_3k ? \ - DSS_SIGNATURE_LEN : \ - CSS_SIGNATURE_LEN) - -#define ICP_QAT_CSS_AE_IMG_LEN (sizeof(struct icp_qat_simg_ae_mode) + \ - ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ - ICP_QAT_SIMG_AE_INSTS_LEN) -#define ICP_QAT_CSS_AE_SIMG_LEN(handle) (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_PUB_LEN(handle) + \ - ICP_QAT_CSS_SIGNATURE_LEN(handle) + \ - ICP_QAT_CSS_AE_IMG_LEN) -#define ICP_QAT_AE_IMG_OFFSET(handle) (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ - ICP_QAT_CSS_SIGNATURE_LEN(handle)) -#define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 -#define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 - -#define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) -#define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) -#define ICP_QAT_SHARED_USTORE_MODE(ae_mode) (((ae_mode) >> 0xb) & 0x1) -#define RELOADABLE_CTX_SHARED_MODE(ae_mode) (((ae_mode) >> 0xc) & 0x1) - -#define ICP_QAT_LOC_MEM0_MODE(ae_mode) (((ae_mode) >> 0x8) & 0x1) -#define ICP_QAT_LOC_MEM1_MODE(ae_mode) (((ae_mode) >> 0x9) & 0x1) -#define ICP_QAT_LOC_MEM2_MODE(ae_mode) (((ae_mode) >> 0x6) & 0x1) -#define ICP_QAT_LOC_MEM3_MODE(ae_mode) (((ae_mode) >> 0x7) & 0x1) -#define ICP_QAT_LOC_TINDEX_MODE(ae_mode) (((ae_mode) >> 0xe) & 0x1) - -enum icp_qat_uof_mem_region { - ICP_QAT_UOF_SRAM_REGION = 0x0, - ICP_QAT_UOF_LMEM_REGION = 0x3, - ICP_QAT_UOF_UMEM_REGION = 0x5 -}; - -enum icp_qat_uof_regtype { - ICP_NO_DEST = 0, - ICP_GPA_REL = 1, - ICP_GPA_ABS = 2, - ICP_GPB_REL = 3, - ICP_GPB_ABS = 4, - ICP_SR_REL = 5, - ICP_SR_RD_REL = 6, - ICP_SR_WR_REL = 7, - ICP_SR_ABS = 8, - ICP_SR_RD_ABS = 9, - ICP_SR_WR_ABS = 10, - ICP_DR_REL = 19, - ICP_DR_RD_REL = 20, - ICP_DR_WR_REL = 21, - ICP_DR_ABS = 22, - ICP_DR_RD_ABS = 23, - ICP_DR_WR_ABS = 24, - ICP_LMEM = 26, - ICP_LMEM0 = 27, - ICP_LMEM1 = 28, - ICP_NEIGH_REL = 31, - ICP_LMEM2 = 61, - ICP_LMEM3 = 62, -}; - -enum icp_qat_css_fwtype { - CSS_AE_FIRMWARE = 0, - CSS_MMP_FIRMWARE = 1 -}; - -struct icp_qat_uclo_page { - struct icp_qat_uclo_encap_page *encap_page; - struct icp_qat_uclo_region *region; - unsigned int flags; -}; - -struct icp_qat_uclo_region { - struct icp_qat_uclo_page *loaded; - struct icp_qat_uclo_page *page; -}; - -struct icp_qat_uclo_aeslice { - struct icp_qat_uclo_region *region; - struct icp_qat_uclo_page *page; - struct icp_qat_uclo_page *cur_page[ICP_QAT_UCLO_MAX_CTX]; - struct icp_qat_uclo_encapme *encap_image; - unsigned int ctx_mask_assigned; - unsigned int new_uaddr[ICP_QAT_UCLO_MAX_CTX]; -}; - -struct icp_qat_uclo_aedata { - unsigned int slice_num; - unsigned int eff_ustore_size; - struct icp_qat_uclo_aeslice ae_slices[ICP_QAT_UCLO_MAX_CTX]; -}; - -struct icp_qat_uof_encap_obj { - char *beg_uof; - struct icp_qat_uof_objhdr *obj_hdr; - struct icp_qat_uof_chunkhdr *chunk_hdr; - struct icp_qat_uof_varmem_seg *var_mem_seg; -}; - -struct icp_qat_uclo_encap_uwblock { - unsigned int start_addr; - unsigned int words_num; - u64 micro_words; -}; - -struct icp_qat_uclo_encap_page { - unsigned int def_page; - unsigned int page_region; - unsigned int beg_addr_v; - unsigned int beg_addr_p; - unsigned int micro_words_num; - unsigned int uwblock_num; - struct icp_qat_uclo_encap_uwblock *uwblock; -}; - -struct icp_qat_uclo_encapme { - struct icp_qat_uof_image *img_ptr; - struct icp_qat_uclo_encap_page *page; - unsigned int ae_reg_num; - struct icp_qat_uof_ae_reg *ae_reg; - unsigned int init_regsym_num; - struct icp_qat_uof_init_regsym *init_regsym; - unsigned int sbreak_num; - struct icp_qat_uof_sbreak *sbreak; - unsigned int uwords_num; -}; - -struct icp_qat_uclo_init_mem_table { - unsigned int entry_num; - struct icp_qat_uof_initmem *init_mem; -}; - -struct icp_qat_uclo_objhdr { - char *file_buff; - unsigned int checksum; - unsigned int size; -}; - -struct icp_qat_uof_strtable { - unsigned int table_len; - unsigned int reserved; - u64 strings; -}; - -struct icp_qat_uclo_objhandle { - unsigned int prod_type; - unsigned int prod_rev; - struct icp_qat_uclo_objhdr *obj_hdr; - struct icp_qat_uof_encap_obj encap_uof_obj; - struct icp_qat_uof_strtable str_table; - struct icp_qat_uclo_encapme ae_uimage[ICP_QAT_UCLO_MAX_UIMAGE]; - struct icp_qat_uclo_aedata ae_data[ICP_QAT_UCLO_MAX_AE]; - struct icp_qat_uclo_init_mem_table init_mem_tab; - struct icp_qat_uof_batch_init *lm_init_tab[ICP_QAT_UCLO_MAX_AE]; - struct icp_qat_uof_batch_init *umem_init_tab[ICP_QAT_UCLO_MAX_AE]; - int uimage_num; - int uword_in_bytes; - int global_inited; - unsigned int ae_num; - unsigned int ustore_phy_size; - void *obj_buf; - u64 *uword_buf; -}; - -struct icp_qat_uof_uword_block { - unsigned int start_addr; - unsigned int words_num; - unsigned int uword_offset; - unsigned int reserved; -}; - -struct icp_qat_uof_filehdr { - unsigned short file_id; - unsigned short reserved1; - char min_ver; - char maj_ver; - unsigned short reserved2; - unsigned short max_chunks; - unsigned short num_chunks; -}; - -struct icp_qat_uof_filechunkhdr { - char chunk_id[ICP_QAT_UOF_OBJID_LEN]; - unsigned int checksum; - unsigned int offset; - unsigned int size; -}; - -struct icp_qat_uof_objhdr { - unsigned int ac_dev_type; - unsigned short min_cpu_ver; - unsigned short max_cpu_ver; - short max_chunks; - short num_chunks; - unsigned int reserved1; - unsigned int reserved2; -}; - -struct icp_qat_uof_chunkhdr { - char chunk_id[ICP_QAT_UOF_OBJID_LEN]; - unsigned int offset; - unsigned int size; -}; - -struct icp_qat_uof_memvar_attr { - unsigned int offset_in_byte; - unsigned int value; -}; - -struct icp_qat_uof_initmem { - unsigned int sym_name; - char region; - char scope; - unsigned short reserved1; - unsigned int addr; - unsigned int num_in_bytes; - unsigned int val_attr_num; -}; - -struct icp_qat_uof_init_regsym { - unsigned int sym_name; - char init_type; - char value_type; - char reg_type; - unsigned char ctx; - unsigned int reg_addr; - unsigned int value; -}; - -struct icp_qat_uof_varmem_seg { - unsigned int sram_base; - unsigned int sram_size; - unsigned int sram_alignment; - unsigned int sdram_base; - unsigned int sdram_size; - unsigned int sdram_alignment; - unsigned int sdram1_base; - unsigned int sdram1_size; - unsigned int sdram1_alignment; - unsigned int scratch_base; - unsigned int scratch_size; - unsigned int scratch_alignment; -}; - -struct icp_qat_uof_gtid { - char tool_id[ICP_QAT_UOF_OBJID_LEN]; - int tool_ver; - unsigned int reserved1; - unsigned int reserved2; -}; - -struct icp_qat_uof_sbreak { - unsigned int page_num; - unsigned int virt_uaddr; - unsigned char sbreak_type; - unsigned char reg_type; - unsigned short reserved1; - unsigned int addr_offset; - unsigned int reg_addr; -}; - -struct icp_qat_uof_code_page { - unsigned int page_region; - unsigned int page_num; - unsigned char def_page; - unsigned char reserved2; - unsigned short reserved1; - unsigned int beg_addr_v; - unsigned int beg_addr_p; - unsigned int neigh_reg_tab_offset; - unsigned int uc_var_tab_offset; - unsigned int imp_var_tab_offset; - unsigned int imp_expr_tab_offset; - unsigned int code_area_offset; -}; - -struct icp_qat_uof_image { - unsigned int img_name; - unsigned int ae_assigned; - unsigned int ctx_assigned; - unsigned int ac_dev_type; - unsigned int entry_address; - unsigned int fill_pattern[2]; - unsigned int reloadable_size; - unsigned char sensitivity; - unsigned char reserved; - unsigned short ae_mode; - unsigned short max_ver; - unsigned short min_ver; - unsigned short image_attrib; - unsigned short reserved2; - unsigned short page_region_num; - unsigned short numpages; - unsigned int reg_tab_offset; - unsigned int init_reg_sym_tab; - unsigned int sbreak_tab; - unsigned int app_metadata; -}; - -struct icp_qat_uof_objtable { - unsigned int entry_num; -}; - -struct icp_qat_uof_ae_reg { - unsigned int name; - unsigned int vis_name; - unsigned short type; - unsigned short addr; - unsigned short access_mode; - unsigned char visible; - unsigned char reserved1; - unsigned short ref_count; - unsigned short reserved2; - unsigned int xo_id; -}; - -struct icp_qat_uof_code_area { - unsigned int micro_words_num; - unsigned int uword_block_tab; -}; - -struct icp_qat_uof_batch_init { - unsigned int ae; - unsigned int addr; - unsigned int *value; - unsigned int size; - struct icp_qat_uof_batch_init *next; -}; - -struct icp_qat_suof_img_hdr { - char *simg_buf; - unsigned long simg_len; - char *css_header; - char *css_key; - char *css_signature; - char *css_simg; - unsigned long simg_size; - unsigned int ae_num; - unsigned int ae_mask; - unsigned int fw_type; - unsigned long simg_name; - unsigned long appmeta_data; -}; - -struct icp_qat_suof_img_tbl { - unsigned int num_simgs; - struct icp_qat_suof_img_hdr *simg_hdr; -}; - -struct icp_qat_suof_handle { - unsigned int file_id; - unsigned int check_sum; - char min_ver; - char maj_ver; - char fw_type; - char *suof_buf; - unsigned int suof_size; - char *sym_str; - unsigned int sym_size; - struct icp_qat_suof_img_tbl img_table; -}; - -struct icp_qat_fw_auth_desc { - unsigned int img_len; - unsigned int ae_mask; - unsigned int css_hdr_high; - unsigned int css_hdr_low; - unsigned int img_high; - unsigned int img_low; - unsigned int signature_high; - unsigned int signature_low; - unsigned int fwsk_pub_high; - unsigned int fwsk_pub_low; - unsigned int img_ae_mode_data_high; - unsigned int img_ae_mode_data_low; - unsigned int img_ae_init_data_high; - unsigned int img_ae_init_data_low; - unsigned int img_ae_insts_high; - unsigned int img_ae_insts_low; -}; - -struct icp_qat_auth_chunk { - struct icp_qat_fw_auth_desc fw_auth_desc; - u64 chunk_size; - u64 chunk_bus_addr; -}; - -struct icp_qat_css_hdr { - unsigned int module_type; - unsigned int header_len; - unsigned int header_ver; - unsigned int module_id; - unsigned int module_vendor; - unsigned int date; - unsigned int size; - unsigned int key_size; - unsigned int module_size; - unsigned int exponent_size; - unsigned int fw_type; - unsigned int reserved[21]; -}; - -struct icp_qat_simg_ae_mode { - unsigned int file_id; - unsigned short maj_ver; - unsigned short min_ver; - unsigned int dev_type; - unsigned short devmax_ver; - unsigned short devmin_ver; - unsigned int ae_mask; - unsigned int ctx_enables; - char fw_type; - char ctx_mode; - char nn_mode; - char lm0_mode; - char lm1_mode; - char scs_mode; - char lm2_mode; - char lm3_mode; - char tindex_mode; - unsigned char reserved[7]; - char simg_name[256]; - char appmeta_data[256]; -}; - -struct icp_qat_suof_filehdr { - unsigned int file_id; - unsigned int check_sum; - char min_ver; - char maj_ver; - char fw_type; - char reserved; - unsigned short max_chunks; - unsigned short num_chunks; -}; - -struct icp_qat_suof_chunk_hdr { - char chunk_id[ICP_QAT_SUOF_OBJ_ID_LEN]; - u64 offset; - u64 size; -}; - -struct icp_qat_suof_strtable { - unsigned int tab_length; - unsigned int strings; -}; - -struct icp_qat_suof_objhdr { - unsigned int img_length; - unsigned int reserved; -}; - -struct icp_qat_mof_file_hdr { - unsigned int file_id; - unsigned int checksum; - char min_ver; - char maj_ver; - unsigned short reserved; - unsigned short max_chunks; - unsigned short num_chunks; -}; - -struct icp_qat_mof_chunkhdr { - char chunk_id[ICP_QAT_MOF_OBJ_ID_LEN]; - u64 offset; - u64 size; -}; - -struct icp_qat_mof_str_table { - unsigned int tab_len; - unsigned int strings; -}; - -struct icp_qat_mof_obj_hdr { - unsigned short max_chunks; - unsigned short num_chunks; - unsigned int reserved; -}; - -struct icp_qat_mof_obj_chunkhdr { - char chunk_id[ICP_QAT_MOF_OBJ_CHUNKID_LEN]; - u64 offset; - u64 size; - unsigned int name; - unsigned int reserved; -}; - -struct icp_qat_mof_objhdr { - char *obj_name; - char *obj_buf; - unsigned int obj_size; -}; - -struct icp_qat_mof_table { - unsigned int num_objs; - struct icp_qat_mof_objhdr *obj_hdr; -}; - -struct icp_qat_mof_handle { - unsigned int file_id; - unsigned int checksum; - char min_ver; - char maj_ver; - char *mof_buf; - u32 mof_size; - char *sym_str; - unsigned int sym_size; - char *uobjs_hdr; - char *sobjs_hdr; - struct icp_qat_mof_table obj_table; -}; -#endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c deleted file mode 100644 index 538dcbfbcd26..000000000000 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ /dev/null @@ -1,1424 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "qat_algs_send.h" -#include "adf_common_drv.h" -#include "qat_crypto.h" -#include "icp_qat_hw.h" -#include "icp_qat_fw.h" -#include "icp_qat_fw_la.h" -#include "qat_bl.h" - -#define QAT_AES_HW_CONFIG_ENC(alg, mode) \ - ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ - ICP_QAT_HW_CIPHER_NO_CONVERT, \ - ICP_QAT_HW_CIPHER_ENCRYPT) - -#define QAT_AES_HW_CONFIG_DEC(alg, mode) \ - ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ - ICP_QAT_HW_CIPHER_KEY_CONVERT, \ - ICP_QAT_HW_CIPHER_DECRYPT) - -#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \ - ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ - ICP_QAT_HW_CIPHER_NO_CONVERT, \ - ICP_QAT_HW_CIPHER_DECRYPT) - -#define HW_CAP_AES_V2(accel_dev) \ - (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ - ICP_ACCEL_CAPABILITIES_AES_V2) - -static DEFINE_MUTEX(algs_lock); -static unsigned int active_devs; - -/* Common content descriptor */ -struct qat_alg_cd { - union { - struct qat_enc { /* Encrypt content desc */ - struct icp_qat_hw_cipher_algo_blk cipher; - struct icp_qat_hw_auth_algo_blk hash; - } qat_enc_cd; - struct qat_dec { /* Decrypt content desc */ - struct icp_qat_hw_auth_algo_blk hash; - struct icp_qat_hw_cipher_algo_blk cipher; - } qat_dec_cd; - }; -} __aligned(64); - -struct qat_alg_aead_ctx { - struct qat_alg_cd *enc_cd; - struct qat_alg_cd *dec_cd; - dma_addr_t enc_cd_paddr; - dma_addr_t dec_cd_paddr; - struct icp_qat_fw_la_bulk_req enc_fw_req; - struct icp_qat_fw_la_bulk_req dec_fw_req; - struct crypto_shash *hash_tfm; - enum icp_qat_hw_auth_algo qat_hash_alg; - struct qat_crypto_instance *inst; - union { - struct sha1_state sha1; - struct sha256_state sha256; - struct sha512_state sha512; - }; - char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */ - char opad[SHA512_BLOCK_SIZE]; -}; - -struct qat_alg_skcipher_ctx { - struct icp_qat_hw_cipher_algo_blk *enc_cd; - struct icp_qat_hw_cipher_algo_blk *dec_cd; - dma_addr_t enc_cd_paddr; - dma_addr_t dec_cd_paddr; - struct icp_qat_fw_la_bulk_req enc_fw_req; - struct icp_qat_fw_la_bulk_req dec_fw_req; - struct qat_crypto_instance *inst; - struct crypto_skcipher *ftfm; - struct crypto_cipher *tweak; - bool fallback; - int mode; -}; - -static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) -{ - switch (qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - return ICP_QAT_HW_SHA1_STATE1_SZ; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - return ICP_QAT_HW_SHA256_STATE1_SZ; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - return ICP_QAT_HW_SHA512_STATE1_SZ; - default: - return -EFAULT; - } - return -EFAULT; -} - -static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, - struct qat_alg_aead_ctx *ctx, - const u8 *auth_key, - unsigned int auth_keylen) -{ - SHASH_DESC_ON_STACK(shash, ctx->hash_tfm); - int block_size = crypto_shash_blocksize(ctx->hash_tfm); - int digest_size = crypto_shash_digestsize(ctx->hash_tfm); - __be32 *hash_state_out; - __be64 *hash512_state_out; - int i, offset; - - memset(ctx->ipad, 0, block_size); - memset(ctx->opad, 0, block_size); - shash->tfm = ctx->hash_tfm; - - if (auth_keylen > block_size) { - int ret = crypto_shash_digest(shash, auth_key, - auth_keylen, ctx->ipad); - if (ret) - return ret; - - memcpy(ctx->opad, ctx->ipad, digest_size); - } else { - memcpy(ctx->ipad, auth_key, auth_keylen); - memcpy(ctx->opad, auth_key, auth_keylen); - } - - for (i = 0; i < block_size; i++) { - char *ipad_ptr = ctx->ipad + i; - char *opad_ptr = ctx->opad + i; - *ipad_ptr ^= HMAC_IPAD_VALUE; - *opad_ptr ^= HMAC_OPAD_VALUE; - } - - if (crypto_shash_init(shash)) - return -EFAULT; - - if (crypto_shash_update(shash, ctx->ipad, block_size)) - return -EFAULT; - - hash_state_out = (__be32 *)hash->sha.state1; - hash512_state_out = (__be64 *)hash_state_out; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - if (crypto_shash_export(shash, &ctx->sha1)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha1.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - if (crypto_shash_export(shash, &ctx->sha256)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha256.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - if (crypto_shash_export(shash, &ctx->sha512)) - return -EFAULT; - for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) - *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]); - break; - default: - return -EFAULT; - } - - if (crypto_shash_init(shash)) - return -EFAULT; - - if (crypto_shash_update(shash, ctx->opad, block_size)) - return -EFAULT; - - offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); - if (offset < 0) - return -EFAULT; - - hash_state_out = (__be32 *)(hash->sha.state1 + offset); - hash512_state_out = (__be64 *)hash_state_out; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - if (crypto_shash_export(shash, &ctx->sha1)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha1.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - if (crypto_shash_export(shash, &ctx->sha256)) - return -EFAULT; - for (i = 0; i < digest_size >> 2; i++, hash_state_out++) - *hash_state_out = cpu_to_be32(ctx->sha256.state[i]); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - if (crypto_shash_export(shash, &ctx->sha512)) - return -EFAULT; - for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) - *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]); - break; - default: - return -EFAULT; - } - memzero_explicit(ctx->ipad, block_size); - memzero_explicit(ctx->opad, block_size); - return 0; -} - -static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) -{ - header->hdr_flags = - ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); - header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA; - header->comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, - QAT_COMN_PTR_TYPE_SGL); - ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_PARTIAL_NONE); - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_16BYTE_DATA); - ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_PROTO); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); -} - -static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, - int alg, - struct crypto_authenc_keys *keys, - int mode) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); - unsigned int digestsize = crypto_aead_authsize(aead_tfm); - struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd; - struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher; - struct icp_qat_hw_auth_algo_blk *hash = - (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx + - sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen); - struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req; - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - void *ptr = &req_tmpl->cd_ctrl; - struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; - struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; - - /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); - memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); - hash->sha.inner_setup.auth_config.config = - ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, - ctx->qat_hash_alg, digestsize); - hash->sha.inner_setup.auth_counter.counter = - cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); - - if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) - return -EFAULT; - - /* Request setup */ - qat_alg_init_common_hdr(header); - header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); - cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; - cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; - - /* Cipher CD config setup */ - cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; - cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; - cipher_cd_ctrl->cipher_cfg_offset = 0; - ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); - /* Auth CD config setup */ - hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3; - hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; - hash_cd_ctrl->inner_res_sz = digestsize; - hash_cd_ctrl->final_sz = digestsize; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - hash_cd_ctrl->inner_state1_sz = - round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); - hash_cd_ctrl->inner_state2_sz = - round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; - break; - default: - break; - } - hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + - ((sizeof(struct icp_qat_hw_auth_setup) + - round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); - ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); - return 0; -} - -static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, - int alg, - struct crypto_authenc_keys *keys, - int mode) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); - unsigned int digestsize = crypto_aead_authsize(aead_tfm); - struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd; - struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash; - struct icp_qat_hw_cipher_algo_blk *cipher = - (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + - sizeof(struct icp_qat_hw_auth_setup) + - roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); - struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - void *ptr = &req_tmpl->cd_ctrl; - struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; - struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; - struct icp_qat_fw_la_auth_req_params *auth_param = - (struct icp_qat_fw_la_auth_req_params *) - ((char *)&req_tmpl->serv_specif_rqpars + - sizeof(struct icp_qat_fw_la_cipher_req_params)); - - /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode); - memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); - hash->sha.inner_setup.auth_config.config = - ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, - ctx->qat_hash_alg, - digestsize); - hash->sha.inner_setup.auth_counter.counter = - cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); - - if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) - return -EFAULT; - - /* Request setup */ - qat_alg_init_common_hdr(header); - header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_CMP_AUTH_RES); - cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; - cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; - - /* Cipher CD config setup */ - cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; - cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; - cipher_cd_ctrl->cipher_cfg_offset = - (sizeof(struct icp_qat_hw_auth_setup) + - roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3; - ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); - - /* Auth CD config setup */ - hash_cd_ctrl->hash_cfg_offset = 0; - hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; - hash_cd_ctrl->inner_res_sz = digestsize; - hash_cd_ctrl->final_sz = digestsize; - - switch (ctx->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - hash_cd_ctrl->inner_state1_sz = - round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); - hash_cd_ctrl->inner_state2_sz = - round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; - break; - default: - break; - } - - hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + - ((sizeof(struct icp_qat_hw_auth_setup) + - round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); - auth_param->auth_res_sz = digestsize; - ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); - return 0; -} - -static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, - struct icp_qat_fw_la_bulk_req *req, - struct icp_qat_hw_cipher_algo_blk *cd, - const u8 *key, unsigned int keylen) -{ - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; - struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; - bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); - int mode = ctx->mode; - - qat_alg_init_common_hdr(header); - header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; - cd_pars->u.s.content_desc_params_sz = - sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; - - if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { - ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); - - /* Store both XTS keys in CD, only the first key is sent - * to the HW, the second key is used for tweak calculation - */ - memcpy(cd->ucs_aes.key, key, keylen); - keylen = keylen / 2; - } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { - ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); - memcpy(cd->ucs_aes.key, key, keylen); - keylen = round_up(keylen, 16); - } else { - memcpy(cd->aes.key, key, keylen); - } - - /* Cipher CD config setup */ - cd_ctrl->cipher_key_sz = keylen >> 3; - cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; - cd_ctrl->cipher_cfg_offset = 0; - ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); -} - -static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx, - int alg, const u8 *key, - unsigned int keylen, int mode) -{ - struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd; - struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req; - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - - qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen); - cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; - enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); -} - -static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, - u8 *key_reverse) -{ - struct crypto_aes_ctx aes_expanded; - int nrounds; - u8 *key; - - aes_expandkey(&aes_expanded, key_forward, keylen); - if (keylen == AES_KEYSIZE_128) { - nrounds = 10; - key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); - memcpy(key_reverse, key, AES_BLOCK_SIZE); - } else { - /* AES_KEYSIZE_256 */ - nrounds = 14; - key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); - memcpy(key_reverse, key, AES_BLOCK_SIZE); - memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - } -} - -static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, - int alg, const u8 *key, - unsigned int keylen, int mode) -{ - struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; - struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); - - qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen); - cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; - - if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { - /* Key reversing not supported, set no convert */ - dec_cd->aes.cipher_config.val = - QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode); - - /* In-place key reversal */ - qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2, - dec_cd->ucs_aes.key); - } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) { - dec_cd->aes.cipher_config.val = - QAT_AES_HW_CONFIG_DEC(alg, mode); - } else { - dec_cd->aes.cipher_config.val = - QAT_AES_HW_CONFIG_ENC(alg, mode); - } -} - -static int qat_alg_validate_key(int key_len, int *alg, int mode) -{ - if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) { - switch (key_len) { - case AES_KEYSIZE_128: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; - break; - case AES_KEYSIZE_192: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; - break; - case AES_KEYSIZE_256: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; - break; - default: - return -EINVAL; - } - } else { - switch (key_len) { - case AES_KEYSIZE_128 << 1: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; - break; - case AES_KEYSIZE_256 << 1: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; - break; - default: - return -EINVAL; - } - } - return 0; -} - -static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen, int mode) -{ - struct crypto_authenc_keys keys; - int alg; - - if (crypto_authenc_extractkeys(&keys, key, keylen)) - goto bad_key; - - if (qat_alg_validate_key(keys.enckeylen, &alg, mode)) - goto bad_key; - - if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode)) - goto error; - - if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode)) - goto error; - - memzero_explicit(&keys, sizeof(keys)); - return 0; -bad_key: - memzero_explicit(&keys, sizeof(keys)); - return -EINVAL; -error: - memzero_explicit(&keys, sizeof(keys)); - return -EFAULT; -} - -static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx, - const u8 *key, - unsigned int keylen, - int mode) -{ - int alg; - - if (qat_alg_validate_key(keylen, &alg, mode)) - return -EINVAL; - - qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode); - qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode); - return 0; -} - -static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); - memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); - - return qat_alg_aead_init_sessions(tfm, key, keylen, - ICP_QAT_HW_CIPHER_CBC_MODE); -} - -static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct qat_crypto_instance *inst = NULL; - int node = numa_node_id(); - struct device *dev; - int ret; - - inst = qat_crypto_get_instance_node(node); - if (!inst) - return -EINVAL; - dev = &GET_DEV(inst->accel_dev); - ctx->inst = inst; - ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), - &ctx->enc_cd_paddr, - GFP_ATOMIC); - if (!ctx->enc_cd) { - ret = -ENOMEM; - goto out_free_inst; - } - ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), - &ctx->dec_cd_paddr, - GFP_ATOMIC); - if (!ctx->dec_cd) { - ret = -ENOMEM; - goto out_free_enc; - } - - ret = qat_alg_aead_init_sessions(tfm, key, keylen, - ICP_QAT_HW_CIPHER_CBC_MODE); - if (ret) - goto out_free_all; - - return 0; - -out_free_all: - memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); - dma_free_coherent(dev, sizeof(struct qat_alg_cd), - ctx->dec_cd, ctx->dec_cd_paddr); - ctx->dec_cd = NULL; -out_free_enc: - memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); - dma_free_coherent(dev, sizeof(struct qat_alg_cd), - ctx->enc_cd, ctx->enc_cd_paddr); - ctx->enc_cd = NULL; -out_free_inst: - ctx->inst = NULL; - qat_crypto_put_instance(inst); - return ret; -} - -static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - - if (ctx->enc_cd) - return qat_alg_aead_rekey(tfm, key, keylen); - else - return qat_alg_aead_newkey(tfm, key, keylen); -} - -static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, - struct qat_crypto_request *qat_req) -{ - struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx; - struct qat_crypto_instance *inst = ctx->inst; - struct aead_request *areq = qat_req->aead_req; - u8 stat_filed = qat_resp->comn_resp.comn_status; - int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); - - qat_bl_free_bufl(inst->accel_dev, &qat_req->buf); - if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) - res = -EBADMSG; - aead_request_complete(areq, res); -} - -static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req) -{ - struct skcipher_request *sreq = qat_req->skcipher_req; - u64 iv_lo_prev; - u64 iv_lo; - u64 iv_hi; - - memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE); - - iv_lo = be64_to_cpu(qat_req->iv_lo); - iv_hi = be64_to_cpu(qat_req->iv_hi); - - iv_lo_prev = iv_lo; - iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE); - if (iv_lo < iv_lo_prev) - iv_hi++; - - qat_req->iv_lo = cpu_to_be64(iv_lo); - qat_req->iv_hi = cpu_to_be64(iv_hi); -} - -static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req) -{ - struct skcipher_request *sreq = qat_req->skcipher_req; - int offset = sreq->cryptlen - AES_BLOCK_SIZE; - struct scatterlist *sgl; - - if (qat_req->encryption) - sgl = sreq->dst; - else - sgl = sreq->src; - - scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0); -} - -static void qat_alg_update_iv(struct qat_crypto_request *qat_req) -{ - struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - - switch (ctx->mode) { - case ICP_QAT_HW_CIPHER_CTR_MODE: - qat_alg_update_iv_ctr_mode(qat_req); - break; - case ICP_QAT_HW_CIPHER_CBC_MODE: - qat_alg_update_iv_cbc_mode(qat_req); - break; - case ICP_QAT_HW_CIPHER_XTS_MODE: - break; - default: - dev_warn(dev, "Unsupported IV update for cipher mode %d\n", - ctx->mode); - } -} - -static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, - struct qat_crypto_request *qat_req) -{ - struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; - struct qat_crypto_instance *inst = ctx->inst; - struct skcipher_request *sreq = qat_req->skcipher_req; - u8 stat_filed = qat_resp->comn_resp.comn_status; - int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); - - qat_bl_free_bufl(inst->accel_dev, &qat_req->buf); - if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) - res = -EINVAL; - - if (qat_req->encryption) - qat_alg_update_iv(qat_req); - - memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE); - - skcipher_request_complete(sreq, res); -} - -void qat_alg_callback(void *resp) -{ - struct icp_qat_fw_la_resp *qat_resp = resp; - struct qat_crypto_request *qat_req = - (void *)(__force long)qat_resp->opaque_data; - struct qat_instance_backlog *backlog = qat_req->alg_req.backlog; - - qat_req->cb(qat_resp, qat_req); - - qat_alg_send_backlog(backlog); -} - -static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req, - struct qat_crypto_instance *inst, - struct crypto_async_request *base) -{ - struct qat_alg_req *alg_req = &qat_req->alg_req; - - alg_req->fw_req = (u32 *)&qat_req->req; - alg_req->tx_ring = inst->sym_tx; - alg_req->base = base; - alg_req->backlog = &inst->backlog; - - return qat_alg_send_message(alg_req); -} - -static int qat_alg_aead_dec(struct aead_request *areq) -{ - struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); - struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); - struct qat_crypto_request *qat_req = aead_request_ctx(areq); - struct icp_qat_fw_la_cipher_req_params *cipher_param; - struct icp_qat_fw_la_auth_req_params *auth_param; - struct icp_qat_fw_la_bulk_req *msg; - int digst_size = crypto_aead_authsize(aead_tfm); - gfp_t f = qat_algs_alloc_flags(&areq->base); - int ret; - u32 cipher_len; - - cipher_len = areq->cryptlen - digst_size; - if (cipher_len % AES_BLOCK_SIZE != 0) - return -EINVAL; - - ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst, - &qat_req->buf, NULL, f); - if (unlikely(ret)) - return ret; - - msg = &qat_req->req; - *msg = ctx->dec_fw_req; - qat_req->aead_ctx = ctx; - qat_req->aead_req = areq; - qat_req->cb = qat_aead_alg_callback; - qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; - qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; - qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; - cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - cipher_param->cipher_length = cipher_len; - cipher_param->cipher_offset = areq->assoclen; - memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); - auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param)); - auth_param->auth_off = 0; - auth_param->auth_len = areq->assoclen + cipher_param->cipher_length; - - ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base); - if (ret == -ENOSPC) - qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); - - return ret; -} - -static int qat_alg_aead_enc(struct aead_request *areq) -{ - struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); - struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); - struct qat_crypto_request *qat_req = aead_request_ctx(areq); - struct icp_qat_fw_la_cipher_req_params *cipher_param; - struct icp_qat_fw_la_auth_req_params *auth_param; - gfp_t f = qat_algs_alloc_flags(&areq->base); - struct icp_qat_fw_la_bulk_req *msg; - u8 *iv = areq->iv; - int ret; - - if (areq->cryptlen % AES_BLOCK_SIZE != 0) - return -EINVAL; - - ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst, - &qat_req->buf, NULL, f); - if (unlikely(ret)) - return ret; - - msg = &qat_req->req; - *msg = ctx->enc_fw_req; - qat_req->aead_ctx = ctx; - qat_req->aead_req = areq; - qat_req->cb = qat_aead_alg_callback; - qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; - qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; - qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; - cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param)); - - memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); - cipher_param->cipher_length = areq->cryptlen; - cipher_param->cipher_offset = areq->assoclen; - - auth_param->auth_off = 0; - auth_param->auth_len = areq->assoclen + areq->cryptlen; - - ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base); - if (ret == -ENOSPC) - qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); - - return ret; -} - -static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx, - const u8 *key, unsigned int keylen, - int mode) -{ - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); - memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); - - return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode); -} - -static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx, - const u8 *key, unsigned int keylen, - int mode) -{ - struct qat_crypto_instance *inst = NULL; - struct device *dev; - int node = numa_node_id(); - int ret; - - inst = qat_crypto_get_instance_node(node); - if (!inst) - return -EINVAL; - dev = &GET_DEV(inst->accel_dev); - ctx->inst = inst; - ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), - &ctx->enc_cd_paddr, - GFP_ATOMIC); - if (!ctx->enc_cd) { - ret = -ENOMEM; - goto out_free_instance; - } - ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), - &ctx->dec_cd_paddr, - GFP_ATOMIC); - if (!ctx->dec_cd) { - ret = -ENOMEM; - goto out_free_enc; - } - - ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode); - if (ret) - goto out_free_all; - - return 0; - -out_free_all: - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - dma_free_coherent(dev, sizeof(*ctx->dec_cd), - ctx->dec_cd, ctx->dec_cd_paddr); - ctx->dec_cd = NULL; -out_free_enc: - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - dma_free_coherent(dev, sizeof(*ctx->enc_cd), - ctx->enc_cd, ctx->enc_cd_paddr); - ctx->enc_cd = NULL; -out_free_instance: - ctx->inst = NULL; - qat_crypto_put_instance(inst); - return ret; -} - -static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen, - int mode) -{ - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); - - ctx->mode = mode; - - if (ctx->enc_cd) - return qat_alg_skcipher_rekey(ctx, key, keylen, mode); - else - return qat_alg_skcipher_newkey(ctx, key, keylen, mode); -} - -static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen) -{ - return qat_alg_skcipher_setkey(tfm, key, keylen, - ICP_QAT_HW_CIPHER_CBC_MODE); -} - -static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen) -{ - return qat_alg_skcipher_setkey(tfm, key, keylen, - ICP_QAT_HW_CIPHER_CTR_MODE); -} - -static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen) -{ - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); - int ret; - - ret = xts_verify_key(tfm, key, keylen); - if (ret) - return ret; - - if (keylen >> 1 == AES_KEYSIZE_192) { - ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen); - if (ret) - return ret; - - ctx->fallback = true; - - return 0; - } - - ctx->fallback = false; - - ret = qat_alg_skcipher_setkey(tfm, key, keylen, - ICP_QAT_HW_CIPHER_XTS_MODE); - if (ret) - return ret; - - if (HW_CAP_AES_V2(ctx->inst->accel_dev)) - ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2), - keylen / 2); - - return ret; -} - -static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req) -{ - struct icp_qat_fw_la_cipher_req_params *cipher_param; - struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; - bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); - u8 *iv = qat_req->skcipher_req->iv; - - cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - - if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE) - crypto_cipher_encrypt_one(ctx->tweak, - (u8 *)cipher_param->u.cipher_IV_array, - iv); - else - memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); -} - -static int qat_alg_skcipher_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); - struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); - struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct qat_crypto_request *qat_req = skcipher_request_ctx(req); - struct icp_qat_fw_la_cipher_req_params *cipher_param; - gfp_t f = qat_algs_alloc_flags(&req->base); - struct icp_qat_fw_la_bulk_req *msg; - int ret; - - if (req->cryptlen == 0) - return 0; - - ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst, - &qat_req->buf, NULL, f); - if (unlikely(ret)) - return ret; - - msg = &qat_req->req; - *msg = ctx->enc_fw_req; - qat_req->skcipher_ctx = ctx; - qat_req->skcipher_req = req; - qat_req->cb = qat_skcipher_alg_callback; - qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; - qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; - qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; - qat_req->encryption = true; - cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - cipher_param->cipher_length = req->cryptlen; - cipher_param->cipher_offset = 0; - - qat_alg_set_req_iv(qat_req); - - ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base); - if (ret == -ENOSPC) - qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); - - return ret; -} - -static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req) -{ - if (req->cryptlen % AES_BLOCK_SIZE != 0) - return -EINVAL; - - return qat_alg_skcipher_encrypt(req); -} - -static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm); - struct skcipher_request *nreq = skcipher_request_ctx(req); - - if (req->cryptlen < XTS_BLOCK_SIZE) - return -EINVAL; - - if (ctx->fallback) { - memcpy(nreq, req, sizeof(*req)); - skcipher_request_set_tfm(nreq, ctx->ftfm); - return crypto_skcipher_encrypt(nreq); - } - - return qat_alg_skcipher_encrypt(req); -} - -static int qat_alg_skcipher_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); - struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); - struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct qat_crypto_request *qat_req = skcipher_request_ctx(req); - struct icp_qat_fw_la_cipher_req_params *cipher_param; - gfp_t f = qat_algs_alloc_flags(&req->base); - struct icp_qat_fw_la_bulk_req *msg; - int ret; - - if (req->cryptlen == 0) - return 0; - - ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst, - &qat_req->buf, NULL, f); - if (unlikely(ret)) - return ret; - - msg = &qat_req->req; - *msg = ctx->dec_fw_req; - qat_req->skcipher_ctx = ctx; - qat_req->skcipher_req = req; - qat_req->cb = qat_skcipher_alg_callback; - qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; - qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; - qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; - qat_req->encryption = false; - cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - cipher_param->cipher_length = req->cryptlen; - cipher_param->cipher_offset = 0; - - qat_alg_set_req_iv(qat_req); - qat_alg_update_iv(qat_req); - - ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base); - if (ret == -ENOSPC) - qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); - - return ret; -} - -static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req) -{ - if (req->cryptlen % AES_BLOCK_SIZE != 0) - return -EINVAL; - - return qat_alg_skcipher_decrypt(req); -} - -static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm); - struct skcipher_request *nreq = skcipher_request_ctx(req); - - if (req->cryptlen < XTS_BLOCK_SIZE) - return -EINVAL; - - if (ctx->fallback) { - memcpy(nreq, req, sizeof(*req)); - skcipher_request_set_tfm(nreq, ctx->ftfm); - return crypto_skcipher_decrypt(nreq); - } - - return qat_alg_skcipher_decrypt(req); -} - -static int qat_alg_aead_init(struct crypto_aead *tfm, - enum icp_qat_hw_auth_algo hash, - const char *hash_name) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - - ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); - if (IS_ERR(ctx->hash_tfm)) - return PTR_ERR(ctx->hash_tfm); - ctx->qat_hash_alg = hash; - crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request)); - return 0; -} - -static int qat_alg_aead_sha1_init(struct crypto_aead *tfm) -{ - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); -} - -static int qat_alg_aead_sha256_init(struct crypto_aead *tfm) -{ - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); -} - -static int qat_alg_aead_sha512_init(struct crypto_aead *tfm) -{ - return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); -} - -static void qat_alg_aead_exit(struct crypto_aead *tfm) -{ - struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev; - - crypto_free_shash(ctx->hash_tfm); - - if (!inst) - return; - - dev = &GET_DEV(inst->accel_dev); - if (ctx->enc_cd) { - memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); - dma_free_coherent(dev, sizeof(struct qat_alg_cd), - ctx->enc_cd, ctx->enc_cd_paddr); - } - if (ctx->dec_cd) { - memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); - dma_free_coherent(dev, sizeof(struct qat_alg_cd), - ctx->dec_cd, ctx->dec_cd_paddr); - } - qat_crypto_put_instance(inst); -} - -static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm) -{ - crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request)); - return 0; -} - -static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm) -{ - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); - int reqsize; - - ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->ftfm)) - return PTR_ERR(ctx->ftfm); - - ctx->tweak = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(ctx->tweak)) { - crypto_free_skcipher(ctx->ftfm); - return PTR_ERR(ctx->tweak); - } - - reqsize = max(sizeof(struct qat_crypto_request), - sizeof(struct skcipher_request) + - crypto_skcipher_reqsize(ctx->ftfm)); - crypto_skcipher_set_reqsize(tfm, reqsize); - - return 0; -} - -static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm) -{ - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev; - - if (!inst) - return; - - dev = &GET_DEV(inst->accel_dev); - if (ctx->enc_cd) { - memset(ctx->enc_cd, 0, - sizeof(struct icp_qat_hw_cipher_algo_blk)); - dma_free_coherent(dev, - sizeof(struct icp_qat_hw_cipher_algo_blk), - ctx->enc_cd, ctx->enc_cd_paddr); - } - if (ctx->dec_cd) { - memset(ctx->dec_cd, 0, - sizeof(struct icp_qat_hw_cipher_algo_blk)); - dma_free_coherent(dev, - sizeof(struct icp_qat_hw_cipher_algo_blk), - ctx->dec_cd, ctx->dec_cd_paddr); - } - qat_crypto_put_instance(inst); -} - -static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm) -{ - struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); - - if (ctx->ftfm) - crypto_free_skcipher(ctx->ftfm); - - if (ctx->tweak) - crypto_free_cipher(ctx->tweak); - - qat_alg_skcipher_exit_tfm(tfm); -} - -static struct aead_alg qat_aeads[] = { { - .base = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha1", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_module = THIS_MODULE, - }, - .init = qat_alg_aead_sha1_init, - .exit = qat_alg_aead_exit, - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, -}, { - .base = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha256", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_module = THIS_MODULE, - }, - .init = qat_alg_aead_sha256_init, - .exit = qat_alg_aead_exit, - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, -}, { - .base = { - .cra_name = "authenc(hmac(sha512),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha512", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_module = THIS_MODULE, - }, - .init = qat_alg_aead_sha512_init, - .exit = qat_alg_aead_exit, - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, -} }; - -static struct skcipher_alg qat_skciphers[] = { { - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "qat_aes_cbc", - .base.cra_priority = 4001, - .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), - .base.cra_alignmask = 0, - .base.cra_module = THIS_MODULE, - - .init = qat_alg_skcipher_init_tfm, - .exit = qat_alg_skcipher_exit_tfm, - .setkey = qat_alg_skcipher_cbc_setkey, - .decrypt = qat_alg_skcipher_blk_decrypt, - .encrypt = qat_alg_skcipher_blk_encrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, -}, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "qat_aes_ctr", - .base.cra_priority = 4001, - .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), - .base.cra_alignmask = 0, - .base.cra_module = THIS_MODULE, - - .init = qat_alg_skcipher_init_tfm, - .exit = qat_alg_skcipher_exit_tfm, - .setkey = qat_alg_skcipher_ctr_setkey, - .decrypt = qat_alg_skcipher_decrypt, - .encrypt = qat_alg_skcipher_encrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, -}, { - .base.cra_name = "xts(aes)", - .base.cra_driver_name = "qat_aes_xts", - .base.cra_priority = 4001, - .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), - .base.cra_alignmask = 0, - .base.cra_module = THIS_MODULE, - - .init = qat_alg_skcipher_init_xts_tfm, - .exit = qat_alg_skcipher_exit_xts_tfm, - .setkey = qat_alg_skcipher_xts_setkey, - .decrypt = qat_alg_skcipher_xts_decrypt, - .encrypt = qat_alg_skcipher_xts_encrypt, - .min_keysize = 2 * AES_MIN_KEY_SIZE, - .max_keysize = 2 * AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, -} }; - -int qat_algs_register(void) -{ - int ret = 0; - - mutex_lock(&algs_lock); - if (++active_devs != 1) - goto unlock; - - ret = crypto_register_skciphers(qat_skciphers, - ARRAY_SIZE(qat_skciphers)); - if (ret) - goto unlock; - - ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); - if (ret) - goto unreg_algs; - -unlock: - mutex_unlock(&algs_lock); - return ret; - -unreg_algs: - crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers)); - goto unlock; -} - -void qat_algs_unregister(void) -{ - mutex_lock(&algs_lock); - if (--active_devs != 0) - goto unlock; - - crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); - crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers)); - -unlock: - mutex_unlock(&algs_lock); -} diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.c b/drivers/crypto/qat/qat_common/qat_algs_send.c deleted file mode 100644 index bb80455b3e81..000000000000 --- a/drivers/crypto/qat/qat_common/qat_algs_send.c +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2022 Intel Corporation */ -#include -#include "adf_transport.h" -#include "qat_algs_send.h" -#include "qat_crypto.h" - -#define ADF_MAX_RETRIES 20 - -static int qat_alg_send_message_retry(struct qat_alg_req *req) -{ - int ret = 0, ctr = 0; - - do { - ret = adf_send_message(req->tx_ring, req->fw_req); - } while (ret == -EAGAIN && ctr++ < ADF_MAX_RETRIES); - - if (ret == -EAGAIN) - return -ENOSPC; - - return -EINPROGRESS; -} - -void qat_alg_send_backlog(struct qat_instance_backlog *backlog) -{ - struct qat_alg_req *req, *tmp; - - spin_lock_bh(&backlog->lock); - list_for_each_entry_safe(req, tmp, &backlog->list, list) { - if (adf_send_message(req->tx_ring, req->fw_req)) { - /* The HW ring is full. Do nothing. - * qat_alg_send_backlog() will be invoked again by - * another callback. - */ - break; - } - list_del(&req->list); - crypto_request_complete(req->base, -EINPROGRESS); - } - spin_unlock_bh(&backlog->lock); -} - -static void qat_alg_backlog_req(struct qat_alg_req *req, - struct qat_instance_backlog *backlog) -{ - INIT_LIST_HEAD(&req->list); - - spin_lock_bh(&backlog->lock); - list_add_tail(&req->list, &backlog->list); - spin_unlock_bh(&backlog->lock); -} - -static int qat_alg_send_message_maybacklog(struct qat_alg_req *req) -{ - struct qat_instance_backlog *backlog = req->backlog; - struct adf_etr_ring_data *tx_ring = req->tx_ring; - u32 *fw_req = req->fw_req; - - /* If any request is already backlogged, then add to backlog list */ - if (!list_empty(&backlog->list)) - goto enqueue; - - /* If ring is nearly full, then add to backlog list */ - if (adf_ring_nearly_full(tx_ring)) - goto enqueue; - - /* If adding request to HW ring fails, then add to backlog list */ - if (adf_send_message(tx_ring, fw_req)) - goto enqueue; - - return -EINPROGRESS; - -enqueue: - qat_alg_backlog_req(req, backlog); - - return -EBUSY; -} - -int qat_alg_send_message(struct qat_alg_req *req) -{ - u32 flags = req->base->flags; - - if (flags & CRYPTO_TFM_REQ_MAY_BACKLOG) - return qat_alg_send_message_maybacklog(req); - else - return qat_alg_send_message_retry(req); -} diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.h b/drivers/crypto/qat/qat_common/qat_algs_send.h deleted file mode 100644 index 0baca16e1eff..000000000000 --- a/drivers/crypto/qat/qat_common/qat_algs_send.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef QAT_ALGS_SEND_H -#define QAT_ALGS_SEND_H - -#include -#include "adf_transport_internal.h" - -struct qat_instance_backlog { - struct list_head list; - spinlock_t lock; /* protects backlog list */ -}; - -struct qat_alg_req { - u32 *fw_req; - struct adf_etr_ring_data *tx_ring; - struct crypto_async_request *base; - struct list_head list; - struct qat_instance_backlog *backlog; -}; - -int qat_alg_send_message(struct qat_alg_req *req); -void qat_alg_send_backlog(struct qat_instance_backlog *backlog); - -#endif diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c deleted file mode 100644 index 935a7e012946..000000000000 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ /dev/null @@ -1,1309 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "icp_qat_fw_pke.h" -#include "adf_accel_devices.h" -#include "qat_algs_send.h" -#include "adf_transport.h" -#include "adf_common_drv.h" -#include "qat_crypto.h" - -static DEFINE_MUTEX(algs_lock); -static unsigned int active_devs; - -struct qat_rsa_input_params { - union { - struct { - dma_addr_t m; - dma_addr_t e; - dma_addr_t n; - } enc; - struct { - dma_addr_t c; - dma_addr_t d; - dma_addr_t n; - } dec; - struct { - dma_addr_t c; - dma_addr_t p; - dma_addr_t q; - dma_addr_t dp; - dma_addr_t dq; - dma_addr_t qinv; - } dec_crt; - u64 in_tab[8]; - }; -} __packed __aligned(64); - -struct qat_rsa_output_params { - union { - struct { - dma_addr_t c; - } enc; - struct { - dma_addr_t m; - } dec; - u64 out_tab[8]; - }; -} __packed __aligned(64); - -struct qat_rsa_ctx { - char *n; - char *e; - char *d; - char *p; - char *q; - char *dp; - char *dq; - char *qinv; - dma_addr_t dma_n; - dma_addr_t dma_e; - dma_addr_t dma_d; - dma_addr_t dma_p; - dma_addr_t dma_q; - dma_addr_t dma_dp; - dma_addr_t dma_dq; - dma_addr_t dma_qinv; - unsigned int key_sz; - bool crt_mode; - struct qat_crypto_instance *inst; -} __packed __aligned(64); - -struct qat_dh_input_params { - union { - struct { - dma_addr_t b; - dma_addr_t xa; - dma_addr_t p; - } in; - struct { - dma_addr_t xa; - dma_addr_t p; - } in_g2; - u64 in_tab[8]; - }; -} __packed __aligned(64); - -struct qat_dh_output_params { - union { - dma_addr_t r; - u64 out_tab[8]; - }; -} __packed __aligned(64); - -struct qat_dh_ctx { - char *g; - char *xa; - char *p; - dma_addr_t dma_g; - dma_addr_t dma_xa; - dma_addr_t dma_p; - unsigned int p_size; - bool g2; - struct qat_crypto_instance *inst; -} __packed __aligned(64); - -struct qat_asym_request { - union { - struct qat_rsa_input_params rsa; - struct qat_dh_input_params dh; - } in; - union { - struct qat_rsa_output_params rsa; - struct qat_dh_output_params dh; - } out; - dma_addr_t phy_in; - dma_addr_t phy_out; - char *src_align; - char *dst_align; - struct icp_qat_fw_pke_request req; - union { - struct qat_rsa_ctx *rsa; - struct qat_dh_ctx *dh; - } ctx; - union { - struct akcipher_request *rsa; - struct kpp_request *dh; - } areq; - int err; - void (*cb)(struct icp_qat_fw_pke_resp *resp); - struct qat_alg_req alg_req; -} __aligned(64); - -static int qat_alg_send_asym_message(struct qat_asym_request *qat_req, - struct qat_crypto_instance *inst, - struct crypto_async_request *base) -{ - struct qat_alg_req *alg_req = &qat_req->alg_req; - - alg_req->fw_req = (u32 *)&qat_req->req; - alg_req->tx_ring = inst->pke_tx; - alg_req->base = base; - alg_req->backlog = &inst->backlog; - - return qat_alg_send_message(alg_req); -} - -static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp) -{ - struct qat_asym_request *req = (void *)(__force long)resp->opaque; - struct kpp_request *areq = req->areq.dh; - struct device *dev = &GET_DEV(req->ctx.dh->inst->accel_dev); - int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( - resp->pke_resp_hdr.comn_resp_flags); - - err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; - - if (areq->src) { - dma_unmap_single(dev, req->in.dh.in.b, req->ctx.dh->p_size, - DMA_TO_DEVICE); - kfree_sensitive(req->src_align); - } - - areq->dst_len = req->ctx.dh->p_size; - if (req->dst_align) { - scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, - areq->dst_len, 1); - kfree_sensitive(req->dst_align); - } - - dma_unmap_single(dev, req->out.dh.r, req->ctx.dh->p_size, - DMA_FROM_DEVICE); - - dma_unmap_single(dev, req->phy_in, sizeof(struct qat_dh_input_params), - DMA_TO_DEVICE); - dma_unmap_single(dev, req->phy_out, - sizeof(struct qat_dh_output_params), - DMA_TO_DEVICE); - - kpp_request_complete(areq, err); -} - -#define PKE_DH_1536 0x390c1a49 -#define PKE_DH_G2_1536 0x2e0b1a3e -#define PKE_DH_2048 0x4d0c1a60 -#define PKE_DH_G2_2048 0x3e0b1a55 -#define PKE_DH_3072 0x510c1a77 -#define PKE_DH_G2_3072 0x3a0b1a6c -#define PKE_DH_4096 0x690c1a8e -#define PKE_DH_G2_4096 0x4a0b1a83 - -static unsigned long qat_dh_fn_id(unsigned int len, bool g2) -{ - unsigned int bitslen = len << 3; - - switch (bitslen) { - case 1536: - return g2 ? PKE_DH_G2_1536 : PKE_DH_1536; - case 2048: - return g2 ? PKE_DH_G2_2048 : PKE_DH_2048; - case 3072: - return g2 ? PKE_DH_G2_3072 : PKE_DH_3072; - case 4096: - return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; - default: - return 0; - } -} - -static int qat_dh_compute_value(struct kpp_request *req) -{ - struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); - struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_asym_request *qat_req = - PTR_ALIGN(kpp_request_ctx(req), 64); - struct icp_qat_fw_pke_request *msg = &qat_req->req; - gfp_t flags = qat_algs_alloc_flags(&req->base); - int n_input_params = 0; - u8 *vaddr; - int ret; - - if (unlikely(!ctx->xa)) - return -EINVAL; - - if (req->dst_len < ctx->p_size) { - req->dst_len = ctx->p_size; - return -EOVERFLOW; - } - - if (req->src_len > ctx->p_size) - return -EINVAL; - - memset(msg, '\0', sizeof(*msg)); - ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, - ICP_QAT_FW_COMN_REQ_FLAG_SET); - - msg->pke_hdr.cd_pars.func_id = qat_dh_fn_id(ctx->p_size, - !req->src && ctx->g2); - if (unlikely(!msg->pke_hdr.cd_pars.func_id)) - return -EINVAL; - - qat_req->cb = qat_dh_cb; - qat_req->ctx.dh = ctx; - qat_req->areq.dh = req; - msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; - msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - - /* - * If no source is provided use g as base - */ - if (req->src) { - qat_req->in.dh.in.xa = ctx->dma_xa; - qat_req->in.dh.in.p = ctx->dma_p; - n_input_params = 3; - } else { - if (ctx->g2) { - qat_req->in.dh.in_g2.xa = ctx->dma_xa; - qat_req->in.dh.in_g2.p = ctx->dma_p; - n_input_params = 2; - } else { - qat_req->in.dh.in.b = ctx->dma_g; - qat_req->in.dh.in.xa = ctx->dma_xa; - qat_req->in.dh.in.p = ctx->dma_p; - n_input_params = 3; - } - } - - ret = -ENOMEM; - if (req->src) { - /* - * src can be of any size in valid range, but HW expects it to - * be the same as modulo p so in case it is different we need - * to allocate a new buf and copy src data. - * In other case we just need to map the user provided buffer. - * Also need to make sure that it is in contiguous buffer. - */ - if (sg_is_last(req->src) && req->src_len == ctx->p_size) { - qat_req->src_align = NULL; - vaddr = sg_virt(req->src); - } else { - int shift = ctx->p_size - req->src_len; - - qat_req->src_align = kzalloc(ctx->p_size, flags); - if (unlikely(!qat_req->src_align)) - return ret; - - scatterwalk_map_and_copy(qat_req->src_align + shift, - req->src, 0, req->src_len, 0); - - vaddr = qat_req->src_align; - } - - qat_req->in.dh.in.b = dma_map_single(dev, vaddr, ctx->p_size, - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.dh.in.b))) - goto unmap_src; - } - /* - * dst can be of any size in valid range, but HW expects it to be the - * same as modulo m so in case it is different we need to allocate a - * new buf and copy src data. - * In other case we just need to map the user provided buffer. - * Also need to make sure that it is in contiguous buffer. - */ - if (sg_is_last(req->dst) && req->dst_len == ctx->p_size) { - qat_req->dst_align = NULL; - vaddr = sg_virt(req->dst); - } else { - qat_req->dst_align = kzalloc(ctx->p_size, flags); - if (unlikely(!qat_req->dst_align)) - goto unmap_src; - - vaddr = qat_req->dst_align; - } - qat_req->out.dh.r = dma_map_single(dev, vaddr, ctx->p_size, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.dh.r))) - goto unmap_dst; - - qat_req->in.dh.in_tab[n_input_params] = 0; - qat_req->out.dh.out_tab[1] = 0; - /* Mapping in.in.b or in.in_g2.xa is the same */ - qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh, - sizeof(struct qat_dh_input_params), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) - goto unmap_dst; - - qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh, - sizeof(struct qat_dh_output_params), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) - goto unmap_in_params; - - msg->pke_mid.src_data_addr = qat_req->phy_in; - msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (u64)(__force long)qat_req; - msg->input_param_count = n_input_params; - msg->output_param_count = 1; - - ret = qat_alg_send_asym_message(qat_req, inst, &req->base); - if (ret == -ENOSPC) - goto unmap_all; - - return ret; - -unmap_all: - if (!dma_mapping_error(dev, qat_req->phy_out)) - dma_unmap_single(dev, qat_req->phy_out, - sizeof(struct qat_dh_output_params), - DMA_TO_DEVICE); -unmap_in_params: - if (!dma_mapping_error(dev, qat_req->phy_in)) - dma_unmap_single(dev, qat_req->phy_in, - sizeof(struct qat_dh_input_params), - DMA_TO_DEVICE); -unmap_dst: - if (!dma_mapping_error(dev, qat_req->out.dh.r)) - dma_unmap_single(dev, qat_req->out.dh.r, ctx->p_size, - DMA_FROM_DEVICE); - kfree_sensitive(qat_req->dst_align); -unmap_src: - if (req->src) { - if (!dma_mapping_error(dev, qat_req->in.dh.in.b)) - dma_unmap_single(dev, qat_req->in.dh.in.b, - ctx->p_size, - DMA_TO_DEVICE); - kfree_sensitive(qat_req->src_align); - } - return ret; -} - -static int qat_dh_check_params_length(unsigned int p_len) -{ - switch (p_len) { - case 1536: - case 2048: - case 3072: - case 4096: - return 0; - } - return -EINVAL; -} - -static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) -{ - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - - if (qat_dh_check_params_length(params->p_size << 3)) - return -EINVAL; - - ctx->p_size = params->p_size; - ctx->p = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL); - if (!ctx->p) - return -ENOMEM; - memcpy(ctx->p, params->p, ctx->p_size); - - /* If g equals 2 don't copy it */ - if (params->g_size == 1 && *(char *)params->g == 0x02) { - ctx->g2 = true; - return 0; - } - - ctx->g = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); - if (!ctx->g) - return -ENOMEM; - memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, - params->g_size); - - return 0; -} - -static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx) -{ - if (ctx->g) { - memset(ctx->g, 0, ctx->p_size); - dma_free_coherent(dev, ctx->p_size, ctx->g, ctx->dma_g); - ctx->g = NULL; - } - if (ctx->xa) { - memset(ctx->xa, 0, ctx->p_size); - dma_free_coherent(dev, ctx->p_size, ctx->xa, ctx->dma_xa); - ctx->xa = NULL; - } - if (ctx->p) { - memset(ctx->p, 0, ctx->p_size); - dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); - ctx->p = NULL; - } - ctx->p_size = 0; - ctx->g2 = false; -} - -static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, - unsigned int len) -{ - struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - struct dh params; - int ret; - - if (crypto_dh_decode_key(buf, len, ¶ms) < 0) - return -EINVAL; - - /* Free old secret if any */ - qat_dh_clear_ctx(dev, ctx); - - ret = qat_dh_set_params(ctx, ¶ms); - if (ret < 0) - goto err_clear_ctx; - - ctx->xa = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_xa, - GFP_KERNEL); - if (!ctx->xa) { - ret = -ENOMEM; - goto err_clear_ctx; - } - memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, - params.key_size); - - return 0; - -err_clear_ctx: - qat_dh_clear_ctx(dev, ctx); - return ret; -} - -static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) -{ - struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); - - return ctx->p_size; -} - -static int qat_dh_init_tfm(struct crypto_kpp *tfm) -{ - struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); - struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(numa_node_id()); - - if (!inst) - return -EINVAL; - - kpp_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64); - - ctx->p_size = 0; - ctx->g2 = false; - ctx->inst = inst; - return 0; -} - -static void qat_dh_exit_tfm(struct crypto_kpp *tfm) -{ - struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - - qat_dh_clear_ctx(dev, ctx); - qat_crypto_put_instance(ctx->inst); -} - -static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) -{ - struct qat_asym_request *req = (void *)(__force long)resp->opaque; - struct akcipher_request *areq = req->areq.rsa; - struct device *dev = &GET_DEV(req->ctx.rsa->inst->accel_dev); - int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( - resp->pke_resp_hdr.comn_resp_flags); - - err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; - - kfree_sensitive(req->src_align); - - dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz, - DMA_TO_DEVICE); - - areq->dst_len = req->ctx.rsa->key_sz; - if (req->dst_align) { - scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, - areq->dst_len, 1); - - kfree_sensitive(req->dst_align); - } - - dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz, - DMA_FROM_DEVICE); - - dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); - dma_unmap_single(dev, req->phy_out, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); - - akcipher_request_complete(areq, err); -} - -void qat_alg_asym_callback(void *_resp) -{ - struct icp_qat_fw_pke_resp *resp = _resp; - struct qat_asym_request *areq = (void *)(__force long)resp->opaque; - struct qat_instance_backlog *backlog = areq->alg_req.backlog; - - areq->cb(resp); - - qat_alg_send_backlog(backlog); -} - -#define PKE_RSA_EP_512 0x1c161b21 -#define PKE_RSA_EP_1024 0x35111bf7 -#define PKE_RSA_EP_1536 0x4d111cdc -#define PKE_RSA_EP_2048 0x6e111dba -#define PKE_RSA_EP_3072 0x7d111ea3 -#define PKE_RSA_EP_4096 0xa5101f7e - -static unsigned long qat_rsa_enc_fn_id(unsigned int len) -{ - unsigned int bitslen = len << 3; - - switch (bitslen) { - case 512: - return PKE_RSA_EP_512; - case 1024: - return PKE_RSA_EP_1024; - case 1536: - return PKE_RSA_EP_1536; - case 2048: - return PKE_RSA_EP_2048; - case 3072: - return PKE_RSA_EP_3072; - case 4096: - return PKE_RSA_EP_4096; - default: - return 0; - } -} - -#define PKE_RSA_DP1_512 0x1c161b3c -#define PKE_RSA_DP1_1024 0x35111c12 -#define PKE_RSA_DP1_1536 0x4d111cf7 -#define PKE_RSA_DP1_2048 0x6e111dda -#define PKE_RSA_DP1_3072 0x7d111ebe -#define PKE_RSA_DP1_4096 0xa5101f98 - -static unsigned long qat_rsa_dec_fn_id(unsigned int len) -{ - unsigned int bitslen = len << 3; - - switch (bitslen) { - case 512: - return PKE_RSA_DP1_512; - case 1024: - return PKE_RSA_DP1_1024; - case 1536: - return PKE_RSA_DP1_1536; - case 2048: - return PKE_RSA_DP1_2048; - case 3072: - return PKE_RSA_DP1_3072; - case 4096: - return PKE_RSA_DP1_4096; - default: - return 0; - } -} - -#define PKE_RSA_DP2_512 0x1c131b57 -#define PKE_RSA_DP2_1024 0x26131c2d -#define PKE_RSA_DP2_1536 0x45111d12 -#define PKE_RSA_DP2_2048 0x59121dfa -#define PKE_RSA_DP2_3072 0x81121ed9 -#define PKE_RSA_DP2_4096 0xb1111fb2 - -static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) -{ - unsigned int bitslen = len << 3; - - switch (bitslen) { - case 512: - return PKE_RSA_DP2_512; - case 1024: - return PKE_RSA_DP2_1024; - case 1536: - return PKE_RSA_DP2_1536; - case 2048: - return PKE_RSA_DP2_2048; - case 3072: - return PKE_RSA_DP2_3072; - case 4096: - return PKE_RSA_DP2_4096; - default: - return 0; - } -} - -static int qat_rsa_enc(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_asym_request *qat_req = - PTR_ALIGN(akcipher_request_ctx(req), 64); - struct icp_qat_fw_pke_request *msg = &qat_req->req; - gfp_t flags = qat_algs_alloc_flags(&req->base); - u8 *vaddr; - int ret; - - if (unlikely(!ctx->n || !ctx->e)) - return -EINVAL; - - if (req->dst_len < ctx->key_sz) { - req->dst_len = ctx->key_sz; - return -EOVERFLOW; - } - - if (req->src_len > ctx->key_sz) - return -EINVAL; - - memset(msg, '\0', sizeof(*msg)); - ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, - ICP_QAT_FW_COMN_REQ_FLAG_SET); - msg->pke_hdr.cd_pars.func_id = qat_rsa_enc_fn_id(ctx->key_sz); - if (unlikely(!msg->pke_hdr.cd_pars.func_id)) - return -EINVAL; - - qat_req->cb = qat_rsa_cb; - qat_req->ctx.rsa = ctx; - qat_req->areq.rsa = req; - msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; - msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - - qat_req->in.rsa.enc.e = ctx->dma_e; - qat_req->in.rsa.enc.n = ctx->dma_n; - ret = -ENOMEM; - - /* - * src can be of any size in valid range, but HW expects it to be the - * same as modulo n so in case it is different we need to allocate a - * new buf and copy src data. - * In other case we just need to map the user provided buffer. - * Also need to make sure that it is in contiguous buffer. - */ - if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { - qat_req->src_align = NULL; - vaddr = sg_virt(req->src); - } else { - int shift = ctx->key_sz - req->src_len; - - qat_req->src_align = kzalloc(ctx->key_sz, flags); - if (unlikely(!qat_req->src_align)) - return ret; - - scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, - 0, req->src_len, 0); - vaddr = qat_req->src_align; - } - - qat_req->in.rsa.enc.m = dma_map_single(dev, vaddr, ctx->key_sz, - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m))) - goto unmap_src; - - if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { - qat_req->dst_align = NULL; - vaddr = sg_virt(req->dst); - } else { - qat_req->dst_align = kzalloc(ctx->key_sz, flags); - if (unlikely(!qat_req->dst_align)) - goto unmap_src; - vaddr = qat_req->dst_align; - } - - qat_req->out.rsa.enc.c = dma_map_single(dev, vaddr, ctx->key_sz, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c))) - goto unmap_dst; - - qat_req->in.rsa.in_tab[3] = 0; - qat_req->out.rsa.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, - sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) - goto unmap_dst; - - qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) - goto unmap_in_params; - - msg->pke_mid.src_data_addr = qat_req->phy_in; - msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (u64)(__force long)qat_req; - msg->input_param_count = 3; - msg->output_param_count = 1; - - ret = qat_alg_send_asym_message(qat_req, inst, &req->base); - if (ret == -ENOSPC) - goto unmap_all; - - return ret; - -unmap_all: - if (!dma_mapping_error(dev, qat_req->phy_out)) - dma_unmap_single(dev, qat_req->phy_out, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); -unmap_in_params: - if (!dma_mapping_error(dev, qat_req->phy_in)) - dma_unmap_single(dev, qat_req->phy_in, - sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); -unmap_dst: - if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c)) - dma_unmap_single(dev, qat_req->out.rsa.enc.c, - ctx->key_sz, DMA_FROM_DEVICE); - kfree_sensitive(qat_req->dst_align); -unmap_src: - if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m)) - dma_unmap_single(dev, qat_req->in.rsa.enc.m, ctx->key_sz, - DMA_TO_DEVICE); - kfree_sensitive(qat_req->src_align); - return ret; -} - -static int qat_rsa_dec(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_asym_request *qat_req = - PTR_ALIGN(akcipher_request_ctx(req), 64); - struct icp_qat_fw_pke_request *msg = &qat_req->req; - gfp_t flags = qat_algs_alloc_flags(&req->base); - u8 *vaddr; - int ret; - - if (unlikely(!ctx->n || !ctx->d)) - return -EINVAL; - - if (req->dst_len < ctx->key_sz) { - req->dst_len = ctx->key_sz; - return -EOVERFLOW; - } - - if (req->src_len > ctx->key_sz) - return -EINVAL; - - memset(msg, '\0', sizeof(*msg)); - ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, - ICP_QAT_FW_COMN_REQ_FLAG_SET); - msg->pke_hdr.cd_pars.func_id = ctx->crt_mode ? - qat_rsa_dec_fn_id_crt(ctx->key_sz) : - qat_rsa_dec_fn_id(ctx->key_sz); - if (unlikely(!msg->pke_hdr.cd_pars.func_id)) - return -EINVAL; - - qat_req->cb = qat_rsa_cb; - qat_req->ctx.rsa = ctx; - qat_req->areq.rsa = req; - msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; - msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - - if (ctx->crt_mode) { - qat_req->in.rsa.dec_crt.p = ctx->dma_p; - qat_req->in.rsa.dec_crt.q = ctx->dma_q; - qat_req->in.rsa.dec_crt.dp = ctx->dma_dp; - qat_req->in.rsa.dec_crt.dq = ctx->dma_dq; - qat_req->in.rsa.dec_crt.qinv = ctx->dma_qinv; - } else { - qat_req->in.rsa.dec.d = ctx->dma_d; - qat_req->in.rsa.dec.n = ctx->dma_n; - } - ret = -ENOMEM; - - /* - * src can be of any size in valid range, but HW expects it to be the - * same as modulo n so in case it is different we need to allocate a - * new buf and copy src data. - * In other case we just need to map the user provided buffer. - * Also need to make sure that it is in contiguous buffer. - */ - if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { - qat_req->src_align = NULL; - vaddr = sg_virt(req->src); - } else { - int shift = ctx->key_sz - req->src_len; - - qat_req->src_align = kzalloc(ctx->key_sz, flags); - if (unlikely(!qat_req->src_align)) - return ret; - - scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, - 0, req->src_len, 0); - vaddr = qat_req->src_align; - } - - qat_req->in.rsa.dec.c = dma_map_single(dev, vaddr, ctx->key_sz, - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c))) - goto unmap_src; - - if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { - qat_req->dst_align = NULL; - vaddr = sg_virt(req->dst); - } else { - qat_req->dst_align = kzalloc(ctx->key_sz, flags); - if (unlikely(!qat_req->dst_align)) - goto unmap_src; - vaddr = qat_req->dst_align; - } - qat_req->out.rsa.dec.m = dma_map_single(dev, vaddr, ctx->key_sz, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m))) - goto unmap_dst; - - if (ctx->crt_mode) - qat_req->in.rsa.in_tab[6] = 0; - else - qat_req->in.rsa.in_tab[3] = 0; - qat_req->out.rsa.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, - sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) - goto unmap_dst; - - qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) - goto unmap_in_params; - - msg->pke_mid.src_data_addr = qat_req->phy_in; - msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (u64)(__force long)qat_req; - if (ctx->crt_mode) - msg->input_param_count = 6; - else - msg->input_param_count = 3; - - msg->output_param_count = 1; - - ret = qat_alg_send_asym_message(qat_req, inst, &req->base); - if (ret == -ENOSPC) - goto unmap_all; - - return ret; - -unmap_all: - if (!dma_mapping_error(dev, qat_req->phy_out)) - dma_unmap_single(dev, qat_req->phy_out, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); -unmap_in_params: - if (!dma_mapping_error(dev, qat_req->phy_in)) - dma_unmap_single(dev, qat_req->phy_in, - sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); -unmap_dst: - if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m)) - dma_unmap_single(dev, qat_req->out.rsa.dec.m, - ctx->key_sz, DMA_FROM_DEVICE); - kfree_sensitive(qat_req->dst_align); -unmap_src: - if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c)) - dma_unmap_single(dev, qat_req->in.rsa.dec.c, ctx->key_sz, - DMA_TO_DEVICE); - kfree_sensitive(qat_req->src_align); - return ret; -} - -static int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value, - size_t vlen) -{ - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - const char *ptr = value; - int ret; - - while (!*ptr && vlen) { - ptr++; - vlen--; - } - - ctx->key_sz = vlen; - ret = -EINVAL; - /* invalid key size provided */ - if (!qat_rsa_enc_fn_id(ctx->key_sz)) - goto err; - - ret = -ENOMEM; - ctx->n = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_n, GFP_KERNEL); - if (!ctx->n) - goto err; - - memcpy(ctx->n, ptr, ctx->key_sz); - return 0; -err: - ctx->key_sz = 0; - ctx->n = NULL; - return ret; -} - -static int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value, - size_t vlen) -{ - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - const char *ptr = value; - - while (!*ptr && vlen) { - ptr++; - vlen--; - } - - if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) { - ctx->e = NULL; - return -EINVAL; - } - - ctx->e = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); - if (!ctx->e) - return -ENOMEM; - - memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); - return 0; -} - -static int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value, - size_t vlen) -{ - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - const char *ptr = value; - int ret; - - while (!*ptr && vlen) { - ptr++; - vlen--; - } - - ret = -EINVAL; - if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) - goto err; - - ret = -ENOMEM; - ctx->d = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); - if (!ctx->d) - goto err; - - memcpy(ctx->d + (ctx->key_sz - vlen), ptr, vlen); - return 0; -err: - ctx->d = NULL; - return ret; -} - -static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len) -{ - while (!**ptr && *len) { - (*ptr)++; - (*len)--; - } -} - -static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) -{ - struct qat_crypto_instance *inst = ctx->inst; - struct device *dev = &GET_DEV(inst->accel_dev); - const char *ptr; - unsigned int len; - unsigned int half_key_sz = ctx->key_sz / 2; - - /* p */ - ptr = rsa_key->p; - len = rsa_key->p_sz; - qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) - goto err; - ctx->p = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); - if (!ctx->p) - goto err; - memcpy(ctx->p + (half_key_sz - len), ptr, len); - - /* q */ - ptr = rsa_key->q; - len = rsa_key->q_sz; - qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) - goto free_p; - ctx->q = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); - if (!ctx->q) - goto free_p; - memcpy(ctx->q + (half_key_sz - len), ptr, len); - - /* dp */ - ptr = rsa_key->dp; - len = rsa_key->dp_sz; - qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) - goto free_q; - ctx->dp = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dp, - GFP_KERNEL); - if (!ctx->dp) - goto free_q; - memcpy(ctx->dp + (half_key_sz - len), ptr, len); - - /* dq */ - ptr = rsa_key->dq; - len = rsa_key->dq_sz; - qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) - goto free_dp; - ctx->dq = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dq, - GFP_KERNEL); - if (!ctx->dq) - goto free_dp; - memcpy(ctx->dq + (half_key_sz - len), ptr, len); - - /* qinv */ - ptr = rsa_key->qinv; - len = rsa_key->qinv_sz; - qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) - goto free_dq; - ctx->qinv = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_qinv, - GFP_KERNEL); - if (!ctx->qinv) - goto free_dq; - memcpy(ctx->qinv + (half_key_sz - len), ptr, len); - - ctx->crt_mode = true; - return; - -free_dq: - memset(ctx->dq, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); - ctx->dq = NULL; -free_dp: - memset(ctx->dp, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); - ctx->dp = NULL; -free_q: - memset(ctx->q, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); - ctx->q = NULL; -free_p: - memset(ctx->p, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); - ctx->p = NULL; -err: - ctx->crt_mode = false; -} - -static void qat_rsa_clear_ctx(struct device *dev, struct qat_rsa_ctx *ctx) -{ - unsigned int half_key_sz = ctx->key_sz / 2; - - /* Free the old key if any */ - if (ctx->n) - dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); - if (ctx->e) - dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); - if (ctx->d) { - memset(ctx->d, '\0', ctx->key_sz); - dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); - } - if (ctx->p) { - memset(ctx->p, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); - } - if (ctx->q) { - memset(ctx->q, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); - } - if (ctx->dp) { - memset(ctx->dp, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); - } - if (ctx->dq) { - memset(ctx->dq, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); - } - if (ctx->qinv) { - memset(ctx->qinv, '\0', half_key_sz); - dma_free_coherent(dev, half_key_sz, ctx->qinv, ctx->dma_qinv); - } - - ctx->n = NULL; - ctx->e = NULL; - ctx->d = NULL; - ctx->p = NULL; - ctx->q = NULL; - ctx->dp = NULL; - ctx->dq = NULL; - ctx->qinv = NULL; - ctx->crt_mode = false; - ctx->key_sz = 0; -} - -static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, - unsigned int keylen, bool private) -{ - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - struct rsa_key rsa_key; - int ret; - - qat_rsa_clear_ctx(dev, ctx); - - if (private) - ret = rsa_parse_priv_key(&rsa_key, key, keylen); - else - ret = rsa_parse_pub_key(&rsa_key, key, keylen); - if (ret < 0) - goto free; - - ret = qat_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz); - if (ret < 0) - goto free; - ret = qat_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); - if (ret < 0) - goto free; - if (private) { - ret = qat_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); - if (ret < 0) - goto free; - qat_rsa_setkey_crt(ctx, &rsa_key); - } - - if (!ctx->n || !ctx->e) { - /* invalid key provided */ - ret = -EINVAL; - goto free; - } - if (private && !ctx->d) { - /* invalid private key provided */ - ret = -EINVAL; - goto free; - } - - return 0; -free: - qat_rsa_clear_ctx(dev, ctx); - return ret; -} - -static int qat_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, - unsigned int keylen) -{ - return qat_rsa_setkey(tfm, key, keylen, false); -} - -static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, - unsigned int keylen) -{ - return qat_rsa_setkey(tfm, key, keylen, true); -} - -static unsigned int qat_rsa_max_size(struct crypto_akcipher *tfm) -{ - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - - return ctx->key_sz; -} - -static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) -{ - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(numa_node_id()); - - if (!inst) - return -EINVAL; - - akcipher_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64); - - ctx->key_sz = 0; - ctx->inst = inst; - return 0; -} - -static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm) -{ - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - - qat_rsa_clear_ctx(dev, ctx); - qat_crypto_put_instance(ctx->inst); -} - -static struct akcipher_alg rsa = { - .encrypt = qat_rsa_enc, - .decrypt = qat_rsa_dec, - .set_pub_key = qat_rsa_setpubkey, - .set_priv_key = qat_rsa_setprivkey, - .max_size = qat_rsa_max_size, - .init = qat_rsa_init_tfm, - .exit = qat_rsa_exit_tfm, - .base = { - .cra_name = "rsa", - .cra_driver_name = "qat-rsa", - .cra_priority = 1000, - .cra_module = THIS_MODULE, - .cra_ctxsize = sizeof(struct qat_rsa_ctx), - }, -}; - -static struct kpp_alg dh = { - .set_secret = qat_dh_set_secret, - .generate_public_key = qat_dh_compute_value, - .compute_shared_secret = qat_dh_compute_value, - .max_size = qat_dh_max_size, - .init = qat_dh_init_tfm, - .exit = qat_dh_exit_tfm, - .base = { - .cra_name = "dh", - .cra_driver_name = "qat-dh", - .cra_priority = 1000, - .cra_module = THIS_MODULE, - .cra_ctxsize = sizeof(struct qat_dh_ctx), - }, -}; - -int qat_asym_algs_register(void) -{ - int ret = 0; - - mutex_lock(&algs_lock); - if (++active_devs == 1) { - rsa.base.cra_flags = 0; - ret = crypto_register_akcipher(&rsa); - if (ret) - goto unlock; - ret = crypto_register_kpp(&dh); - } -unlock: - mutex_unlock(&algs_lock); - return ret; -} - -void qat_asym_algs_unregister(void) -{ - mutex_lock(&algs_lock); - if (--active_devs == 0) { - crypto_unregister_akcipher(&rsa); - crypto_unregister_kpp(&dh); - } - mutex_unlock(&algs_lock); -} diff --git a/drivers/crypto/qat/qat_common/qat_bl.c b/drivers/crypto/qat/qat_common/qat_bl.c deleted file mode 100644 index 76baed0a76c0..000000000000 --- a/drivers/crypto/qat/qat_common/qat_bl.c +++ /dev/null @@ -1,410 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2014 - 2022 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "qat_bl.h" -#include "qat_crypto.h" - -void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, - struct qat_request_buffs *buf) -{ - struct device *dev = &GET_DEV(accel_dev); - struct qat_alg_buf_list *bl = buf->bl; - struct qat_alg_buf_list *blout = buf->blout; - dma_addr_t blp = buf->blp; - dma_addr_t blpout = buf->bloutp; - size_t sz = buf->sz; - size_t sz_out = buf->sz_out; - int bl_dma_dir; - int i; - - bl_dma_dir = blp != blpout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; - - for (i = 0; i < bl->num_bufs; i++) - dma_unmap_single(dev, bl->buffers[i].addr, - bl->buffers[i].len, bl_dma_dir); - - dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); - - if (!buf->sgl_src_valid) - kfree(bl); - - if (blp != blpout) { - for (i = 0; i < blout->num_mapped_bufs; i++) { - dma_unmap_single(dev, blout->buffers[i].addr, - blout->buffers[i].len, - DMA_FROM_DEVICE); - } - dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); - - if (!buf->sgl_dst_valid) - kfree(blout); - } -} - -static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct scatterlist *sglout, - struct qat_request_buffs *buf, - dma_addr_t extra_dst_buff, - size_t sz_extra_dst_buff, - unsigned int sskip, - unsigned int dskip, - gfp_t flags) -{ - struct device *dev = &GET_DEV(accel_dev); - int i, sg_nctr = 0; - int n = sg_nents(sgl); - struct qat_alg_buf_list *bufl; - struct qat_alg_buf_list *buflout = NULL; - dma_addr_t blp = DMA_MAPPING_ERROR; - dma_addr_t bloutp = DMA_MAPPING_ERROR; - struct scatterlist *sg; - size_t sz_out, sz = struct_size(bufl, buffers, n); - int node = dev_to_node(&GET_DEV(accel_dev)); - unsigned int left; - int bufl_dma_dir; - - if (unlikely(!n)) - return -EINVAL; - - buf->sgl_src_valid = false; - buf->sgl_dst_valid = false; - - if (n > QAT_MAX_BUFF_DESC) { - bufl = kzalloc_node(sz, flags, node); - if (unlikely(!bufl)) - return -ENOMEM; - } else { - bufl = &buf->sgl_src.sgl_hdr; - memset(bufl, 0, sizeof(struct qat_alg_buf_list)); - buf->sgl_src_valid = true; - } - - bufl_dma_dir = sgl != sglout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; - - for (i = 0; i < n; i++) - bufl->buffers[i].addr = DMA_MAPPING_ERROR; - - left = sskip; - - for_each_sg(sgl, sg, n, i) { - int y = sg_nctr; - - if (!sg->length) - continue; - - if (left >= sg->length) { - left -= sg->length; - continue; - } - bufl->buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, - sg->length - left, - bufl_dma_dir); - bufl->buffers[y].len = sg->length; - if (unlikely(dma_mapping_error(dev, bufl->buffers[y].addr))) - goto err_in; - sg_nctr++; - if (left) { - bufl->buffers[y].len -= left; - left = 0; - } - } - bufl->num_bufs = sg_nctr; - blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, blp))) - goto err_in; - buf->bl = bufl; - buf->blp = blp; - buf->sz = sz; - /* Handle out of place operation */ - if (sgl != sglout) { - struct qat_alg_buf *buffers; - int extra_buff = extra_dst_buff ? 1 : 0; - int n_sglout = sg_nents(sglout); - - n = n_sglout + extra_buff; - sz_out = struct_size(buflout, buffers, n); - left = dskip; - - sg_nctr = 0; - - if (n > QAT_MAX_BUFF_DESC) { - buflout = kzalloc_node(sz_out, flags, node); - if (unlikely(!buflout)) - goto err_in; - } else { - buflout = &buf->sgl_dst.sgl_hdr; - memset(buflout, 0, sizeof(struct qat_alg_buf_list)); - buf->sgl_dst_valid = true; - } - - buffers = buflout->buffers; - for (i = 0; i < n; i++) - buffers[i].addr = DMA_MAPPING_ERROR; - - for_each_sg(sglout, sg, n_sglout, i) { - int y = sg_nctr; - - if (!sg->length) - continue; - - if (left >= sg->length) { - left -= sg->length; - continue; - } - buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, - sg->length - left, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, buffers[y].addr))) - goto err_out; - buffers[y].len = sg->length; - sg_nctr++; - if (left) { - buffers[y].len -= left; - left = 0; - } - } - if (extra_buff) { - buffers[sg_nctr].addr = extra_dst_buff; - buffers[sg_nctr].len = sz_extra_dst_buff; - } - - buflout->num_bufs = sg_nctr; - buflout->num_bufs += extra_buff; - buflout->num_mapped_bufs = sg_nctr; - bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, bloutp))) - goto err_out; - buf->blout = buflout; - buf->bloutp = bloutp; - buf->sz_out = sz_out; - } else { - /* Otherwise set the src and dst to the same address */ - buf->bloutp = buf->blp; - buf->sz_out = 0; - } - return 0; - -err_out: - if (!dma_mapping_error(dev, bloutp)) - dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE); - - n = sg_nents(sglout); - for (i = 0; i < n; i++) { - if (buflout->buffers[i].addr == extra_dst_buff) - break; - if (!dma_mapping_error(dev, buflout->buffers[i].addr)) - dma_unmap_single(dev, buflout->buffers[i].addr, - buflout->buffers[i].len, - DMA_FROM_DEVICE); - } - - if (!buf->sgl_dst_valid) - kfree(buflout); - -err_in: - if (!dma_mapping_error(dev, blp)) - dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); - - n = sg_nents(sgl); - for (i = 0; i < n; i++) - if (!dma_mapping_error(dev, bufl->buffers[i].addr)) - dma_unmap_single(dev, bufl->buffers[i].addr, - bufl->buffers[i].len, - bufl_dma_dir); - - if (!buf->sgl_src_valid) - kfree(bufl); - - dev_err(dev, "Failed to map buf for dma\n"); - return -ENOMEM; -} - -int qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct scatterlist *sglout, - struct qat_request_buffs *buf, - struct qat_sgl_to_bufl_params *params, - gfp_t flags) -{ - dma_addr_t extra_dst_buff = 0; - size_t sz_extra_dst_buff = 0; - unsigned int sskip = 0; - unsigned int dskip = 0; - - if (params) { - extra_dst_buff = params->extra_dst_buff; - sz_extra_dst_buff = params->sz_extra_dst_buff; - sskip = params->sskip; - dskip = params->dskip; - } - - return __qat_bl_sgl_to_bufl(accel_dev, sgl, sglout, buf, - extra_dst_buff, sz_extra_dst_buff, - sskip, dskip, flags); -} - -static void qat_bl_sgl_unmap(struct adf_accel_dev *accel_dev, - struct qat_alg_buf_list *bl) -{ - struct device *dev = &GET_DEV(accel_dev); - int n = bl->num_bufs; - int i; - - for (i = 0; i < n; i++) - if (!dma_mapping_error(dev, bl->buffers[i].addr)) - dma_unmap_single(dev, bl->buffers[i].addr, - bl->buffers[i].len, DMA_FROM_DEVICE); -} - -static int qat_bl_sgl_map(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct qat_alg_buf_list **bl) -{ - struct device *dev = &GET_DEV(accel_dev); - struct qat_alg_buf_list *bufl; - int node = dev_to_node(dev); - struct scatterlist *sg; - int n, i, sg_nctr; - size_t sz; - - n = sg_nents(sgl); - sz = struct_size(bufl, buffers, n); - bufl = kzalloc_node(sz, GFP_KERNEL, node); - if (unlikely(!bufl)) - return -ENOMEM; - - for (i = 0; i < n; i++) - bufl->buffers[i].addr = DMA_MAPPING_ERROR; - - sg_nctr = 0; - for_each_sg(sgl, sg, n, i) { - int y = sg_nctr; - - if (!sg->length) - continue; - - bufl->buffers[y].addr = dma_map_single(dev, sg_virt(sg), - sg->length, - DMA_FROM_DEVICE); - bufl->buffers[y].len = sg->length; - if (unlikely(dma_mapping_error(dev, bufl->buffers[y].addr))) - goto err_map; - sg_nctr++; - } - bufl->num_bufs = sg_nctr; - bufl->num_mapped_bufs = sg_nctr; - - *bl = bufl; - - return 0; - -err_map: - for (i = 0; i < n; i++) - if (!dma_mapping_error(dev, bufl->buffers[i].addr)) - dma_unmap_single(dev, bufl->buffers[i].addr, - bufl->buffers[i].len, - DMA_FROM_DEVICE); - kfree(bufl); - *bl = NULL; - - return -ENOMEM; -} - -static void qat_bl_sgl_free_unmap(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct qat_alg_buf_list *bl, - bool free_bl) -{ - if (bl) { - qat_bl_sgl_unmap(accel_dev, bl); - - if (free_bl) - kfree(bl); - } - if (sgl) - sgl_free(sgl); -} - -static int qat_bl_sgl_alloc_map(struct adf_accel_dev *accel_dev, - struct scatterlist **sgl, - struct qat_alg_buf_list **bl, - unsigned int dlen, - gfp_t gfp) -{ - struct scatterlist *dst; - int ret; - - dst = sgl_alloc(dlen, gfp, NULL); - if (!dst) { - dev_err(&GET_DEV(accel_dev), "sg_alloc failed\n"); - return -ENOMEM; - } - - ret = qat_bl_sgl_map(accel_dev, dst, bl); - if (ret) - goto err; - - *sgl = dst; - - return 0; - -err: - sgl_free(dst); - *sgl = NULL; - return ret; -} - -int qat_bl_realloc_map_new_dst(struct adf_accel_dev *accel_dev, - struct scatterlist **sg, - unsigned int dlen, - struct qat_request_buffs *qat_bufs, - gfp_t gfp) -{ - struct device *dev = &GET_DEV(accel_dev); - dma_addr_t new_blp = DMA_MAPPING_ERROR; - struct qat_alg_buf_list *new_bl; - struct scatterlist *new_sg; - size_t new_bl_size; - int ret; - - ret = qat_bl_sgl_alloc_map(accel_dev, &new_sg, &new_bl, dlen, gfp); - if (ret) - return ret; - - new_bl_size = struct_size(new_bl, buffers, new_bl->num_bufs); - - /* Map new firmware SGL descriptor */ - new_blp = dma_map_single(dev, new_bl, new_bl_size, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, new_blp))) - goto err; - - /* Unmap old firmware SGL descriptor */ - dma_unmap_single(dev, qat_bufs->bloutp, qat_bufs->sz_out, DMA_TO_DEVICE); - - /* Free and unmap old scatterlist */ - qat_bl_sgl_free_unmap(accel_dev, *sg, qat_bufs->blout, - !qat_bufs->sgl_dst_valid); - - qat_bufs->sgl_dst_valid = false; - qat_bufs->blout = new_bl; - qat_bufs->bloutp = new_blp; - qat_bufs->sz_out = new_bl_size; - - *sg = new_sg; - - return 0; -err: - qat_bl_sgl_free_unmap(accel_dev, new_sg, new_bl, true); - - if (!dma_mapping_error(dev, new_blp)) - dma_unmap_single(dev, new_blp, new_bl_size, DMA_TO_DEVICE); - - return -ENOMEM; -} diff --git a/drivers/crypto/qat/qat_common/qat_bl.h b/drivers/crypto/qat/qat_common/qat_bl.h deleted file mode 100644 index d87e4f35ac39..000000000000 --- a/drivers/crypto/qat/qat_common/qat_bl.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2014 - 2022 Intel Corporation */ -#ifndef QAT_BL_H -#define QAT_BL_H -#include -#include -#include - -#define QAT_MAX_BUFF_DESC 4 - -struct qat_alg_buf { - u32 len; - u32 resrvd; - u64 addr; -} __packed; - -struct qat_alg_buf_list { - u64 resrvd; - u32 num_bufs; - u32 num_mapped_bufs; - struct qat_alg_buf buffers[]; -} __packed; - -struct qat_alg_fixed_buf_list { - struct qat_alg_buf_list sgl_hdr; - struct qat_alg_buf descriptors[QAT_MAX_BUFF_DESC]; -} __packed __aligned(64); - -struct qat_request_buffs { - struct qat_alg_buf_list *bl; - dma_addr_t blp; - struct qat_alg_buf_list *blout; - dma_addr_t bloutp; - size_t sz; - size_t sz_out; - bool sgl_src_valid; - bool sgl_dst_valid; - struct qat_alg_fixed_buf_list sgl_src; - struct qat_alg_fixed_buf_list sgl_dst; -}; - -struct qat_sgl_to_bufl_params { - dma_addr_t extra_dst_buff; - size_t sz_extra_dst_buff; - unsigned int sskip; - unsigned int dskip; -}; - -void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, - struct qat_request_buffs *buf); -int qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, - struct scatterlist *sgl, - struct scatterlist *sglout, - struct qat_request_buffs *buf, - struct qat_sgl_to_bufl_params *params, - gfp_t flags); - -static inline gfp_t qat_algs_alloc_flags(struct crypto_async_request *req) -{ - return req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; -} - -int qat_bl_realloc_map_new_dst(struct adf_accel_dev *accel_dev, - struct scatterlist **newd, - unsigned int dlen, - struct qat_request_buffs *qat_bufs, - gfp_t gfp); - -#endif diff --git a/drivers/crypto/qat/qat_common/qat_comp_algs.c b/drivers/crypto/qat/qat_common/qat_comp_algs.c deleted file mode 100644 index b533984906ec..000000000000 --- a/drivers/crypto/qat/qat_common/qat_comp_algs.c +++ /dev/null @@ -1,489 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "qat_bl.h" -#include "qat_comp_req.h" -#include "qat_compression.h" -#include "qat_algs_send.h" - -#define QAT_RFC_1950_HDR_SIZE 2 -#define QAT_RFC_1950_FOOTER_SIZE 4 -#define QAT_RFC_1950_CM_DEFLATE 8 -#define QAT_RFC_1950_CM_DEFLATE_CINFO_32K 7 -#define QAT_RFC_1950_CM_MASK 0x0f -#define QAT_RFC_1950_CM_OFFSET 4 -#define QAT_RFC_1950_DICT_MASK 0x20 -#define QAT_RFC_1950_COMP_HDR 0x785e - -static DEFINE_MUTEX(algs_lock); -static unsigned int active_devs; - -enum direction { - DECOMPRESSION = 0, - COMPRESSION = 1, -}; - -struct qat_compression_req; - -struct qat_compression_ctx { - u8 comp_ctx[QAT_COMP_CTX_SIZE]; - struct qat_compression_instance *inst; - int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp); -}; - -struct qat_dst { - bool is_null; - int resubmitted; -}; - -struct qat_compression_req { - u8 req[QAT_COMP_REQ_SIZE]; - struct qat_compression_ctx *qat_compression_ctx; - struct acomp_req *acompress_req; - struct qat_request_buffs buf; - enum direction dir; - int actual_dlen; - struct qat_alg_req alg_req; - struct work_struct resubmit; - struct qat_dst dst; -}; - -static int qat_alg_send_dc_message(struct qat_compression_req *qat_req, - struct qat_compression_instance *inst, - struct crypto_async_request *base) -{ - struct qat_alg_req *alg_req = &qat_req->alg_req; - - alg_req->fw_req = (u32 *)&qat_req->req; - alg_req->tx_ring = inst->dc_tx; - alg_req->base = base; - alg_req->backlog = &inst->backlog; - - return qat_alg_send_message(alg_req); -} - -static void qat_comp_resubmit(struct work_struct *work) -{ - struct qat_compression_req *qat_req = - container_of(work, struct qat_compression_req, resubmit); - struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx; - struct adf_accel_dev *accel_dev = ctx->inst->accel_dev; - struct qat_request_buffs *qat_bufs = &qat_req->buf; - struct qat_compression_instance *inst = ctx->inst; - struct acomp_req *areq = qat_req->acompress_req; - struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq); - unsigned int dlen = CRYPTO_ACOMP_DST_MAX; - u8 *req = qat_req->req; - dma_addr_t dfbuf; - int ret; - - areq->dlen = dlen; - - dev_dbg(&GET_DEV(accel_dev), "[%s][%s] retry NULL dst request - dlen = %d\n", - crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)), - qat_req->dir == COMPRESSION ? "comp" : "decomp", dlen); - - ret = qat_bl_realloc_map_new_dst(accel_dev, &areq->dst, dlen, qat_bufs, - qat_algs_alloc_flags(&areq->base)); - if (ret) - goto err; - - qat_req->dst.resubmitted = true; - - dfbuf = qat_req->buf.bloutp; - qat_comp_override_dst(req, dfbuf, dlen); - - ret = qat_alg_send_dc_message(qat_req, inst, &areq->base); - if (ret != -ENOSPC) - return; - -err: - qat_bl_free_bufl(accel_dev, qat_bufs); - acomp_request_complete(areq, ret); -} - -static int parse_zlib_header(u16 zlib_h) -{ - int ret = -EINVAL; - __be16 header; - u8 *header_p; - u8 cmf, flg; - - header = cpu_to_be16(zlib_h); - header_p = (u8 *)&header; - - flg = header_p[0]; - cmf = header_p[1]; - - if (cmf >> QAT_RFC_1950_CM_OFFSET > QAT_RFC_1950_CM_DEFLATE_CINFO_32K) - return ret; - - if ((cmf & QAT_RFC_1950_CM_MASK) != QAT_RFC_1950_CM_DEFLATE) - return ret; - - if (flg & QAT_RFC_1950_DICT_MASK) - return ret; - - return 0; -} - -static int qat_comp_rfc1950_callback(struct qat_compression_req *qat_req, - void *resp) -{ - struct acomp_req *areq = qat_req->acompress_req; - enum direction dir = qat_req->dir; - __be32 qat_produced_adler; - - qat_produced_adler = cpu_to_be32(qat_comp_get_produced_adler32(resp)); - - if (dir == COMPRESSION) { - __be16 zlib_header; - - zlib_header = cpu_to_be16(QAT_RFC_1950_COMP_HDR); - scatterwalk_map_and_copy(&zlib_header, areq->dst, 0, QAT_RFC_1950_HDR_SIZE, 1); - areq->dlen += QAT_RFC_1950_HDR_SIZE; - - scatterwalk_map_and_copy(&qat_produced_adler, areq->dst, areq->dlen, - QAT_RFC_1950_FOOTER_SIZE, 1); - areq->dlen += QAT_RFC_1950_FOOTER_SIZE; - } else { - __be32 decomp_adler; - int footer_offset; - int consumed; - - consumed = qat_comp_get_consumed_ctr(resp); - footer_offset = consumed + QAT_RFC_1950_HDR_SIZE; - if (footer_offset + QAT_RFC_1950_FOOTER_SIZE > areq->slen) - return -EBADMSG; - - scatterwalk_map_and_copy(&decomp_adler, areq->src, footer_offset, - QAT_RFC_1950_FOOTER_SIZE, 0); - - if (qat_produced_adler != decomp_adler) - return -EBADMSG; - } - return 0; -} - -static void qat_comp_generic_callback(struct qat_compression_req *qat_req, - void *resp) -{ - struct acomp_req *areq = qat_req->acompress_req; - struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx; - struct adf_accel_dev *accel_dev = ctx->inst->accel_dev; - struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq); - struct qat_compression_instance *inst = ctx->inst; - int consumed, produced; - s8 cmp_err, xlt_err; - int res = -EBADMSG; - int status; - u8 cnv; - - status = qat_comp_get_cmp_status(resp); - status |= qat_comp_get_xlt_status(resp); - cmp_err = qat_comp_get_cmp_err(resp); - xlt_err = qat_comp_get_xlt_err(resp); - - consumed = qat_comp_get_consumed_ctr(resp); - produced = qat_comp_get_produced_ctr(resp); - - dev_dbg(&GET_DEV(accel_dev), - "[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d", - crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)), - qat_req->dir == COMPRESSION ? "comp " : "decomp", - status ? "ERR" : "OK ", - areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err); - - areq->dlen = 0; - - if (qat_req->dir == DECOMPRESSION && qat_req->dst.is_null) { - if (cmp_err == ERR_CODE_OVERFLOW_ERROR) { - if (qat_req->dst.resubmitted) { - dev_dbg(&GET_DEV(accel_dev), - "Output does not fit destination buffer\n"); - res = -EOVERFLOW; - goto end; - } - - INIT_WORK(&qat_req->resubmit, qat_comp_resubmit); - adf_misc_wq_queue_work(&qat_req->resubmit); - return; - } - } - - if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) - goto end; - - if (qat_req->dir == COMPRESSION) { - cnv = qat_comp_get_cmp_cnv_flag(resp); - if (unlikely(!cnv)) { - dev_err(&GET_DEV(accel_dev), - "Verified compression not supported\n"); - goto end; - } - - if (unlikely(produced > qat_req->actual_dlen)) { - memset(inst->dc_data->ovf_buff, 0, - inst->dc_data->ovf_buff_sz); - dev_dbg(&GET_DEV(accel_dev), - "Actual buffer overflow: produced=%d, dlen=%d\n", - produced, qat_req->actual_dlen); - goto end; - } - } - - res = 0; - areq->dlen = produced; - - if (ctx->qat_comp_callback) - res = ctx->qat_comp_callback(qat_req, resp); - -end: - qat_bl_free_bufl(accel_dev, &qat_req->buf); - acomp_request_complete(areq, res); -} - -void qat_comp_alg_callback(void *resp) -{ - struct qat_compression_req *qat_req = - (void *)(__force long)qat_comp_get_opaque(resp); - struct qat_instance_backlog *backlog = qat_req->alg_req.backlog; - - qat_comp_generic_callback(qat_req, resp); - - qat_alg_send_backlog(backlog); -} - -static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm) -{ - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); - struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm); - struct qat_compression_instance *inst; - int node; - - if (tfm->node == NUMA_NO_NODE) - node = numa_node_id(); - else - node = tfm->node; - - memset(ctx, 0, sizeof(*ctx)); - inst = qat_compression_get_instance_node(node); - if (!inst) - return -EINVAL; - ctx->inst = inst; - - ctx->inst->build_deflate_ctx(ctx->comp_ctx); - - return 0; -} - -static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm) -{ - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); - struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm); - - qat_compression_put_instance(ctx->inst); - memset(ctx, 0, sizeof(*ctx)); -} - -static int qat_comp_alg_rfc1950_init_tfm(struct crypto_acomp *acomp_tfm) -{ - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); - struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm); - int ret; - - ret = qat_comp_alg_init_tfm(acomp_tfm); - ctx->qat_comp_callback = &qat_comp_rfc1950_callback; - - return ret; -} - -static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir, - unsigned int shdr, unsigned int sftr, - unsigned int dhdr, unsigned int dftr) -{ - struct qat_compression_req *qat_req = acomp_request_ctx(areq); - struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(areq); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); - struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm); - struct qat_compression_instance *inst = ctx->inst; - gfp_t f = qat_algs_alloc_flags(&areq->base); - struct qat_sgl_to_bufl_params params = {0}; - int slen = areq->slen - shdr - sftr; - int dlen = areq->dlen - dhdr - dftr; - dma_addr_t sfbuf, dfbuf; - u8 *req = qat_req->req; - size_t ovf_buff_sz; - int ret; - - params.sskip = shdr; - params.dskip = dhdr; - - if (!areq->src || !slen) - return -EINVAL; - - if (areq->dst && !dlen) - return -EINVAL; - - qat_req->dst.is_null = false; - - /* Handle acomp requests that require the allocation of a destination - * buffer. The size of the destination buffer is double the source - * buffer (rounded up to the size of a page) to fit the decompressed - * output or an expansion on the data for compression. - */ - if (!areq->dst) { - qat_req->dst.is_null = true; - - dlen = round_up(2 * slen, PAGE_SIZE); - areq->dst = sgl_alloc(dlen, f, NULL); - if (!areq->dst) - return -ENOMEM; - - dlen -= dhdr + dftr; - areq->dlen = dlen; - qat_req->dst.resubmitted = false; - } - - if (dir == COMPRESSION) { - params.extra_dst_buff = inst->dc_data->ovf_buff_p; - ovf_buff_sz = inst->dc_data->ovf_buff_sz; - params.sz_extra_dst_buff = ovf_buff_sz; - } - - ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst, - &qat_req->buf, ¶ms, f); - if (unlikely(ret)) - return ret; - - sfbuf = qat_req->buf.blp; - dfbuf = qat_req->buf.bloutp; - qat_req->qat_compression_ctx = ctx; - qat_req->acompress_req = areq; - qat_req->dir = dir; - - if (dir == COMPRESSION) { - qat_req->actual_dlen = dlen; - dlen += ovf_buff_sz; - qat_comp_create_compression_req(ctx->comp_ctx, req, - (u64)(__force long)sfbuf, slen, - (u64)(__force long)dfbuf, dlen, - (u64)(__force long)qat_req); - } else { - qat_comp_create_decompression_req(ctx->comp_ctx, req, - (u64)(__force long)sfbuf, slen, - (u64)(__force long)dfbuf, dlen, - (u64)(__force long)qat_req); - } - - ret = qat_alg_send_dc_message(qat_req, inst, &areq->base); - if (ret == -ENOSPC) - qat_bl_free_bufl(inst->accel_dev, &qat_req->buf); - - return ret; -} - -static int qat_comp_alg_compress(struct acomp_req *req) -{ - return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, 0, 0); -} - -static int qat_comp_alg_decompress(struct acomp_req *req) -{ - return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0); -} - -static int qat_comp_alg_rfc1950_compress(struct acomp_req *req) -{ - if (!req->dst && req->dlen != 0) - return -EINVAL; - - if (req->dst && req->dlen <= QAT_RFC_1950_HDR_SIZE + QAT_RFC_1950_FOOTER_SIZE) - return -EINVAL; - - return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, - QAT_RFC_1950_HDR_SIZE, - QAT_RFC_1950_FOOTER_SIZE); -} - -static int qat_comp_alg_rfc1950_decompress(struct acomp_req *req) -{ - struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); - struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm); - struct adf_accel_dev *accel_dev = ctx->inst->accel_dev; - u16 zlib_header; - int ret; - - if (req->slen <= QAT_RFC_1950_HDR_SIZE + QAT_RFC_1950_FOOTER_SIZE) - return -EBADMSG; - - scatterwalk_map_and_copy(&zlib_header, req->src, 0, QAT_RFC_1950_HDR_SIZE, 0); - - ret = parse_zlib_header(zlib_header); - if (ret) { - dev_dbg(&GET_DEV(accel_dev), "Error parsing zlib header\n"); - return ret; - } - - return qat_comp_alg_compress_decompress(req, DECOMPRESSION, QAT_RFC_1950_HDR_SIZE, - QAT_RFC_1950_FOOTER_SIZE, 0, 0); -} - -static struct acomp_alg qat_acomp[] = { { - .base = { - .cra_name = "deflate", - .cra_driver_name = "qat_deflate", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct qat_compression_ctx), - .cra_module = THIS_MODULE, - }, - .init = qat_comp_alg_init_tfm, - .exit = qat_comp_alg_exit_tfm, - .compress = qat_comp_alg_compress, - .decompress = qat_comp_alg_decompress, - .dst_free = sgl_free, - .reqsize = sizeof(struct qat_compression_req), -}, { - .base = { - .cra_name = "zlib-deflate", - .cra_driver_name = "qat_zlib_deflate", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_ctxsize = sizeof(struct qat_compression_ctx), - .cra_module = THIS_MODULE, - }, - .init = qat_comp_alg_rfc1950_init_tfm, - .exit = qat_comp_alg_exit_tfm, - .compress = qat_comp_alg_rfc1950_compress, - .decompress = qat_comp_alg_rfc1950_decompress, - .dst_free = sgl_free, - .reqsize = sizeof(struct qat_compression_req), -} }; - -int qat_comp_algs_register(void) -{ - int ret = 0; - - mutex_lock(&algs_lock); - if (++active_devs == 1) - ret = crypto_register_acomps(qat_acomp, ARRAY_SIZE(qat_acomp)); - mutex_unlock(&algs_lock); - return ret; -} - -void qat_comp_algs_unregister(void) -{ - mutex_lock(&algs_lock); - if (--active_devs == 0) - crypto_unregister_acomps(qat_acomp, ARRAY_SIZE(qat_acomp)); - mutex_unlock(&algs_lock); -} diff --git a/drivers/crypto/qat/qat_common/qat_comp_req.h b/drivers/crypto/qat/qat_common/qat_comp_req.h deleted file mode 100644 index 404e32c5e778..000000000000 --- a/drivers/crypto/qat/qat_common/qat_comp_req.h +++ /dev/null @@ -1,123 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef _QAT_COMP_REQ_H_ -#define _QAT_COMP_REQ_H_ - -#include "icp_qat_fw_comp.h" - -#define QAT_COMP_REQ_SIZE (sizeof(struct icp_qat_fw_comp_req)) -#define QAT_COMP_CTX_SIZE (QAT_COMP_REQ_SIZE * 2) - -static inline void qat_comp_create_req(void *ctx, void *req, u64 src, u32 slen, - u64 dst, u32 dlen, u64 opaque) -{ - struct icp_qat_fw_comp_req *fw_tmpl = ctx; - struct icp_qat_fw_comp_req *fw_req = req; - struct icp_qat_fw_comp_req_params *req_pars = &fw_req->comp_pars; - - memcpy(fw_req, fw_tmpl, sizeof(*fw_req)); - fw_req->comn_mid.src_data_addr = src; - fw_req->comn_mid.src_length = slen; - fw_req->comn_mid.dest_data_addr = dst; - fw_req->comn_mid.dst_length = dlen; - fw_req->comn_mid.opaque_data = opaque; - req_pars->comp_len = slen; - req_pars->out_buffer_sz = dlen; -} - -static inline void qat_comp_override_dst(void *req, u64 dst, u32 dlen) -{ - struct icp_qat_fw_comp_req *fw_req = req; - struct icp_qat_fw_comp_req_params *req_pars = &fw_req->comp_pars; - - fw_req->comn_mid.dest_data_addr = dst; - fw_req->comn_mid.dst_length = dlen; - req_pars->out_buffer_sz = dlen; -} - -static inline void qat_comp_create_compression_req(void *ctx, void *req, - u64 src, u32 slen, - u64 dst, u32 dlen, - u64 opaque) -{ - qat_comp_create_req(ctx, req, src, slen, dst, dlen, opaque); -} - -static inline void qat_comp_create_decompression_req(void *ctx, void *req, - u64 src, u32 slen, - u64 dst, u32 dlen, - u64 opaque) -{ - struct icp_qat_fw_comp_req *fw_tmpl = ctx; - - fw_tmpl++; - qat_comp_create_req(fw_tmpl, req, src, slen, dst, dlen, opaque); -} - -static inline u32 qat_comp_get_consumed_ctr(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - - return qat_resp->comp_resp_pars.input_byte_counter; -} - -static inline u32 qat_comp_get_produced_ctr(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - - return qat_resp->comp_resp_pars.output_byte_counter; -} - -static inline u32 qat_comp_get_produced_adler32(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - - return qat_resp->comp_resp_pars.crc.legacy.curr_adler_32; -} - -static inline u64 qat_comp_get_opaque(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - - return qat_resp->opaque_data; -} - -static inline s8 qat_comp_get_cmp_err(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - - return qat_resp->comn_resp.comn_error.cmp_err_code; -} - -static inline s8 qat_comp_get_xlt_err(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - - return qat_resp->comn_resp.comn_error.xlat_err_code; -} - -static inline s8 qat_comp_get_cmp_status(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - u8 stat_filed = qat_resp->comn_resp.comn_status; - - return ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(stat_filed); -} - -static inline s8 qat_comp_get_xlt_status(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - u8 stat_filed = qat_resp->comn_resp.comn_status; - - return ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(stat_filed); -} - -static inline u8 qat_comp_get_cmp_cnv_flag(void *resp) -{ - struct icp_qat_fw_comp_resp *qat_resp = resp; - u8 flags = qat_resp->comn_resp.hdr_flags; - - return ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(flags); -} - -#endif diff --git a/drivers/crypto/qat/qat_common/qat_compression.c b/drivers/crypto/qat/qat_common/qat_compression.c deleted file mode 100644 index 3f1f35283266..000000000000 --- a/drivers/crypto/qat/qat_common/qat_compression.c +++ /dev/null @@ -1,297 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_transport.h" -#include "adf_transport_access_macros.h" -#include "adf_cfg.h" -#include "adf_cfg_strings.h" -#include "qat_compression.h" -#include "icp_qat_fw.h" - -#define SEC ADF_KERNEL_SEC - -static struct service_hndl qat_compression; - -void qat_compression_put_instance(struct qat_compression_instance *inst) -{ - atomic_dec(&inst->refctr); - adf_dev_put(inst->accel_dev); -} - -static int qat_compression_free_instances(struct adf_accel_dev *accel_dev) -{ - struct qat_compression_instance *inst; - struct list_head *list_ptr, *tmp; - int i; - - list_for_each_safe(list_ptr, tmp, &accel_dev->compression_list) { - inst = list_entry(list_ptr, - struct qat_compression_instance, list); - - for (i = 0; i < atomic_read(&inst->refctr); i++) - qat_compression_put_instance(inst); - - if (inst->dc_tx) - adf_remove_ring(inst->dc_tx); - - if (inst->dc_rx) - adf_remove_ring(inst->dc_rx); - - list_del(list_ptr); - kfree(inst); - } - return 0; -} - -struct qat_compression_instance *qat_compression_get_instance_node(int node) -{ - struct qat_compression_instance *inst = NULL; - struct adf_accel_dev *accel_dev = NULL; - unsigned long best = ~0; - struct list_head *itr; - - list_for_each(itr, adf_devmgr_get_head()) { - struct adf_accel_dev *tmp_dev; - unsigned long ctr; - int tmp_dev_node; - - tmp_dev = list_entry(itr, struct adf_accel_dev, list); - tmp_dev_node = dev_to_node(&GET_DEV(tmp_dev)); - - if ((node == tmp_dev_node || tmp_dev_node < 0) && - adf_dev_started(tmp_dev) && !list_empty(&tmp_dev->compression_list)) { - ctr = atomic_read(&tmp_dev->ref_count); - if (best > ctr) { - accel_dev = tmp_dev; - best = ctr; - } - } - } - - if (!accel_dev) { - pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node); - /* Get any started device */ - list_for_each(itr, adf_devmgr_get_head()) { - struct adf_accel_dev *tmp_dev; - - tmp_dev = list_entry(itr, struct adf_accel_dev, list); - if (adf_dev_started(tmp_dev) && - !list_empty(&tmp_dev->compression_list)) { - accel_dev = tmp_dev; - break; - } - } - } - - if (!accel_dev) - return NULL; - - best = ~0; - list_for_each(itr, &accel_dev->compression_list) { - struct qat_compression_instance *tmp_inst; - unsigned long ctr; - - tmp_inst = list_entry(itr, struct qat_compression_instance, list); - ctr = atomic_read(&tmp_inst->refctr); - if (best > ctr) { - inst = tmp_inst; - best = ctr; - } - } - if (inst) { - if (adf_dev_get(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); - return NULL; - } - atomic_inc(&inst->refctr); - } - return inst; -} - -static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) -{ - struct qat_compression_instance *inst; - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - unsigned long num_inst, num_msg_dc; - unsigned long bank; - int msg_size; - int ret; - int i; - - INIT_LIST_HEAD(&accel_dev->compression_list); - strscpy(key, ADF_NUM_DC, sizeof(key)); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - return ret; - - ret = kstrtoul(val, 10, &num_inst); - if (ret) - return ret; - - for (i = 0; i < num_inst; i++) { - inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!inst) { - ret = -ENOMEM; - goto err; - } - - list_add_tail(&inst->list, &accel_dev->compression_list); - inst->id = i; - atomic_set(&inst->refctr, 0); - inst->accel_dev = accel_dev; - inst->build_deflate_ctx = GET_DC_OPS(accel_dev)->build_deflate_ctx; - - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - return ret; - - ret = kstrtoul(val, 10, &bank); - if (ret) - return ret; - - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - return ret; - - ret = kstrtoul(val, 10, &num_msg_dc); - if (ret) - return ret; - - msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, - msg_size, key, NULL, 0, &inst->dc_tx); - if (ret) - return ret; - - msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, - msg_size, key, qat_comp_alg_callback, 0, - &inst->dc_rx); - if (ret) - return ret; - - inst->dc_data = accel_dev->dc_data; - INIT_LIST_HEAD(&inst->backlog.list); - spin_lock_init(&inst->backlog.lock); - } - return 0; -err: - qat_compression_free_instances(accel_dev); - return ret; -} - -static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) -{ - struct device *dev = &GET_DEV(accel_dev); - dma_addr_t obuff_p = DMA_MAPPING_ERROR; - size_t ovf_buff_sz = QAT_COMP_MAX_SKID; - struct adf_dc_data *dc_data = NULL; - u8 *obuff = NULL; - - dc_data = devm_kzalloc(dev, sizeof(*dc_data), GFP_KERNEL); - if (!dc_data) - goto err; - - obuff = kzalloc_node(ovf_buff_sz, GFP_KERNEL, dev_to_node(dev)); - if (!obuff) - goto err; - - obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, obuff_p))) - goto err; - - dc_data->ovf_buff = obuff; - dc_data->ovf_buff_p = obuff_p; - dc_data->ovf_buff_sz = ovf_buff_sz; - - accel_dev->dc_data = dc_data; - - return 0; - -err: - accel_dev->dc_data = NULL; - kfree(obuff); - devm_kfree(dev, dc_data); - return -ENOMEM; -} - -static void qat_free_dc_data(struct adf_accel_dev *accel_dev) -{ - struct adf_dc_data *dc_data = accel_dev->dc_data; - struct device *dev = &GET_DEV(accel_dev); - - if (!dc_data) - return; - - dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, - DMA_FROM_DEVICE); - memset(dc_data->ovf_buff, 0, dc_data->ovf_buff_sz); - kfree(dc_data->ovf_buff); - devm_kfree(dev, dc_data); - accel_dev->dc_data = NULL; -} - -static int qat_compression_init(struct adf_accel_dev *accel_dev) -{ - int ret; - - ret = qat_compression_alloc_dc_data(accel_dev); - if (ret) - return ret; - - ret = qat_compression_create_instances(accel_dev); - if (ret) - qat_free_dc_data(accel_dev); - - return ret; -} - -static int qat_compression_shutdown(struct adf_accel_dev *accel_dev) -{ - qat_free_dc_data(accel_dev); - return qat_compression_free_instances(accel_dev); -} - -static int qat_compression_event_handler(struct adf_accel_dev *accel_dev, - enum adf_event event) -{ - int ret; - - switch (event) { - case ADF_EVENT_INIT: - ret = qat_compression_init(accel_dev); - break; - case ADF_EVENT_SHUTDOWN: - ret = qat_compression_shutdown(accel_dev); - break; - case ADF_EVENT_RESTARTING: - case ADF_EVENT_RESTARTED: - case ADF_EVENT_START: - case ADF_EVENT_STOP: - default: - ret = 0; - } - return ret; -} - -int qat_compression_register(void) -{ - memset(&qat_compression, 0, sizeof(qat_compression)); - qat_compression.event_hld = qat_compression_event_handler; - qat_compression.name = "qat_compression"; - return adf_service_register(&qat_compression); -} - -int qat_compression_unregister(void) -{ - return adf_service_unregister(&qat_compression); -} diff --git a/drivers/crypto/qat/qat_common/qat_compression.h b/drivers/crypto/qat/qat_common/qat_compression.h deleted file mode 100644 index aebac2302dcf..000000000000 --- a/drivers/crypto/qat/qat_common/qat_compression.h +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef _QAT_COMPRESSION_H_ -#define _QAT_COMPRESSION_H_ - -#include -#include -#include "adf_accel_devices.h" -#include "qat_algs_send.h" - -#define QAT_COMP_MAX_SKID 4096 - -struct qat_compression_instance { - struct adf_etr_ring_data *dc_tx; - struct adf_etr_ring_data *dc_rx; - struct adf_accel_dev *accel_dev; - struct list_head list; - unsigned long state; - int id; - atomic_t refctr; - struct qat_instance_backlog backlog; - struct adf_dc_data *dc_data; - void (*build_deflate_ctx)(void *ctx); -}; - -static inline bool adf_hw_dev_has_compression(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 mask = ~hw_device->accel_capabilities_mask; - - if (mask & ADF_ACCEL_CAPABILITIES_COMPRESSION) - return false; - - return true; -} - -#endif diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c deleted file mode 100644 index 40c8e74d1cf9..000000000000 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ /dev/null @@ -1,287 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_transport.h" -#include "adf_cfg.h" -#include "adf_cfg_strings.h" -#include "adf_gen2_hw_data.h" -#include "qat_crypto.h" -#include "icp_qat_fw.h" - -#define SEC ADF_KERNEL_SEC - -static struct service_hndl qat_crypto; - -void qat_crypto_put_instance(struct qat_crypto_instance *inst) -{ - atomic_dec(&inst->refctr); - adf_dev_put(inst->accel_dev); -} - -static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) -{ - struct qat_crypto_instance *inst, *tmp; - int i; - - list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) { - for (i = 0; i < atomic_read(&inst->refctr); i++) - qat_crypto_put_instance(inst); - - if (inst->sym_tx) - adf_remove_ring(inst->sym_tx); - - if (inst->sym_rx) - adf_remove_ring(inst->sym_rx); - - if (inst->pke_tx) - adf_remove_ring(inst->pke_tx); - - if (inst->pke_rx) - adf_remove_ring(inst->pke_rx); - - list_del(&inst->list); - kfree(inst); - } - return 0; -} - -struct qat_crypto_instance *qat_crypto_get_instance_node(int node) -{ - struct adf_accel_dev *accel_dev = NULL, *tmp_dev; - struct qat_crypto_instance *inst = NULL, *tmp_inst; - unsigned long best = ~0; - - list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) { - unsigned long ctr; - - if ((node == dev_to_node(&GET_DEV(tmp_dev)) || - dev_to_node(&GET_DEV(tmp_dev)) < 0) && - adf_dev_started(tmp_dev) && - !list_empty(&tmp_dev->crypto_list)) { - ctr = atomic_read(&tmp_dev->ref_count); - if (best > ctr) { - accel_dev = tmp_dev; - best = ctr; - } - } - } - - if (!accel_dev) { - pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node); - /* Get any started device */ - list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) { - if (adf_dev_started(tmp_dev) && - !list_empty(&tmp_dev->crypto_list)) { - accel_dev = tmp_dev; - break; - } - } - } - - if (!accel_dev) - return NULL; - - best = ~0; - list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) { - unsigned long ctr; - - ctr = atomic_read(&tmp_inst->refctr); - if (best > ctr) { - inst = tmp_inst; - best = ctr; - } - } - if (inst) { - if (adf_dev_get(accel_dev)) { - dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); - return NULL; - } - atomic_inc(&inst->refctr); - } - return inst; -} - -/** - * qat_crypto_vf_dev_config() - * create dev config required to create crypto inst. - * - * @accel_dev: Pointer to acceleration device. - * - * Function creates device configuration required to create - * asym, sym or, crypto instances - * - * Return: 0 on success, error code otherwise. - */ -int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev) -{ - u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map; - - if (ring_to_svc_map != ADF_GEN2_DEFAULT_RING_TO_SRV_MAP) { - dev_err(&GET_DEV(accel_dev), - "Unsupported ring/service mapping present on PF"); - return -EFAULT; - } - - return GET_HW_DATA(accel_dev)->dev_config(accel_dev); -} - -static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) -{ - unsigned long num_inst, num_msg_sym, num_msg_asym; - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - unsigned long sym_bank, asym_bank; - struct qat_crypto_instance *inst; - int msg_size; - int ret; - int i; - - INIT_LIST_HEAD(&accel_dev->crypto_list); - ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val); - if (ret) - return ret; - - ret = kstrtoul(val, 0, &num_inst); - if (ret) - return ret; - - for (i = 0; i < num_inst; i++) { - inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!inst) { - ret = -ENOMEM; - goto err; - } - - list_add_tail(&inst->list, &accel_dev->crypto_list); - inst->id = i; - atomic_set(&inst->refctr, 0); - inst->accel_dev = accel_dev; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - goto err; - - ret = kstrtoul(val, 10, &sym_bank); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - goto err; - - ret = kstrtoul(val, 10, &asym_bank); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - goto err; - - ret = kstrtoul(val, 10, &num_msg_sym); - if (ret) - goto err; - - num_msg_sym = num_msg_sym >> 1; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); - if (ret) - goto err; - - ret = kstrtoul(val, 10, &num_msg_asym); - if (ret) - goto err; - num_msg_asym = num_msg_asym >> 1; - - msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, - msg_size, key, NULL, 0, &inst->sym_tx); - if (ret) - goto err; - - msg_size = msg_size >> 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, - msg_size, key, NULL, 0, &inst->pke_tx); - if (ret) - goto err; - - msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, - msg_size, key, qat_alg_callback, 0, - &inst->sym_rx); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, - msg_size, key, qat_alg_asym_callback, 0, - &inst->pke_rx); - if (ret) - goto err; - - INIT_LIST_HEAD(&inst->backlog.list); - spin_lock_init(&inst->backlog.lock); - } - return 0; -err: - qat_crypto_free_instances(accel_dev); - return ret; -} - -static int qat_crypto_init(struct adf_accel_dev *accel_dev) -{ - if (qat_crypto_create_instances(accel_dev)) - return -EFAULT; - - return 0; -} - -static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev) -{ - return qat_crypto_free_instances(accel_dev); -} - -static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev, - enum adf_event event) -{ - int ret; - - switch (event) { - case ADF_EVENT_INIT: - ret = qat_crypto_init(accel_dev); - break; - case ADF_EVENT_SHUTDOWN: - ret = qat_crypto_shutdown(accel_dev); - break; - case ADF_EVENT_RESTARTING: - case ADF_EVENT_RESTARTED: - case ADF_EVENT_START: - case ADF_EVENT_STOP: - default: - ret = 0; - } - return ret; -} - -int qat_crypto_register(void) -{ - memset(&qat_crypto, 0, sizeof(qat_crypto)); - qat_crypto.event_hld = qat_crypto_event_handler; - qat_crypto.name = "qat_crypto"; - return adf_service_register(&qat_crypto); -} - -int qat_crypto_unregister(void) -{ - return adf_service_unregister(&qat_crypto); -} diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h deleted file mode 100644 index 6a0e961bb9dc..000000000000 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef _QAT_CRYPTO_INSTANCE_H_ -#define _QAT_CRYPTO_INSTANCE_H_ - -#include -#include -#include -#include "adf_accel_devices.h" -#include "icp_qat_fw_la.h" -#include "qat_algs_send.h" -#include "qat_bl.h" - -struct qat_crypto_instance { - struct adf_etr_ring_data *sym_tx; - struct adf_etr_ring_data *sym_rx; - struct adf_etr_ring_data *pke_tx; - struct adf_etr_ring_data *pke_rx; - struct adf_accel_dev *accel_dev; - struct list_head list; - unsigned long state; - int id; - atomic_t refctr; - struct qat_instance_backlog backlog; -}; - -struct qat_crypto_request; - -struct qat_crypto_request { - struct icp_qat_fw_la_bulk_req req; - union { - struct qat_alg_aead_ctx *aead_ctx; - struct qat_alg_skcipher_ctx *skcipher_ctx; - }; - union { - struct aead_request *aead_req; - struct skcipher_request *skcipher_req; - }; - struct qat_request_buffs buf; - void (*cb)(struct icp_qat_fw_la_resp *resp, - struct qat_crypto_request *req); - union { - struct { - __be64 iv_hi; - __be64 iv_lo; - }; - u8 iv[AES_BLOCK_SIZE]; - }; - bool encryption; - struct qat_alg_req alg_req; -}; - -static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 mask = ~hw_device->accel_capabilities_mask; - - if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) - return false; - if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) - return false; - if (mask & ADF_ACCEL_CAPABILITIES_AUTHENTICATION) - return false; - - return true; -} - -#endif diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c deleted file mode 100644 index cbb946a80076..000000000000 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ /dev/null @@ -1,1594 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include - -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "icp_qat_hal.h" -#include "icp_qat_uclo.h" - -#define BAD_REGADDR 0xffff -#define MAX_RETRY_TIMES 10000 -#define INIT_CTX_ARB_VALUE 0x0 -#define INIT_CTX_ENABLE_VALUE 0x0 -#define INIT_PC_VALUE 0x0 -#define INIT_WAKEUP_EVENTS_VALUE 0x1 -#define INIT_SIG_EVENTS_VALUE 0x1 -#define INIT_CCENABLE_VALUE 0x2000 -#define RST_CSR_QAT_LSB 20 -#define RST_CSR_AE_LSB 0 -#define MC_TIMESTAMP_ENABLE (0x1 << 7) - -#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \ - (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ - (~(1 << CE_REG_PAR_ERR_BITPOS))) -#define INSERT_IMMED_GPRA_CONST(inst, const_val) \ - (inst = ((inst & 0xFFFF00C03FFull) | \ - ((((const_val) << 12) & 0x0FF00000ull) | \ - (((const_val) << 10) & 0x0003FC00ull)))) -#define INSERT_IMMED_GPRB_CONST(inst, const_val) \ - (inst = ((inst & 0xFFFF00FFF00ull) | \ - ((((const_val) << 12) & 0x0FF00000ull) | \ - (((const_val) << 0) & 0x000000FFull)))) - -#define AE(handle, ae) ((handle)->hal_handle->aes[ae]) - -static const u64 inst_4b[] = { - 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, - 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, - 0x0A021000000ull -}; - -static const u64 inst[] = { - 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, - 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, - 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, - 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, - 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, - 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, - 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, - 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, - 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, - 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, - 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, - 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, - 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, - 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, - 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, - 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, - 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, - 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, - 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, - 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, - 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, - 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, -}; - -void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask) -{ - AE(handle, ae).live_ctx_mask = ctx_mask; -} - -#define CSR_RETRY_TIMES 500 -static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int csr) -{ - unsigned int iterations = CSR_RETRY_TIMES; - int value; - - do { - value = GET_AE_CSR(handle, ae, csr); - if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) - return value; - } while (iterations--); - - pr_err("QAT: Read CSR timeout\n"); - return 0; -} - -static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int csr, - unsigned int value) -{ - unsigned int iterations = CSR_RETRY_TIMES; - - do { - SET_AE_CSR(handle, ae, csr, value); - if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) - return 0; - } while (iterations--); - - pr_err("QAT: Write CSR Timeout\n"); - return -EFAULT; -} - -static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - unsigned int *events) -{ - unsigned int cur_ctx; - - cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT); - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); -} - -static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int cycles, - int chk_inactive) -{ - unsigned int base_cnt = 0, cur_cnt = 0; - unsigned int csr = (1 << ACS_ABO_BITPOS); - int times = MAX_RETRY_TIMES; - int elapsed_cycles = 0; - - base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); - base_cnt &= 0xffff; - while ((int)cycles > elapsed_cycles && times--) { - if (chk_inactive) - csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); - - cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); - cur_cnt &= 0xffff; - elapsed_cycles = cur_cnt - base_cnt; - - if (elapsed_cycles < 0) - elapsed_cycles += 0x10000; - - /* ensure at least 8 time cycles elapsed in wait_cycles */ - if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) - return 0; - } - if (times < 0) { - pr_err("QAT: wait_num_cycles time out\n"); - return -EFAULT; - } - return 0; -} - -#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) -#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) - -int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char mode) -{ - unsigned int csr, new_csr; - - if (mode != 4 && mode != 8) { - pr_err("QAT: bad ctx mode=%d\n", mode); - return -EINVAL; - } - - /* Sets the accelaration engine context mode to either four or eight */ - csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - csr = IGNORE_W1C_MASK & csr; - new_csr = (mode == 4) ? - SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : - CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); - return 0; -} - -int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char mode) -{ - unsigned int csr, new_csr; - - csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - csr &= IGNORE_W1C_MASK; - - new_csr = (mode) ? - SET_BIT(csr, CE_NN_MODE_BITPOS) : - CLR_BIT(csr, CE_NN_MODE_BITPOS); - - if (new_csr != csr) - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); - - return 0; -} - -int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, enum icp_qat_uof_regtype lm_type, - unsigned char mode) -{ - unsigned int csr, new_csr; - - csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - csr &= IGNORE_W1C_MASK; - switch (lm_type) { - case ICP_LMEM0: - new_csr = (mode) ? - SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : - CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); - break; - case ICP_LMEM1: - new_csr = (mode) ? - SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : - CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); - break; - case ICP_LMEM2: - new_csr = (mode) ? - SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : - CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); - break; - case ICP_LMEM3: - new_csr = (mode) ? - SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : - CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); - break; - default: - pr_err("QAT: lmType = 0x%x\n", lm_type); - return -EINVAL; - } - - if (new_csr != csr) - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); - return 0; -} - -void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char mode) -{ - unsigned int csr, new_csr; - - csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - csr &= IGNORE_W1C_MASK; - new_csr = (mode) ? - SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : - CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); - if (new_csr != csr) - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); -} - -static unsigned short qat_hal_get_reg_addr(unsigned int type, - unsigned short reg_num) -{ - unsigned short reg_addr; - - switch (type) { - case ICP_GPA_ABS: - case ICP_GPB_ABS: - reg_addr = 0x80 | (reg_num & 0x7f); - break; - case ICP_GPA_REL: - case ICP_GPB_REL: - reg_addr = reg_num & 0x1f; - break; - case ICP_SR_RD_REL: - case ICP_SR_WR_REL: - case ICP_SR_REL: - reg_addr = 0x180 | (reg_num & 0x1f); - break; - case ICP_SR_ABS: - reg_addr = 0x140 | ((reg_num & 0x3) << 1); - break; - case ICP_DR_RD_REL: - case ICP_DR_WR_REL: - case ICP_DR_REL: - reg_addr = 0x1c0 | (reg_num & 0x1f); - break; - case ICP_DR_ABS: - reg_addr = 0x100 | ((reg_num & 0x3) << 1); - break; - case ICP_NEIGH_REL: - reg_addr = 0x280 | (reg_num & 0x1f); - break; - case ICP_LMEM0: - reg_addr = 0x200; - break; - case ICP_LMEM1: - reg_addr = 0x220; - break; - case ICP_LMEM2: - reg_addr = 0x2c0; - break; - case ICP_LMEM3: - reg_addr = 0x2e0; - break; - case ICP_NO_DEST: - reg_addr = 0x300 | (reg_num & 0xff); - break; - default: - reg_addr = BAD_REGADDR; - break; - } - return reg_addr; -} - -void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) -{ - unsigned int reset_mask = handle->chip_info->icp_rst_mask; - unsigned int reset_csr = handle->chip_info->icp_rst_csr; - unsigned int csr_val; - - csr_val = GET_CAP_CSR(handle, reset_csr); - csr_val |= reset_mask; - SET_CAP_CSR(handle, reset_csr, csr_val); -} - -static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask, - unsigned int ae_csr, unsigned int csr_val) -{ - unsigned int ctx, cur_ctx; - - cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); - - for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!(ctx_mask & (1 << ctx))) - continue; - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); - } - - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); -} - -static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - unsigned int ae_csr) -{ - unsigned int cur_ctx, csr_val; - - cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr); - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); - - return csr_val; -} - -static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask, - unsigned int events) -{ - unsigned int ctx, cur_ctx; - - cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); - for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!(ctx_mask & (1 << ctx))) - continue; - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); - } - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); -} - -static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask, - unsigned int events) -{ - unsigned int ctx, cur_ctx; - - cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); - for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!(ctx_mask & (1 << ctx))) - continue; - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); - qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, - events); - } - qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); -} - -static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned int base_cnt, cur_cnt; - unsigned char ae; - int times = MAX_RETRY_TIMES; - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); - base_cnt &= 0xffff; - - do { - cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); - cur_cnt &= 0xffff; - } while (times-- && (cur_cnt == base_cnt)); - - if (times < 0) { - pr_err("QAT: AE%d is inactive!!\n", ae); - return -EFAULT; - } - } - - return 0; -} - -int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, - unsigned int ae) -{ - unsigned int enable = 0, active = 0; - - enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); - if ((enable & (0xff << CE_ENABLE_BITPOS)) || - (active & (1 << ACS_ABO_BITPOS))) - return 1; - else - return 0; -} - -static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned int misc_ctl_csr, misc_ctl; - unsigned char ae; - - misc_ctl_csr = handle->chip_info->misc_ctl_csr; - /* stop the timestamp timers */ - misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr); - if (misc_ctl & MC_TIMESTAMP_ENABLE) - SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl & - (~MC_TIMESTAMP_ENABLE)); - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); - qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); - } - /* start timestamp timers */ - SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); -} - -#define ESRAM_AUTO_TINIT BIT(2) -#define ESRAM_AUTO_TINIT_DONE BIT(3) -#define ESRAM_AUTO_INIT_USED_CYCLES (1640) -#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C -static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) -{ - void __iomem *csr_addr = - (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v + - ESRAM_AUTO_INIT_CSR_OFFSET); - unsigned int csr_val; - int times = 30; - - if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC) - return 0; - - csr_val = ADF_CSR_RD(csr_addr, 0); - if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) - return 0; - - csr_val = ADF_CSR_RD(csr_addr, 0); - csr_val |= ESRAM_AUTO_TINIT; - ADF_CSR_WR(csr_addr, 0, csr_val); - - do { - qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); - csr_val = ADF_CSR_RD(csr_addr, 0); - } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); - if (times < 0) { - pr_err("QAT: Fail to init eSram!\n"); - return -EFAULT; - } - return 0; -} - -#define SHRAM_INIT_CYCLES 2060 -int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) -{ - unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr; - unsigned int reset_mask = handle->chip_info->icp_rst_mask; - unsigned int reset_csr = handle->chip_info->icp_rst_csr; - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned char ae = 0; - unsigned int times = 100; - unsigned int csr_val; - - /* write to the reset csr */ - csr_val = GET_CAP_CSR(handle, reset_csr); - csr_val &= ~reset_mask; - do { - SET_CAP_CSR(handle, reset_csr, csr_val); - if (!(times--)) - goto out_err; - csr_val = GET_CAP_CSR(handle, reset_csr); - csr_val &= reset_mask; - } while (csr_val); - /* enable clock */ - csr_val = GET_CAP_CSR(handle, clk_csr); - csr_val |= reset_mask; - SET_CAP_CSR(handle, clk_csr, csr_val); - if (qat_hal_check_ae_alive(handle)) - goto out_err; - - /* Set undefined power-up/reset states to reasonable default values */ - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, - INIT_CTX_ENABLE_VALUE); - qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, - CTX_STS_INDIRECT, - handle->hal_handle->upc_mask & - INIT_PC_VALUE); - qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); - qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); - qat_hal_put_wakeup_event(handle, ae, - ICP_QAT_UCLO_AE_ALL_CTX, - INIT_WAKEUP_EVENTS_VALUE); - qat_hal_put_sig_event(handle, ae, - ICP_QAT_UCLO_AE_ALL_CTX, - INIT_SIG_EVENTS_VALUE); - } - if (qat_hal_init_esram(handle)) - goto out_err; - if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) - goto out_err; - qat_hal_reset_timestamp(handle); - - return 0; -out_err: - pr_err("QAT: failed to get device out of reset\n"); - return -EFAULT; -} - -static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask) -{ - unsigned int ctx; - - ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - ctx &= IGNORE_W1C_MASK & - (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); -} - -static u64 qat_hal_parity_64bit(u64 word) -{ - word ^= word >> 1; - word ^= word >> 2; - word ^= word >> 4; - word ^= word >> 8; - word ^= word >> 16; - word ^= word >> 32; - return word & 1; -} - -static u64 qat_hal_set_uword_ecc(u64 uword) -{ - u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, - bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, - bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, - bit6_mask = 0xdaf69a46910ULL; - - /* clear the ecc bits */ - uword &= ~(0x7fULL << 0x2C); - uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; - uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; - uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; - uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; - uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; - uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; - uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; - return uword; -} - -void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int uaddr, - unsigned int words_num, u64 *uword) -{ - unsigned int ustore_addr; - unsigned int i; - - ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); - uaddr |= UA_ECS; - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); - for (i = 0; i < words_num; i++) { - unsigned int uwrd_lo, uwrd_hi; - u64 tmp; - - tmp = qat_hal_set_uword_ecc(uword[i]); - uwrd_lo = (unsigned int)(tmp & 0xffffffff); - uwrd_hi = (unsigned int)(tmp >> 0x20); - qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); - qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); - } - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); -} - -static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask) -{ - unsigned int ctx; - - ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - ctx &= IGNORE_W1C_MASK; - ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; - ctx |= (ctx_mask << CE_ENABLE_BITPOS); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); -} - -static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned char ae; - unsigned short reg; - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { - qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, - reg, 0); - qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS, - reg, 0); - } - } -} - -static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned char ae; - unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; - int times = MAX_RETRY_TIMES; - unsigned int csr_val = 0; - unsigned int savctx = 0; - int ret = 0; - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); - csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); - qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); - csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - csr_val &= IGNORE_W1C_MASK; - if (handle->chip_info->nn) - csr_val |= CE_NN_MODE; - - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); - qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), - (u64 *)inst); - qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, - handle->hal_handle->upc_mask & - INIT_PC_VALUE); - savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); - qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); - qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); - qat_hal_wr_indr_csr(handle, ae, ctx_mask, - CTX_SIG_EVENTS_INDIRECT, 0); - qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); - qat_hal_enable_ctx(handle, ae, ctx_mask); - } - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - /* wait for AE to finish */ - do { - ret = qat_hal_wait_cycles(handle, ae, 20, 1); - } while (ret && times--); - - if (times < 0) { - pr_err("QAT: clear GPR of AE %d failed", ae); - return -EINVAL; - } - qat_hal_disable_ctx(handle, ae, ctx_mask); - qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, - savctx & ACS_ACNO); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, - INIT_CTX_ENABLE_VALUE); - qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, - handle->hal_handle->upc_mask & - INIT_PC_VALUE); - qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); - qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); - qat_hal_put_wakeup_event(handle, ae, ctx_mask, - INIT_WAKEUP_EVENTS_VALUE); - qat_hal_put_sig_event(handle, ae, ctx_mask, - INIT_SIG_EVENTS_VALUE); - } - return 0; -} - -static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, - struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); - unsigned int max_en_ae_id = 0; - struct adf_bar *sram_bar; - unsigned int csr_val = 0; - unsigned long ae_mask; - unsigned char ae = 0; - int ret = 0; - - handle->pci_dev = pci_info->pci_dev; - switch (handle->pci_dev->device) { - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - handle->chip_info->mmp_sram_size = 0; - handle->chip_info->nn = false; - handle->chip_info->lm2lm3 = true; - handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; - handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; - handle->chip_info->icp_rst_mask = 0x100015; - handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; - handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; - handle->chip_info->wakeup_event_val = 0x80000000; - handle->chip_info->fw_auth = true; - handle->chip_info->css_3k = true; - handle->chip_info->tgroup_share_ustore = true; - handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; - handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; - handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX; - handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX; - handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; - handle->chip_info->fcu_loaded_ae_pos = 0; - - handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX; - handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX; - handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX; - handle->hal_cap_ae_local_csr_addr_v = - (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v - + LOCAL_TO_XFER_REG_OFFSET); - break; - case PCI_DEVICE_ID_INTEL_QAT_C62X: - case PCI_DEVICE_ID_INTEL_QAT_C3XXX: - handle->chip_info->mmp_sram_size = 0; - handle->chip_info->nn = true; - handle->chip_info->lm2lm3 = false; - handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; - handle->chip_info->icp_rst_csr = ICP_RESET; - handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | - (hw_data->accel_mask << RST_CSR_QAT_LSB); - handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; - handle->chip_info->misc_ctl_csr = MISC_CONTROL; - handle->chip_info->wakeup_event_val = WAKEUP_EVENT; - handle->chip_info->fw_auth = true; - handle->chip_info->css_3k = false; - handle->chip_info->tgroup_share_ustore = false; - handle->chip_info->fcu_ctl_csr = FCU_CONTROL; - handle->chip_info->fcu_sts_csr = FCU_STATUS; - handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; - handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; - handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; - handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; - handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; - handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; - handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; - handle->hal_cap_ae_local_csr_addr_v = - (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v - + LOCAL_TO_XFER_REG_OFFSET); - break; - case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: - handle->chip_info->mmp_sram_size = 0x40000; - handle->chip_info->nn = true; - handle->chip_info->lm2lm3 = false; - handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; - handle->chip_info->icp_rst_csr = ICP_RESET; - handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | - (hw_data->accel_mask << RST_CSR_QAT_LSB); - handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; - handle->chip_info->misc_ctl_csr = MISC_CONTROL; - handle->chip_info->wakeup_event_val = WAKEUP_EVENT; - handle->chip_info->fw_auth = false; - handle->chip_info->css_3k = false; - handle->chip_info->tgroup_share_ustore = false; - handle->chip_info->fcu_ctl_csr = 0; - handle->chip_info->fcu_sts_csr = 0; - handle->chip_info->fcu_dram_addr_hi = 0; - handle->chip_info->fcu_dram_addr_lo = 0; - handle->chip_info->fcu_loaded_ae_csr = 0; - handle->chip_info->fcu_loaded_ae_pos = 0; - handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; - handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; - handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; - handle->hal_cap_ae_local_csr_addr_v = - (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v - + LOCAL_TO_XFER_REG_OFFSET); - break; - default: - ret = -EINVAL; - goto out_err; - } - - if (handle->chip_info->mmp_sram_size > 0) { - sram_bar = - &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; - handle->hal_sram_addr_v = sram_bar->virt_addr; - } - handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; - handle->hal_handle->ae_mask = hw_data->ae_mask; - handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; - handle->hal_handle->slice_mask = hw_data->accel_mask; - handle->cfg_ae_mask = ALL_AE_MASK; - /* create AE objects */ - handle->hal_handle->upc_mask = 0x1ffff; - handle->hal_handle->max_ustore = 0x4000; - - ae_mask = handle->hal_handle->ae_mask; - for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { - handle->hal_handle->aes[ae].free_addr = 0; - handle->hal_handle->aes[ae].free_size = - handle->hal_handle->max_ustore; - handle->hal_handle->aes[ae].ustore_size = - handle->hal_handle->max_ustore; - handle->hal_handle->aes[ae].live_ctx_mask = - ICP_QAT_UCLO_AE_ALL_CTX; - max_en_ae_id = ae; - } - handle->hal_handle->ae_max_num = max_en_ae_id + 1; - - /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); - csr_val |= 0x1; - qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); - } -out_err: - return ret; -} - -int qat_hal_init(struct adf_accel_dev *accel_dev) -{ - struct icp_qat_fw_loader_handle *handle; - int ret = 0; - - handle = kzalloc(sizeof(*handle), GFP_KERNEL); - if (!handle) - return -ENOMEM; - - handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); - if (!handle->hal_handle) { - ret = -ENOMEM; - goto out_hal_handle; - } - - handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL); - if (!handle->chip_info) { - ret = -ENOMEM; - goto out_chip_info; - } - - ret = qat_hal_chip_init(handle, accel_dev); - if (ret) { - dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n"); - goto out_err; - } - - /* take all AEs out of reset */ - ret = qat_hal_clr_reset(handle); - if (ret) { - dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); - goto out_err; - } - - qat_hal_clear_xfer(handle); - if (!handle->chip_info->fw_auth) { - ret = qat_hal_clear_gpr(handle); - if (ret) - goto out_err; - } - - accel_dev->fw_loader->fw_loader = handle; - return 0; - -out_err: - kfree(handle->chip_info); -out_chip_info: - kfree(handle->hal_handle); -out_hal_handle: - kfree(handle); - return ret; -} - -void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) -{ - if (!handle) - return; - kfree(handle->chip_info); - kfree(handle->hal_handle); - kfree(handle); -} - -int qat_hal_start(struct icp_qat_fw_loader_handle *handle) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - u32 wakeup_val = handle->chip_info->wakeup_event_val; - u32 fcu_ctl_csr, fcu_sts_csr; - unsigned int fcu_sts; - unsigned char ae; - u32 ae_ctr = 0; - int retry = 0; - - if (handle->chip_info->fw_auth) { - fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; - fcu_sts_csr = handle->chip_info->fcu_sts_csr; - ae_ctr = hweight32(ae_mask); - SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); - do { - msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); - if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) - return ae_ctr; - } while (retry++ < FW_AUTH_MAX_RETRY); - pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts); - return 0; - } else { - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val); - qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); - ae_ctr++; - } - return ae_ctr; - } -} - -void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask) -{ - if (!handle->chip_info->fw_auth) - qat_hal_disable_ctx(handle, ae, ctx_mask); -} - -void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int ctx_mask, unsigned int upc) -{ - qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, - handle->hal_handle->upc_mask & upc); -} - -static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int uaddr, - unsigned int words_num, u64 *uword) -{ - unsigned int i, uwrd_lo, uwrd_hi; - unsigned int ustore_addr, misc_control; - - misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); - qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, - misc_control & 0xfffffffb); - ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); - uaddr |= UA_ECS; - for (i = 0; i < words_num; i++) { - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); - uaddr++; - uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER); - uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER); - uword[i] = uwrd_hi; - uword[i] = (uword[i] << 0x20) | uwrd_lo; - } - qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); -} - -void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int uaddr, - unsigned int words_num, unsigned int *data) -{ - unsigned int i, ustore_addr; - - ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); - uaddr |= UA_ECS; - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); - for (i = 0; i < words_num; i++) { - unsigned int uwrd_lo, uwrd_hi, tmp; - - uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | - ((data[i] & 0xff00) << 2) | - (0x3 << 8) | (data[i] & 0xff); - uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); - uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8; - tmp = ((data[i] >> 0x10) & 0xffff); - uwrd_hi |= (hweight32(tmp) & 0x1) << 9; - qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); - qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); - } - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); -} - -#define MAX_EXEC_INST 100 -static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - u64 *micro_inst, unsigned int inst_num, - int code_off, unsigned int max_cycle, - unsigned int *endpc) -{ - unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0; - unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0; - unsigned int ind_t_index = 0, ind_t_index_byte = 0; - unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0; - unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0; - u64 savuwords[MAX_EXEC_INST]; - unsigned int ind_cnt_sig; - unsigned int ind_sig, act_sig; - unsigned int csr_val = 0, newcsr_val; - unsigned int savctx; - unsigned int savcc, wakeup_events, savpc; - unsigned int ctxarb_ctl, ctx_enables; - - if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) { - pr_err("QAT: invalid instruction num %d\n", inst_num); - return -EINVAL; - } - /* save current context */ - ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT); - ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT); - ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx, - INDIRECT_LM_ADDR_0_BYTE_INDEX); - ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, - INDIRECT_LM_ADDR_1_BYTE_INDEX); - if (handle->chip_info->lm2lm3) { - ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx, - LM_ADDR_2_INDIRECT); - ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx, - LM_ADDR_3_INDIRECT); - ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx, - INDIRECT_LM_ADDR_2_BYTE_INDEX); - ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx, - INDIRECT_LM_ADDR_3_BYTE_INDEX); - ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx, - INDIRECT_T_INDEX); - ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx, - INDIRECT_T_INDEX_BYTE_INDEX); - } - if (inst_num <= MAX_EXEC_INST) - qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); - qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); - savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT); - savpc = (savpc & handle->hal_handle->upc_mask) >> 0; - ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - ctx_enables &= IGNORE_W1C_MASK; - savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE); - savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); - ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); - ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx, - FUTURE_COUNT_SIGNAL_INDIRECT); - ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx, - CTX_SIG_EVENTS_INDIRECT); - act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE); - /* execute micro codes */ - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); - qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); - qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); - if (code_off) - qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); - qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); - qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); - qat_hal_enable_ctx(handle, ae, (1 << ctx)); - /* wait for micro codes to finish */ - if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) - return -EFAULT; - if (endpc) { - unsigned int ctx_status; - - ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx, - CTX_STS_INDIRECT); - *endpc = ctx_status & handle->hal_handle->upc_mask; - } - /* retore to saved context */ - qat_hal_disable_ctx(handle, ae, (1 << ctx)); - if (inst_num <= MAX_EXEC_INST) - qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); - qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, - handle->hal_handle->upc_mask & savpc); - csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); - newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); - qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); - qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); - qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); - qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), - LM_ADDR_0_INDIRECT, ind_lm_addr0); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), - LM_ADDR_1_INDIRECT, ind_lm_addr1); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), - INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), - INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); - if (handle->chip_info->lm2lm3) { - qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT, - ind_lm_addr2); - qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT, - ind_lm_addr3); - qat_hal_wr_indr_csr(handle, ae, BIT(ctx), - INDIRECT_LM_ADDR_2_BYTE_INDEX, - ind_lm_addr_byte2); - qat_hal_wr_indr_csr(handle, ae, BIT(ctx), - INDIRECT_LM_ADDR_3_BYTE_INDEX, - ind_lm_addr_byte3); - qat_hal_wr_indr_csr(handle, ae, BIT(ctx), - INDIRECT_T_INDEX, ind_t_index); - qat_hal_wr_indr_csr(handle, ae, BIT(ctx), - INDIRECT_T_INDEX_BYTE_INDEX, - ind_t_index_byte); - } - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), - FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); - qat_hal_wr_indr_csr(handle, ae, (1 << ctx), - CTX_SIG_EVENTS_INDIRECT, ind_sig); - qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); - - return 0; -} - -static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int *data) -{ - unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; - unsigned int ctxarb_cntl, ustore_addr, ctx_enables; - unsigned short reg_addr; - int status = 0; - u64 insts, savuword; - - reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); - if (reg_addr == BAD_REGADDR) { - pr_err("QAT: bad regaddr=0x%x\n", reg_addr); - return -EINVAL; - } - switch (reg_type) { - case ICP_GPA_REL: - insts = 0xA070000000ull | (reg_addr & 0x3ff); - break; - default: - insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10); - break; - } - savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); - ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); - ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - ctx_enables &= IGNORE_W1C_MASK; - if (ctx != (savctx & ACS_ACNO)) - qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, - ctx & ACS_ACNO); - qat_hal_get_uwords(handle, ae, 0, 1, &savuword); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); - ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); - uaddr = UA_ECS; - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); - insts = qat_hal_set_uword_ecc(insts); - uwrd_lo = (unsigned int)(insts & 0xffffffff); - uwrd_hi = (unsigned int)(insts >> 0x20); - qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); - qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); - /* delay for at least 8 cycles */ - qat_hal_wait_cycles(handle, ae, 0x8, 0); - /* - * read ALU output - * the instruction should have been executed - * prior to clearing the ECS in putUwords - */ - *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT); - qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); - qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); - if (ctx != (savctx & ACS_ACNO)) - qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, - savctx & ACS_ACNO); - qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); - - return status; -} - -static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int data) -{ - unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; - u64 insts[] = { - 0x0F440000000ull, - 0x0F040000000ull, - 0x0F0000C0300ull, - 0x0E000010000ull - }; - const int num_inst = ARRAY_SIZE(insts), code_off = 1; - const int imm_w1 = 0, imm_w0 = 1; - - dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); - if (dest_addr == BAD_REGADDR) { - pr_err("QAT: bad destAddr=0x%x\n", dest_addr); - return -EINVAL; - } - - data16lo = 0xffff & data; - data16hi = 0xffff & (data >> 0x10); - src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) - (0xff & data16hi)); - src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) - (0xff & data16lo)); - switch (reg_type) { - case ICP_GPA_REL: - insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | - ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); - insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | - ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); - break; - default: - insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | - ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); - - insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | - ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); - break; - } - - return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst, - code_off, num_inst * 0x5, NULL); -} - -int qat_hal_get_ins_num(void) -{ - return ARRAY_SIZE(inst_4b); -} - -static int qat_hal_concat_micro_code(u64 *micro_inst, - unsigned int inst_num, unsigned int size, - unsigned int addr, unsigned int *value) -{ - int i; - unsigned int cur_value; - const u64 *inst_arr; - int fixup_offset; - int usize = 0; - int orig_num; - - orig_num = inst_num; - cur_value = value[0]; - inst_arr = inst_4b; - usize = ARRAY_SIZE(inst_4b); - fixup_offset = inst_num; - for (i = 0; i < usize; i++) - micro_inst[inst_num++] = inst_arr[i]; - INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); - fixup_offset++; - INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); - fixup_offset++; - INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); - fixup_offset++; - INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); - - return inst_num - orig_num; -} - -static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - int *pfirst_exec, u64 *micro_inst, - unsigned int inst_num) -{ - int stat = 0; - unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; - unsigned int gprb0 = 0, gprb1 = 0; - - if (*pfirst_exec) { - qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); - qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); - qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); - qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); - qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); - *pfirst_exec = 0; - } - stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1, - inst_num * 0x5, NULL); - if (stat != 0) - return -EFAULT; - qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); - qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); - qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); - qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); - qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); - - return 0; -} - -int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, - struct icp_qat_uof_batch_init *lm_init_header) -{ - struct icp_qat_uof_batch_init *plm_init; - u64 *micro_inst_arry; - int micro_inst_num; - int alloc_inst_size; - int first_exec = 1; - int stat = 0; - - plm_init = lm_init_header->next; - alloc_inst_size = lm_init_header->size; - if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) - alloc_inst_size = handle->hal_handle->max_ustore; - micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64), - GFP_KERNEL); - if (!micro_inst_arry) - return -ENOMEM; - micro_inst_num = 0; - while (plm_init) { - unsigned int addr, *value, size; - - ae = plm_init->ae; - addr = plm_init->addr; - value = plm_init->value; - size = plm_init->size; - micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry, - micro_inst_num, - size, addr, value); - plm_init = plm_init->next; - } - /* exec micro codes */ - if (micro_inst_arry && micro_inst_num > 0) { - micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; - stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, - micro_inst_arry, - micro_inst_num); - } - kfree(micro_inst_arry); - return stat; -} - -static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int val) -{ - int status = 0; - unsigned int reg_addr; - unsigned int ctx_enables; - unsigned short mask; - unsigned short dr_offset = 0x10; - - ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - if (CE_INUSE_CONTEXTS & ctx_enables) { - if (ctx & 0x1) { - pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); - return -EINVAL; - } - mask = 0x1f; - dr_offset = 0x20; - } else { - mask = 0x0f; - } - if (reg_num & ~mask) - return -EINVAL; - reg_addr = reg_num + (ctx << 0x5); - switch (reg_type) { - case ICP_SR_RD_REL: - case ICP_SR_REL: - SET_AE_XFER(handle, ae, reg_addr, val); - break; - case ICP_DR_RD_REL: - case ICP_DR_REL: - SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); - break; - default: - status = -EINVAL; - break; - } - return status; -} - -static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int data) -{ - unsigned int gprval, ctx_enables; - unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, - data16low; - unsigned short reg_mask; - int status = 0; - u64 micro_inst[] = { - 0x0F440000000ull, - 0x0F040000000ull, - 0x0A000000000ull, - 0x0F0000C0300ull, - 0x0E000010000ull - }; - const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; - const unsigned short gprnum = 0, dly = num_inst * 0x5; - - ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - if (CE_INUSE_CONTEXTS & ctx_enables) { - if (ctx & 0x1) { - pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); - return -EINVAL; - } - reg_mask = (unsigned short)~0x1f; - } else { - reg_mask = (unsigned short)~0xf; - } - if (reg_num & reg_mask) - return -EINVAL; - xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); - if (xfr_addr == BAD_REGADDR) { - pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); - return -EINVAL; - } - status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); - if (status) { - pr_err("QAT: failed to read register"); - return status; - } - gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); - data16low = 0xffff & data; - data16hi = 0xffff & (data >> 0x10); - src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, - (unsigned short)(0xff & data16hi)); - src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, - (unsigned short)(0xff & data16low)); - micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | - ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); - micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | - ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); - micro_inst[0x2] = micro_inst[0x2] | - ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10); - status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst, - code_off, dly, NULL); - qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); - return status; -} - -static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx, - unsigned short nn, unsigned int val) -{ - unsigned int ctx_enables; - int stat = 0; - - ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - ctx_enables &= IGNORE_W1C_MASK; - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); - - stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); - qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); - return stat; -} - -static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle - *handle, unsigned char ae, - unsigned short absreg_num, - unsigned short *relreg, - unsigned char *ctx) -{ - unsigned int ctx_enables; - - ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); - if (ctx_enables & CE_INUSE_CONTEXTS) { - /* 4-ctx mode */ - *relreg = absreg_num & 0x1F; - *ctx = (absreg_num >> 0x4) & 0x6; - } else { - /* 8-ctx mode */ - *relreg = absreg_num & 0x0F; - *ctx = (absreg_num >> 0x4) & 0x7; - } - return 0; -} - -int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int regdata) -{ - int stat = 0; - unsigned short reg; - unsigned char ctx = 0; - enum icp_qat_uof_regtype type; - - if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) - return -EINVAL; - - do { - if (ctx_mask == 0) { - qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, - &ctx); - type = reg_type - 1; - } else { - reg = reg_num; - type = reg_type; - if (!test_bit(ctx, &ctx_mask)) - continue; - } - stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); - if (stat) { - pr_err("QAT: write gpr fail\n"); - return -EINVAL; - } - } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); - - return 0; -} - -int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int regdata) -{ - int stat = 0; - unsigned short reg; - unsigned char ctx = 0; - enum icp_qat_uof_regtype type; - - if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) - return -EINVAL; - - do { - if (ctx_mask == 0) { - qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, - &ctx); - type = reg_type - 3; - } else { - reg = reg_num; - type = reg_type; - if (!test_bit(ctx, &ctx_mask)) - continue; - } - stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, - regdata); - if (stat) { - pr_err("QAT: write wr xfer fail\n"); - return -EINVAL; - } - } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); - - return 0; -} - -int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_num, unsigned int regdata) -{ - int stat = 0; - unsigned short reg; - unsigned char ctx = 0; - enum icp_qat_uof_regtype type; - - if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) - return -EINVAL; - - do { - if (ctx_mask == 0) { - qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, - &ctx); - type = reg_type - 3; - } else { - reg = reg_num; - type = reg_type; - if (!test_bit(ctx, &ctx_mask)) - continue; - } - stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, - regdata); - if (stat) { - pr_err("QAT: write rd xfer fail\n"); - return -EINVAL; - } - } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); - - return 0; -} - -int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned long ctx_mask, - unsigned short reg_num, unsigned int regdata) -{ - int stat = 0; - unsigned char ctx; - if (!handle->chip_info->nn) { - dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n", - handle->pci_dev->device); - return -EINVAL; - } - - if (ctx_mask == 0) - return -EINVAL; - - for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!test_bit(ctx, &ctx_mask)) - continue; - stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); - if (stat) { - pr_err("QAT: write neigh error\n"); - return -EINVAL; - } - } - - return 0; -} diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c deleted file mode 100644 index 3ba8ca20b3d7..000000000000 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ /dev/null @@ -1,2133 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "icp_qat_uclo.h" -#include "icp_qat_hal.h" -#include "icp_qat_fw_loader_handle.h" - -#define UWORD_CPYBUF_SIZE 1024 -#define INVLD_UWORD 0xffffffffffull -#define PID_MINOR_REV 0xf -#define PID_MAJOR_REV (0xf << 4) - -static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, - unsigned int ae, unsigned int image_num) -{ - struct icp_qat_uclo_aedata *ae_data; - struct icp_qat_uclo_encapme *encap_image; - struct icp_qat_uclo_page *page = NULL; - struct icp_qat_uclo_aeslice *ae_slice = NULL; - - ae_data = &obj_handle->ae_data[ae]; - encap_image = &obj_handle->ae_uimage[image_num]; - ae_slice = &ae_data->ae_slices[ae_data->slice_num]; - ae_slice->encap_image = encap_image; - - if (encap_image->img_ptr) { - ae_slice->ctx_mask_assigned = - encap_image->img_ptr->ctx_assigned; - ae_data->eff_ustore_size = obj_handle->ustore_phy_size; - } else { - ae_slice->ctx_mask_assigned = 0; - } - ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL); - if (!ae_slice->region) - return -ENOMEM; - ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL); - if (!ae_slice->page) - goto out_err; - page = ae_slice->page; - page->encap_page = encap_image->page; - ae_slice->page->region = ae_slice->region; - ae_data->slice_num++; - return 0; -out_err: - kfree(ae_slice->region); - ae_slice->region = NULL; - return -ENOMEM; -} - -static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) -{ - unsigned int i; - - if (!ae_data) { - pr_err("QAT: bad argument, ae_data is NULL\n "); - return -EINVAL; - } - - for (i = 0; i < ae_data->slice_num; i++) { - kfree(ae_data->ae_slices[i].region); - ae_data->ae_slices[i].region = NULL; - kfree(ae_data->ae_slices[i].page); - ae_data->ae_slices[i].page = NULL; - } - return 0; -} - -static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, - unsigned int str_offset) -{ - if (!str_table->table_len || str_offset > str_table->table_len) - return NULL; - return (char *)(((uintptr_t)(str_table->strings)) + str_offset); -} - -static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) -{ - int maj = hdr->maj_ver & 0xff; - int min = hdr->min_ver & 0xff; - - if (hdr->file_id != ICP_QAT_UOF_FID) { - pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); - return -EINVAL; - } - if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { - pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", - maj, min); - return -EINVAL; - } - return 0; -} - -static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) -{ - int maj = suof_hdr->maj_ver & 0xff; - int min = suof_hdr->min_ver & 0xff; - - if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { - pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); - return -EINVAL; - } - if (suof_hdr->fw_type != 0) { - pr_err("QAT: unsupported firmware type\n"); - return -EINVAL; - } - if (suof_hdr->num_chunks <= 0x1) { - pr_err("QAT: SUOF chunk amount is incorrect\n"); - return -EINVAL; - } - if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { - pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", - maj, min); - return -EINVAL; - } - return 0; -} - -static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, - unsigned int addr, unsigned int *val, - unsigned int num_in_bytes) -{ - unsigned int outval; - unsigned char *ptr = (unsigned char *)val; - - while (num_in_bytes) { - memcpy(&outval, ptr, 4); - SRAM_WRITE(handle, addr, outval); - num_in_bytes -= 4; - ptr += 4; - addr += 4; - } -} - -static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned int addr, - unsigned int *val, - unsigned int num_in_bytes) -{ - unsigned int outval; - unsigned char *ptr = (unsigned char *)val; - - addr >>= 0x2; /* convert to uword address */ - - while (num_in_bytes) { - memcpy(&outval, ptr, 4); - qat_hal_wr_umem(handle, ae, addr++, 1, &outval); - num_in_bytes -= 4; - ptr += 4; - } -} - -static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, - struct icp_qat_uof_batch_init - *umem_init_header) -{ - struct icp_qat_uof_batch_init *umem_init; - - if (!umem_init_header) - return; - umem_init = umem_init_header->next; - while (umem_init) { - unsigned int addr, *value, size; - - ae = umem_init->ae; - addr = umem_init->addr; - value = umem_init->value; - size = umem_init->size; - qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); - umem_init = umem_init->next; - } -} - -static void -qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uof_batch_init **base) -{ - struct icp_qat_uof_batch_init *umem_init; - - umem_init = *base; - while (umem_init) { - struct icp_qat_uof_batch_init *pre; - - pre = umem_init; - umem_init = umem_init->next; - kfree(pre); - } - *base = NULL; -} - -static int qat_uclo_parse_num(char *str, unsigned int *num) -{ - char buf[16] = {0}; - unsigned long ae = 0; - int i; - - strncpy(buf, str, 15); - for (i = 0; i < 16; i++) { - if (!isdigit(buf[i])) { - buf[i] = '\0'; - break; - } - } - if ((kstrtoul(buf, 10, &ae))) - return -EFAULT; - - *num = (unsigned int)ae; - return 0; -} - -static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uof_initmem *init_mem, - unsigned int size_range, unsigned int *ae) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - char *str; - - if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { - pr_err("QAT: initmem is out of range"); - return -EINVAL; - } - if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { - pr_err("QAT: Memory scope for init_mem error\n"); - return -EINVAL; - } - str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); - if (!str) { - pr_err("QAT: AE name assigned in UOF init table is NULL\n"); - return -EINVAL; - } - if (qat_uclo_parse_num(str, ae)) { - pr_err("QAT: Parse num for AE number failed\n"); - return -EINVAL; - } - if (*ae >= ICP_QAT_UCLO_MAX_AE) { - pr_err("QAT: ae %d out of range\n", *ae); - return -EINVAL; - } - return 0; -} - -static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle - *handle, struct icp_qat_uof_initmem - *init_mem, unsigned int ae, - struct icp_qat_uof_batch_init - **init_tab_base) -{ - struct icp_qat_uof_batch_init *init_header, *tail; - struct icp_qat_uof_batch_init *mem_init, *tail_old; - struct icp_qat_uof_memvar_attr *mem_val_attr; - unsigned int i, flag = 0; - - mem_val_attr = - (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + - sizeof(struct icp_qat_uof_initmem)); - - init_header = *init_tab_base; - if (!init_header) { - init_header = kzalloc(sizeof(*init_header), GFP_KERNEL); - if (!init_header) - return -ENOMEM; - init_header->size = 1; - *init_tab_base = init_header; - flag = 1; - } - tail_old = init_header; - while (tail_old->next) - tail_old = tail_old->next; - tail = tail_old; - for (i = 0; i < init_mem->val_attr_num; i++) { - mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL); - if (!mem_init) - goto out_err; - mem_init->ae = ae; - mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; - mem_init->value = &mem_val_attr->value; - mem_init->size = 4; - mem_init->next = NULL; - tail->next = mem_init; - tail = mem_init; - init_header->size += qat_hal_get_ins_num(); - mem_val_attr++; - } - return 0; -out_err: - /* Do not free the list head unless we allocated it. */ - tail_old = tail_old->next; - if (flag) { - kfree(*init_tab_base); - *init_tab_base = NULL; - } - - while (tail_old) { - mem_init = tail_old->next; - kfree(tail_old); - tail_old = mem_init; - } - return -ENOMEM; -} - -static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uof_initmem *init_mem) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned int ae; - - if (qat_uclo_fetch_initmem_ae(handle, init_mem, - handle->chip_info->lm_size, &ae)) - return -EINVAL; - if (qat_uclo_create_batch_init_list(handle, init_mem, ae, - &obj_handle->lm_init_tab[ae])) - return -EINVAL; - return 0; -} - -static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uof_initmem *init_mem) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned int ae, ustore_size, uaddr, i; - struct icp_qat_uclo_aedata *aed; - - ustore_size = obj_handle->ustore_phy_size; - if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) - return -EINVAL; - if (qat_uclo_create_batch_init_list(handle, init_mem, ae, - &obj_handle->umem_init_tab[ae])) - return -EINVAL; - /* set the highest ustore address referenced */ - uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; - aed = &obj_handle->ae_data[ae]; - for (i = 0; i < aed->slice_num; i++) { - if (aed->ae_slices[i].encap_image->uwords_num < uaddr) - aed->ae_slices[i].encap_image->uwords_num = uaddr; - } - return 0; -} - -static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uof_initmem *init_mem) -{ - switch (init_mem->region) { - case ICP_QAT_UOF_LMEM_REGION: - if (qat_uclo_init_lmem_seg(handle, init_mem)) - return -EINVAL; - break; - case ICP_QAT_UOF_UMEM_REGION: - if (qat_uclo_init_umem_seg(handle, init_mem)) - return -EINVAL; - break; - default: - pr_err("QAT: initmem region error. region type=0x%x\n", - init_mem->region); - return -EINVAL; - } - return 0; -} - -static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uclo_encapme *image) -{ - unsigned int i; - struct icp_qat_uclo_encap_page *page; - struct icp_qat_uof_image *uof_image; - unsigned char ae; - unsigned int ustore_size; - unsigned int patt_pos; - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned long cfg_ae_mask = handle->cfg_ae_mask; - u64 *fill_data; - - uof_image = image->img_ptr; - fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64), - GFP_KERNEL); - if (!fill_data) - return -ENOMEM; - for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++) - memcpy(&fill_data[i], &uof_image->fill_pattern, - sizeof(u64)); - page = image->page; - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - unsigned long ae_assigned = uof_image->ae_assigned; - - if (!test_bit(ae, &ae_assigned)) - continue; - - if (!test_bit(ae, &cfg_ae_mask)) - continue; - - ustore_size = obj_handle->ae_data[ae].eff_ustore_size; - patt_pos = page->beg_addr_p + page->micro_words_num; - - qat_hal_wr_uwords(handle, (unsigned char)ae, 0, - page->beg_addr_p, &fill_data[0]); - qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, - ustore_size - patt_pos + 1, - &fill_data[page->beg_addr_p]); - } - kfree(fill_data); - return 0; -} - -static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) -{ - int i, ae; - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; - unsigned long ae_mask = handle->hal_handle->ae_mask; - - for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { - if (initmem->num_in_bytes) { - if (qat_uclo_init_ae_memory(handle, initmem)) - return -EINVAL; - } - initmem = (struct icp_qat_uof_initmem *)((uintptr_t)( - (uintptr_t)initmem + - sizeof(struct icp_qat_uof_initmem)) + - (sizeof(struct icp_qat_uof_memvar_attr) * - initmem->val_attr_num)); - } - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - if (qat_hal_batch_wr_lm(handle, ae, - obj_handle->lm_init_tab[ae])) { - pr_err("QAT: fail to batch init lmem for AE %d\n", ae); - return -EINVAL; - } - qat_uclo_cleanup_batch_init_list(handle, - &obj_handle->lm_init_tab[ae]); - qat_uclo_batch_wr_umem(handle, ae, - obj_handle->umem_init_tab[ae]); - qat_uclo_cleanup_batch_init_list(handle, - &obj_handle-> - umem_init_tab[ae]); - } - return 0; -} - -static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, - char *chunk_id, void *cur) -{ - int i; - struct icp_qat_uof_chunkhdr *chunk_hdr = - (struct icp_qat_uof_chunkhdr *) - ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); - - for (i = 0; i < obj_hdr->num_chunks; i++) { - if ((cur < (void *)&chunk_hdr[i]) && - !strncmp(chunk_hdr[i].chunk_id, chunk_id, - ICP_QAT_UOF_OBJID_LEN)) { - return &chunk_hdr[i]; - } - } - return NULL; -} - -static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) -{ - int i; - unsigned int topbit = 1 << 0xF; - unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); - - reg ^= inbyte << 0x8; - for (i = 0; i < 0x8; i++) { - if (reg & topbit) - reg = (reg << 1) ^ 0x1021; - else - reg <<= 1; - } - return reg & 0xFFFF; -} - -static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num) -{ - unsigned int chksum = 0; - - if (ptr) - while (num--) - chksum = qat_uclo_calc_checksum(chksum, *ptr++); - return chksum; -} - -static struct icp_qat_uclo_objhdr * -qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, - char *chunk_id) -{ - struct icp_qat_uof_filechunkhdr *file_chunk; - struct icp_qat_uclo_objhdr *obj_hdr; - char *chunk; - int i; - - file_chunk = (struct icp_qat_uof_filechunkhdr *) - (buf + sizeof(struct icp_qat_uof_filehdr)); - for (i = 0; i < file_hdr->num_chunks; i++) { - if (!strncmp(file_chunk->chunk_id, chunk_id, - ICP_QAT_UOF_OBJID_LEN)) { - chunk = buf + file_chunk->offset; - if (file_chunk->checksum != qat_uclo_calc_str_checksum( - chunk, file_chunk->size)) - break; - obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL); - if (!obj_hdr) - break; - obj_hdr->file_buff = chunk; - obj_hdr->checksum = file_chunk->checksum; - obj_hdr->size = file_chunk->size; - return obj_hdr; - } - file_chunk++; - } - return NULL; -} - -static int -qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, - struct icp_qat_uof_image *image) -{ - struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; - struct icp_qat_uof_objtable *neigh_reg_tab; - struct icp_qat_uof_code_page *code_page; - - code_page = (struct icp_qat_uof_code_page *) - ((char *)image + sizeof(struct icp_qat_uof_image)); - uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + - code_page->uc_var_tab_offset); - imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + - code_page->imp_var_tab_offset); - imp_expr_tab = (struct icp_qat_uof_objtable *) - (encap_uof_obj->beg_uof + - code_page->imp_expr_tab_offset); - if (uc_var_tab->entry_num || imp_var_tab->entry_num || - imp_expr_tab->entry_num) { - pr_err("QAT: UOF can't contain imported variable to be parsed\n"); - return -EINVAL; - } - neigh_reg_tab = (struct icp_qat_uof_objtable *) - (encap_uof_obj->beg_uof + - code_page->neigh_reg_tab_offset); - if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain neighbor register table\n"); - return -EINVAL; - } - if (image->numpages > 1) { - pr_err("QAT: UOF can't contain multiple pages\n"); - return -EINVAL; - } - if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use shared control store feature\n"); - return -EFAULT; - } - if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use reloadable feature\n"); - return -EFAULT; - } - return 0; -} - -static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj - *encap_uof_obj, - struct icp_qat_uof_image *img, - struct icp_qat_uclo_encap_page *page) -{ - struct icp_qat_uof_code_page *code_page; - struct icp_qat_uof_code_area *code_area; - struct icp_qat_uof_objtable *uword_block_tab; - struct icp_qat_uof_uword_block *uwblock; - int i; - - code_page = (struct icp_qat_uof_code_page *) - ((char *)img + sizeof(struct icp_qat_uof_image)); - page->def_page = code_page->def_page; - page->page_region = code_page->page_region; - page->beg_addr_v = code_page->beg_addr_v; - page->beg_addr_p = code_page->beg_addr_p; - code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + - code_page->code_area_offset); - page->micro_words_num = code_area->micro_words_num; - uword_block_tab = (struct icp_qat_uof_objtable *) - (encap_uof_obj->beg_uof + - code_area->uword_block_tab); - page->uwblock_num = uword_block_tab->entry_num; - uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + - sizeof(struct icp_qat_uof_objtable)); - page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; - for (i = 0; i < uword_block_tab->entry_num; i++) - page->uwblock[i].micro_words = - (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; -} - -static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, - struct icp_qat_uclo_encapme *ae_uimage, - int max_image) -{ - int i, j; - struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; - struct icp_qat_uof_image *image; - struct icp_qat_uof_objtable *ae_regtab; - struct icp_qat_uof_objtable *init_reg_sym_tab; - struct icp_qat_uof_objtable *sbreak_tab; - struct icp_qat_uof_encap_obj *encap_uof_obj = - &obj_handle->encap_uof_obj; - - for (j = 0; j < max_image; j++) { - chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, - ICP_QAT_UOF_IMAG, chunk_hdr); - if (!chunk_hdr) - break; - image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + - chunk_hdr->offset); - ae_regtab = (struct icp_qat_uof_objtable *) - (image->reg_tab_offset + - obj_handle->obj_hdr->file_buff); - ae_uimage[j].ae_reg_num = ae_regtab->entry_num; - ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *) - (((char *)ae_regtab) + - sizeof(struct icp_qat_uof_objtable)); - init_reg_sym_tab = (struct icp_qat_uof_objtable *) - (image->init_reg_sym_tab + - obj_handle->obj_hdr->file_buff); - ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; - ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *) - (((char *)init_reg_sym_tab) + - sizeof(struct icp_qat_uof_objtable)); - sbreak_tab = (struct icp_qat_uof_objtable *) - (image->sbreak_tab + obj_handle->obj_hdr->file_buff); - ae_uimage[j].sbreak_num = sbreak_tab->entry_num; - ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *) - (((char *)sbreak_tab) + - sizeof(struct icp_qat_uof_objtable)); - ae_uimage[j].img_ptr = image; - if (qat_uclo_check_image_compat(encap_uof_obj, image)) - goto out_err; - ae_uimage[j].page = - kzalloc(sizeof(struct icp_qat_uclo_encap_page), - GFP_KERNEL); - if (!ae_uimage[j].page) - goto out_err; - qat_uclo_map_image_page(encap_uof_obj, image, - ae_uimage[j].page); - } - return j; -out_err: - for (i = 0; i < j; i++) - kfree(ae_uimage[i].page); - return 0; -} - -static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) -{ - int i, ae; - int mflag = 0; - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned long cfg_ae_mask = handle->cfg_ae_mask; - - for_each_set_bit(ae, &ae_mask, max_ae) { - if (!test_bit(ae, &cfg_ae_mask)) - continue; - - for (i = 0; i < obj_handle->uimage_num; i++) { - unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned; - - if (!test_bit(ae, &ae_assigned)) - continue; - mflag = 1; - if (qat_uclo_init_ae_data(obj_handle, ae, i)) - return -EINVAL; - } - } - if (!mflag) { - pr_err("QAT: uimage uses AE not set\n"); - return -EINVAL; - } - return 0; -} - -static struct icp_qat_uof_strtable * -qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, - char *tab_name, struct icp_qat_uof_strtable *str_table) -{ - struct icp_qat_uof_chunkhdr *chunk_hdr; - - chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *) - obj_hdr->file_buff, tab_name, NULL); - if (chunk_hdr) { - int hdr_size; - - memcpy(&str_table->table_len, obj_hdr->file_buff + - chunk_hdr->offset, sizeof(str_table->table_len)); - hdr_size = (char *)&str_table->strings - (char *)str_table; - str_table->strings = (uintptr_t)obj_hdr->file_buff + - chunk_hdr->offset + hdr_size; - return str_table; - } - return NULL; -} - -static void -qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, - struct icp_qat_uclo_init_mem_table *init_mem_tab) -{ - struct icp_qat_uof_chunkhdr *chunk_hdr; - - chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, - ICP_QAT_UOF_IMEM, NULL); - if (chunk_hdr) { - memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + - chunk_hdr->offset, sizeof(unsigned int)); - init_mem_tab->init_mem = (struct icp_qat_uof_initmem *) - (encap_uof_obj->beg_uof + chunk_hdr->offset + - sizeof(unsigned int)); - } -} - -static unsigned int -qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) -{ - switch (handle->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: - return ICP_QAT_AC_895XCC_DEV_TYPE; - case PCI_DEVICE_ID_INTEL_QAT_C62X: - return ICP_QAT_AC_C62X_DEV_TYPE; - case PCI_DEVICE_ID_INTEL_QAT_C3XXX: - return ICP_QAT_AC_C3XXX_DEV_TYPE; - case ADF_4XXX_PCI_DEVICE_ID: - case ADF_401XX_PCI_DEVICE_ID: - case ADF_402XX_PCI_DEVICE_ID: - return ICP_QAT_AC_4XXX_A_DEV_TYPE; - default: - pr_err("QAT: unsupported device 0x%x\n", - handle->pci_dev->device); - return 0; - } -} - -static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) -{ - unsigned int maj_ver, prod_type = obj_handle->prod_type; - - if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { - pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", - obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, - prod_type); - return -EINVAL; - } - maj_ver = obj_handle->prod_rev & 0xff; - if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || - obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { - pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); - return -EINVAL; - } - return 0; -} - -static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, - enum icp_qat_uof_regtype reg_type, - unsigned short reg_addr, unsigned int value) -{ - switch (reg_type) { - case ICP_GPA_ABS: - case ICP_GPB_ABS: - ctx_mask = 0; - fallthrough; - case ICP_GPA_REL: - case ICP_GPB_REL: - return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, - reg_addr, value); - case ICP_SR_ABS: - case ICP_DR_ABS: - case ICP_SR_RD_ABS: - case ICP_DR_RD_ABS: - ctx_mask = 0; - fallthrough; - case ICP_SR_REL: - case ICP_DR_REL: - case ICP_SR_RD_REL: - case ICP_DR_RD_REL: - return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type, - reg_addr, value); - case ICP_SR_WR_ABS: - case ICP_DR_WR_ABS: - ctx_mask = 0; - fallthrough; - case ICP_SR_WR_REL: - case ICP_DR_WR_REL: - return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, - reg_addr, value); - case ICP_NEIGH_REL: - return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); - default: - pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); - return -EFAULT; - } - return 0; -} - -static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, - unsigned int ae, - struct icp_qat_uclo_encapme *encap_ae) -{ - unsigned int i; - unsigned char ctx_mask; - struct icp_qat_uof_init_regsym *init_regsym; - - if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == - ICP_QAT_UCLO_MAX_CTX) - ctx_mask = 0xff; - else - ctx_mask = 0x55; - - for (i = 0; i < encap_ae->init_regsym_num; i++) { - unsigned int exp_res; - - init_regsym = &encap_ae->init_regsym[i]; - exp_res = init_regsym->value; - switch (init_regsym->init_type) { - case ICP_QAT_UOF_INIT_REG: - qat_uclo_init_reg(handle, ae, ctx_mask, - (enum icp_qat_uof_regtype) - init_regsym->reg_type, - (unsigned short)init_regsym->reg_addr, - exp_res); - break; - case ICP_QAT_UOF_INIT_REG_CTX: - /* check if ctx is appropriate for the ctxMode */ - if (!((1 << init_regsym->ctx) & ctx_mask)) { - pr_err("QAT: invalid ctx num = 0x%x\n", - init_regsym->ctx); - return -EINVAL; - } - qat_uclo_init_reg(handle, ae, - (unsigned char) - (1 << init_regsym->ctx), - (enum icp_qat_uof_regtype) - init_regsym->reg_type, - (unsigned short)init_regsym->reg_addr, - exp_res); - break; - case ICP_QAT_UOF_INIT_EXPR: - pr_err("QAT: INIT_EXPR feature not supported\n"); - return -EINVAL; - case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: - pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); - return -EINVAL; - default: - break; - } - } - return 0; -} - -static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned long ae_mask = handle->hal_handle->ae_mask; - struct icp_qat_uclo_aedata *aed; - unsigned int s, ae; - - if (obj_handle->global_inited) - return 0; - if (obj_handle->init_mem_tab.entry_num) { - if (qat_uclo_init_memory(handle)) { - pr_err("QAT: initialize memory failed\n"); - return -EINVAL; - } - } - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - aed = &obj_handle->ae_data[ae]; - for (s = 0; s < aed->slice_num; s++) { - if (!aed->ae_slices[s].encap_image) - continue; - if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) - return -EINVAL; - } - } - obj_handle->global_inited = 1; - return 0; -} - -static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uclo_objhandle *obj_handle, - unsigned char ae, - struct icp_qat_uof_image *uof_image) -{ - unsigned char mode; - int ret; - - mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); - return ret; - } - if (handle->chip_info->nn) { - mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_nn_mode(handle, ae, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return ret; - } - } - mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); - return ret; - } - mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); - return ret; - } - if (handle->chip_info->lm2lm3) { - mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); - return ret; - } - mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); - return ret; - } - mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); - qat_hal_set_ae_tindex_mode(handle, ae, mode); - } - return 0; -} - -static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_uof_image *uof_image; - struct icp_qat_uclo_aedata *ae_data; - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned long cfg_ae_mask = handle->cfg_ae_mask; - unsigned char ae, s; - int error; - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - if (!test_bit(ae, &cfg_ae_mask)) - continue; - - ae_data = &obj_handle->ae_data[ae]; - for (s = 0; s < min_t(unsigned int, ae_data->slice_num, - ICP_QAT_UCLO_MAX_CTX); s++) { - if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) - continue; - uof_image = ae_data->ae_slices[s].encap_image->img_ptr; - error = qat_hal_set_modes(handle, obj_handle, ae, - uof_image); - if (error) - return error; - } - } - return 0; -} - -static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - struct icp_qat_uclo_encapme *image; - int a; - - for (a = 0; a < obj_handle->uimage_num; a++) { - image = &obj_handle->ae_uimage[a]; - image->uwords_num = image->page->beg_addr_p + - image->page->micro_words_num; - } -} - -static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned int ae; - - obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; - obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) - obj_handle->obj_hdr->file_buff; - obj_handle->uword_in_bytes = 6; - obj_handle->prod_type = qat_uclo_get_dev_type(handle); - obj_handle->prod_rev = PID_MAJOR_REV | - (PID_MINOR_REV & handle->hal_handle->revision_id); - if (qat_uclo_check_uof_compat(obj_handle)) { - pr_err("QAT: UOF incompatible\n"); - return -EINVAL; - } - obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), - GFP_KERNEL); - if (!obj_handle->uword_buf) - return -ENOMEM; - obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE; - if (!obj_handle->obj_hdr->file_buff || - !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, - &obj_handle->str_table)) { - pr_err("QAT: UOF doesn't have effective images\n"); - goto out_err; - } - obj_handle->uimage_num = - qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, - ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); - if (!obj_handle->uimage_num) - goto out_err; - if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { - pr_err("QAT: Bad object\n"); - goto out_check_uof_aemask_err; - } - qat_uclo_init_uword_num(handle); - qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, - &obj_handle->init_mem_tab); - if (qat_uclo_set_ae_mode(handle)) - goto out_check_uof_aemask_err; - return 0; -out_check_uof_aemask_err: - for (ae = 0; ae < obj_handle->uimage_num; ae++) - kfree(obj_handle->ae_uimage[ae].page); -out_err: - kfree(obj_handle->uword_buf); - return -EFAULT; -} - -static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_suof_filehdr *suof_ptr, - int suof_size) -{ - unsigned int check_sum = 0; - unsigned int min_ver_offset = 0; - struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; - - suof_handle->file_id = ICP_QAT_SUOF_FID; - suof_handle->suof_buf = (char *)suof_ptr; - suof_handle->suof_size = suof_size; - min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, - min_ver); - check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, - min_ver_offset); - if (check_sum != suof_ptr->check_sum) { - pr_err("QAT: incorrect SUOF checksum\n"); - return -EINVAL; - } - suof_handle->check_sum = suof_ptr->check_sum; - suof_handle->min_ver = suof_ptr->min_ver; - suof_handle->maj_ver = suof_ptr->maj_ver; - suof_handle->fw_type = suof_ptr->fw_type; - return 0; -} - -static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_suof_img_hdr *suof_img_hdr, - struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) -{ - struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; - struct icp_qat_simg_ae_mode *ae_mode; - struct icp_qat_suof_objhdr *suof_objhdr; - - suof_img_hdr->simg_buf = (suof_handle->suof_buf + - suof_chunk_hdr->offset + - sizeof(*suof_objhdr)); - suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t) - (suof_handle->suof_buf + - suof_chunk_hdr->offset))->img_length; - - suof_img_hdr->css_header = suof_img_hdr->simg_buf; - suof_img_hdr->css_key = (suof_img_hdr->css_header + - sizeof(struct icp_qat_css_hdr)); - suof_img_hdr->css_signature = suof_img_hdr->css_key + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); - suof_img_hdr->css_simg = suof_img_hdr->css_signature + - ICP_QAT_CSS_SIGNATURE_LEN(handle); - - ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); - suof_img_hdr->ae_mask = ae_mode->ae_mask; - suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; - suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; - suof_img_hdr->fw_type = ae_mode->fw_type; -} - -static void -qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, - struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) -{ - char **sym_str = (char **)&suof_handle->sym_str; - unsigned int *sym_size = &suof_handle->sym_size; - struct icp_qat_suof_strtable *str_table_obj; - - *sym_size = *(unsigned int *)(uintptr_t) - (suof_chunk_hdr->offset + suof_handle->suof_buf); - *sym_str = (char *)(uintptr_t) - (suof_handle->suof_buf + suof_chunk_hdr->offset + - sizeof(str_table_obj->tab_length)); -} - -static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_suof_img_hdr *img_hdr) -{ - struct icp_qat_simg_ae_mode *img_ae_mode = NULL; - unsigned int prod_rev, maj_ver, prod_type; - - prod_type = qat_uclo_get_dev_type(handle); - img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg; - prod_rev = PID_MAJOR_REV | - (PID_MINOR_REV & handle->hal_handle->revision_id); - if (img_ae_mode->dev_type != prod_type) { - pr_err("QAT: incompatible product type %x\n", - img_ae_mode->dev_type); - return -EINVAL; - } - maj_ver = prod_rev & 0xff; - if (maj_ver > img_ae_mode->devmax_ver || - maj_ver < img_ae_mode->devmin_ver) { - pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); - return -EINVAL; - } - return 0; -} - -static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; - - kfree(sobj_handle->img_table.simg_hdr); - sobj_handle->img_table.simg_hdr = NULL; - kfree(handle->sobj_handle); - handle->sobj_handle = NULL; -} - -static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, - unsigned int img_id, unsigned int num_simgs) -{ - struct icp_qat_suof_img_hdr img_header; - - if (img_id != num_simgs - 1) { - memcpy(&img_header, &suof_img_hdr[num_simgs - 1], - sizeof(*suof_img_hdr)); - memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], - sizeof(*suof_img_hdr)); - memcpy(&suof_img_hdr[img_id], &img_header, - sizeof(*suof_img_hdr)); - } -} - -static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_suof_filehdr *suof_ptr, - int suof_size) -{ - struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; - struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; - struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; - int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; - unsigned int i = 0; - struct icp_qat_suof_img_hdr img_header; - - if (!suof_ptr || suof_size == 0) { - pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); - return -EINVAL; - } - if (qat_uclo_check_suof_format(suof_ptr)) - return -EINVAL; - ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); - if (ret) - return ret; - suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *) - ((uintptr_t)suof_ptr + sizeof(*suof_ptr)); - - qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); - suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; - - if (suof_handle->img_table.num_simgs != 0) { - suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs, - sizeof(img_header), - GFP_KERNEL); - if (!suof_img_hdr) - return -ENOMEM; - suof_handle->img_table.simg_hdr = suof_img_hdr; - - for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle, &suof_img_hdr[i], - &suof_chunk_hdr[1 + i]); - ret = qat_uclo_check_simg_compat(handle, - &suof_img_hdr[i]); - if (ret) - return ret; - suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; - if ((suof_img_hdr[i].ae_mask & 0x1) != 0) - ae0_img = i; - } - - if (!handle->chip_info->tgroup_share_ustore) { - qat_uclo_tail_img(suof_img_hdr, ae0_img, - suof_handle->img_table.num_simgs); - } - } - return 0; -} - -#define ADD_ADDR(high, low) ((((u64)high) << 32) + low) -#define BITS_IN_DWORD 32 - -static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_fw_auth_desc *desc) -{ - u32 fcu_sts, retry = 0; - u32 fcu_ctl_csr, fcu_sts_csr; - u32 fcu_dram_hi_csr, fcu_dram_lo_csr; - u64 bus_addr; - - bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - - sizeof(struct icp_qat_auth_chunk); - - fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; - fcu_sts_csr = handle->chip_info->fcu_sts_csr; - fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; - fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; - - SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); - SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); - SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); - - do { - msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); - if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) - goto auth_fail; - if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) - if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) - return 0; - } while (retry++ < FW_AUTH_MAX_RETRY); -auth_fail: - pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", - fcu_sts & FCU_AUTH_STS_MASK, retry); - return -EINVAL; -} - -static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, - int imgid) -{ - struct icp_qat_suof_handle *sobj_handle; - - if (!handle->chip_info->tgroup_share_ustore) - return false; - - sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; - if (handle->hal_handle->admin_ae_mask & - sobj_handle->img_table.simg_hdr[imgid].ae_mask) - return false; - - return true; -} - -static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_fw_auth_desc *desc) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned long desc_ae_mask = desc->ae_mask; - u32 fcu_sts, ae_broadcast_mask = 0; - u32 fcu_loaded_csr, ae_loaded; - u32 fcu_sts_csr, fcu_ctl_csr; - unsigned int ae, retry = 0; - - if (handle->chip_info->tgroup_share_ustore) { - fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; - fcu_sts_csr = handle->chip_info->fcu_sts_csr; - fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; - } else { - pr_err("Chip 0x%x doesn't support broadcast load\n", - handle->pci_dev->device); - return -EINVAL; - } - - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { - pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); - return -EINVAL; - } - - if (test_bit(ae, &desc_ae_mask)) - ae_broadcast_mask |= 1 << ae; - } - - if (ae_broadcast_mask) { - SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, - ae_broadcast_mask); - - SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); - - do { - msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); - fcu_sts &= FCU_AUTH_STS_MASK; - - if (fcu_sts == FCU_STS_LOAD_FAIL) { - pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); - return -EINVAL; - } else if (fcu_sts == FCU_STS_LOAD_DONE) { - ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); - ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; - - if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) - break; - } - } while (retry++ < FW_AUTH_MAX_RETRY); - - if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: broadcast load failed timeout %d\n", retry); - return -EINVAL; - } - } - return 0; -} - -static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, - struct icp_firml_dram_desc *dram_desc, - unsigned int size) -{ - void *vptr; - dma_addr_t ptr; - - vptr = dma_alloc_coherent(&handle->pci_dev->dev, - size, &ptr, GFP_KERNEL); - if (!vptr) - return -ENOMEM; - dram_desc->dram_base_addr_v = vptr; - dram_desc->dram_bus_addr = ptr; - dram_desc->dram_size = size; - return 0; -} - -static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, - struct icp_firml_dram_desc *dram_desc) -{ - if (handle && dram_desc && dram_desc->dram_base_addr_v) { - dma_free_coherent(&handle->pci_dev->dev, - (size_t)(dram_desc->dram_size), - dram_desc->dram_base_addr_v, - dram_desc->dram_bus_addr); - } - - if (dram_desc) - memset(dram_desc, 0, sizeof(*dram_desc)); -} - -static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_fw_auth_desc **desc) -{ - struct icp_firml_dram_desc dram_desc; - - if (*desc) { - dram_desc.dram_base_addr_v = *desc; - dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_bus_addr; - dram_desc.dram_size = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_size; - qat_uclo_simg_free(handle, &dram_desc); - } -} - -static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, - unsigned int fw_type) -{ - char *fw_type_name = fw_type ? "MMP" : "AE"; - unsigned int css_dword_size = sizeof(u32); - - if (handle->chip_info->fw_auth) { - struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); - - if ((css_hdr->header_len * css_dword_size) != header_len) - goto err; - if ((css_hdr->size * css_dword_size) != size) - goto err; - if (fw_type != css_hdr->fw_type) - goto err; - if (size <= header_len) - goto err; - size -= header_len; - } - - if (fw_type == CSS_AE_FIRMWARE) { - if (size < sizeof(struct icp_qat_simg_ae_mode *) + - ICP_QAT_SIMG_AE_INIT_SEQ_LEN) - goto err; - if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) - goto err; - } else if (fw_type == CSS_MMP_FIRMWARE) { - if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) - goto err; - } else { - pr_err("QAT: Unsupported firmware type\n"); - return -EINVAL; - } - return 0; - -err: - pr_err("QAT: Invalid %s firmware image\n", fw_type_name); - return -EINVAL; -} - -static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, - struct icp_qat_fw_auth_desc **desc) -{ - struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - struct icp_qat_fw_auth_desc *auth_desc; - struct icp_qat_auth_chunk *auth_chunk; - u64 virt_addr, bus_addr, virt_base; - unsigned int length, simg_offset = sizeof(*auth_chunk); - struct icp_qat_simg_ae_mode *simg_ae_mode; - struct icp_firml_dram_desc img_desc; - - if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) { - pr_err("QAT: error, input image size overflow %d\n", size); - return -EINVAL; - } - length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? - ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : - size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; - if (qat_uclo_simg_alloc(handle, &img_desc, length)) { - pr_err("QAT: error, allocate continuous dram fail\n"); - return -ENOMEM; - } - - auth_chunk = img_desc.dram_base_addr_v; - auth_chunk->chunk_size = img_desc.dram_size; - auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; - virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; - bus_addr = img_desc.dram_bus_addr + simg_offset; - auth_desc = img_desc.dram_base_addr_v; - auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->css_hdr_low = (unsigned int)bus_addr; - virt_addr = virt_base; - - memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); - /* pub key */ - bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + - sizeof(*css_hdr); - virt_addr = virt_addr + sizeof(*css_hdr); - - auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->fwsk_pub_low = (unsigned int)bus_addr; - - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); - /* padding */ - memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), - 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); - - /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_PAD_LEN(handle)), - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), - sizeof(unsigned int)); - - /* signature */ - bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, - auth_desc->fwsk_pub_low) + - ICP_QAT_CSS_FWSK_PUB_LEN(handle); - virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); - auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->signature_low = (unsigned int)bus_addr; - - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), - ICP_QAT_CSS_SIGNATURE_LEN(handle)); - - bus_addr = ADD_ADDR(auth_desc->signature_high, - auth_desc->signature_low) + - ICP_QAT_CSS_SIGNATURE_LEN(handle); - virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - - auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); - auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), - auth_desc->img_len); - virt_addr = virt_base; - /* AE firmware */ - if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == - CSS_AE_FIRMWARE) { - auth_desc->img_ae_mode_data_high = auth_desc->img_high; - auth_desc->img_ae_mode_data_low = auth_desc->img_low; - bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, - auth_desc->img_ae_mode_data_low) + - sizeof(struct icp_qat_simg_ae_mode); - - auth_desc->img_ae_init_data_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); - auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; - bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; - auth_desc->img_ae_insts_high = (unsigned int) - (bus_addr >> BITS_IN_DWORD); - auth_desc->img_ae_insts_low = (unsigned int)bus_addr; - virt_addr += sizeof(struct icp_qat_css_hdr); - virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); - virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; - auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; - } else { - auth_desc->img_ae_insts_high = auth_desc->img_high; - auth_desc->img_ae_insts_low = auth_desc->img_low; - } - *desc = auth_desc; - return 0; -} - -static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_fw_auth_desc *desc) -{ - unsigned long ae_mask = handle->hal_handle->ae_mask; - u32 fcu_sts_csr, fcu_ctl_csr; - u32 loaded_aes, loaded_csr; - unsigned int i; - u32 fcu_sts; - - fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; - fcu_sts_csr = handle->chip_info->fcu_sts_csr; - loaded_csr = handle->chip_info->fcu_loaded_ae_csr; - - for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { - int retry = 0; - - if (!((desc->ae_mask >> i) & 0x1)) - continue; - if (qat_hal_check_ae_active(handle, i)) { - pr_err("QAT: AE %d is active\n", i); - return -EINVAL; - } - SET_CAP_CSR(handle, fcu_ctl_csr, - (FCU_CTRL_CMD_LOAD | - (1 << FCU_CTRL_BROADCAST_POS) | - (i << FCU_CTRL_AE_POS))); - - do { - msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); - if ((fcu_sts & FCU_AUTH_STS_MASK) == - FCU_STS_LOAD_DONE) { - loaded_aes = GET_CAP_CSR(handle, loaded_csr); - loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; - if (loaded_aes & (1 << i)) - break; - } - } while (retry++ < FW_AUTH_MAX_RETRY); - if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: firmware load failed timeout %x\n", retry); - return -EINVAL; - } - } - return 0; -} - -static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) -{ - struct icp_qat_suof_handle *suof_handle; - - suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL); - if (!suof_handle) - return -ENOMEM; - handle->sobj_handle = suof_handle; - if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { - qat_uclo_del_suof(handle); - pr_err("QAT: map SUOF failed\n"); - return -EINVAL; - } - return 0; -} - -int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) -{ - struct icp_qat_fw_auth_desc *desc = NULL; - int status = 0; - int ret; - - ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); - if (ret) - return ret; - - if (handle->chip_info->fw_auth) { - status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); - if (!status) - status = qat_uclo_auth_fw(handle, desc); - qat_uclo_ummap_auth_fw(handle, &desc); - } else { - if (handle->chip_info->mmp_sram_size < mem_size) { - pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); - return -EFBIG; - } - qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); - } - return status; -} - -static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) -{ - struct icp_qat_uof_filehdr *filehdr; - struct icp_qat_uclo_objhandle *objhdl; - - objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL); - if (!objhdl) - return -ENOMEM; - objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL); - if (!objhdl->obj_buf) - goto out_objbuf_err; - filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; - if (qat_uclo_check_uof_format(filehdr)) - goto out_objhdr_err; - objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, - ICP_QAT_UOF_OBJS); - if (!objhdl->obj_hdr) { - pr_err("QAT: object file chunk is null\n"); - goto out_objhdr_err; - } - handle->obj_handle = objhdl; - if (qat_uclo_parse_uof_obj(handle)) - goto out_overlay_obj_err; - return 0; - -out_overlay_obj_err: - handle->obj_handle = NULL; - kfree(objhdl->obj_hdr); -out_objhdr_err: - kfree(objhdl->obj_buf); -out_objbuf_err: - kfree(objhdl); - return -ENOMEM; -} - -static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_mof_file_hdr *mof_ptr, - u32 mof_size) -{ - struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; - unsigned int min_ver_offset; - unsigned int checksum; - - mobj_handle->file_id = ICP_QAT_MOF_FID; - mobj_handle->mof_buf = (char *)mof_ptr; - mobj_handle->mof_size = mof_size; - - min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, - min_ver); - checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, - min_ver_offset); - if (checksum != mof_ptr->checksum) { - pr_err("QAT: incorrect MOF checksum\n"); - return -EINVAL; - } - - mobj_handle->checksum = mof_ptr->checksum; - mobj_handle->min_ver = mof_ptr->min_ver; - mobj_handle->maj_ver = mof_ptr->maj_ver; - return 0; -} - -static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; - - kfree(mobj_handle->obj_table.obj_hdr); - mobj_handle->obj_table.obj_hdr = NULL; - kfree(handle->mobj_handle); - handle->mobj_handle = NULL; -} - -static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, - char *obj_name, char **obj_ptr, - unsigned int *obj_size) -{ - struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; - unsigned int i; - - for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { - if (!strncmp(obj_hdr[i].obj_name, obj_name, - ICP_QAT_SUOF_OBJ_NAME_LEN)) { - *obj_ptr = obj_hdr[i].obj_buf; - *obj_size = obj_hdr[i].obj_size; - return 0; - } - } - - pr_err("QAT: object %s is not found inside MOF\n", obj_name); - return -EINVAL; -} - -static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, - struct icp_qat_mof_objhdr *mobj_hdr, - struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) -{ - u8 *obj; - - if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, - ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { - obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; - } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, - ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { - obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; - } else { - pr_err("QAT: unsupported chunk id\n"); - return -EINVAL; - } - mobj_hdr->obj_buf = obj; - mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; - mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; - return 0; -} - -static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) -{ - struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; - struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; - struct icp_qat_mof_obj_hdr *uobj_hdr; - struct icp_qat_mof_obj_hdr *sobj_hdr; - struct icp_qat_mof_objhdr *mobj_hdr; - unsigned int uobj_chunk_num = 0; - unsigned int sobj_chunk_num = 0; - unsigned int *valid_chunk; - int ret, i; - - uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; - sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; - if (uobj_hdr) - uobj_chunk_num = uobj_hdr->num_chunks; - if (sobj_hdr) - sobj_chunk_num = sobj_hdr->num_chunks; - - mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * - sizeof(*mobj_hdr), GFP_KERNEL); - if (!mobj_hdr) - return -ENOMEM; - - mobj_handle->obj_table.obj_hdr = mobj_hdr; - valid_chunk = &mobj_handle->obj_table.num_objs; - uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) - ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); - sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) - ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); - - /* map uof objects */ - for (i = 0; i < uobj_chunk_num; i++) { - ret = qat_uclo_map_obj_from_mof(mobj_handle, - &mobj_hdr[*valid_chunk], - &uobj_chunkhdr[i]); - if (ret) - return ret; - (*valid_chunk)++; - } - - /* map suof objects */ - for (i = 0; i < sobj_chunk_num; i++) { - ret = qat_uclo_map_obj_from_mof(mobj_handle, - &mobj_hdr[*valid_chunk], - &sobj_chunkhdr[i]); - if (ret) - return ret; - (*valid_chunk)++; - } - - if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { - pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); - return -EINVAL; - } - return 0; -} - -static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, - struct icp_qat_mof_chunkhdr *mof_chunkhdr) -{ - char **sym_str = (char **)&mobj_handle->sym_str; - unsigned int *sym_size = &mobj_handle->sym_size; - struct icp_qat_mof_str_table *str_table_obj; - - *sym_size = *(unsigned int *)(uintptr_t) - (mof_chunkhdr->offset + mobj_handle->mof_buf); - *sym_str = (char *)(uintptr_t) - (mobj_handle->mof_buf + mof_chunkhdr->offset + - sizeof(str_table_obj->tab_len)); -} - -static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, - struct icp_qat_mof_chunkhdr *mof_chunkhdr) -{ - char *chunk_id = mof_chunkhdr->chunk_id; - - if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) - qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); - else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) - mobj_handle->uobjs_hdr = mobj_handle->mof_buf + - mof_chunkhdr->offset; - else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) - mobj_handle->sobjs_hdr = mobj_handle->mof_buf + - mof_chunkhdr->offset; -} - -static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) -{ - int maj = mof_hdr->maj_ver & 0xff; - int min = mof_hdr->min_ver & 0xff; - - if (mof_hdr->file_id != ICP_QAT_MOF_FID) { - pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); - return -EINVAL; - } - - if (mof_hdr->num_chunks <= 0x1) { - pr_err("QAT: MOF chunk amount is incorrect\n"); - return -EINVAL; - } - if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { - pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", - maj, min); - return -EINVAL; - } - return 0; -} - -static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_mof_file_hdr *mof_ptr, - u32 mof_size, char *obj_name, char **obj_ptr, - unsigned int *obj_size) -{ - struct icp_qat_mof_chunkhdr *mof_chunkhdr; - unsigned int file_id = mof_ptr->file_id; - struct icp_qat_mof_handle *mobj_handle; - unsigned short chunks_num; - unsigned int i; - int ret; - - if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { - if (obj_ptr) - *obj_ptr = (char *)mof_ptr; - if (obj_size) - *obj_size = mof_size; - return 0; - } - if (qat_uclo_check_mof_format(mof_ptr)) - return -EINVAL; - - mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); - if (!mobj_handle) - return -ENOMEM; - - handle->mobj_handle = mobj_handle; - ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); - if (ret) - return ret; - - mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); - chunks_num = mof_ptr->num_chunks; - - /* Parse MOF file chunks */ - for (i = 0; i < chunks_num; i++) - qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); - - /* All sym_objs uobjs and sobjs should be available */ - if (!mobj_handle->sym_str || - (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) - return -EINVAL; - - ret = qat_uclo_map_objs_from_mof(mobj_handle); - if (ret) - return ret; - - /* Seek specified uof object in MOF */ - return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, - obj_ptr, obj_size); -} - -int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, u32 mem_size, char *obj_name) -{ - char *obj_addr; - u32 obj_size; - int ret; - - BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= - (sizeof(handle->hal_handle->ae_mask) * 8)); - - if (!handle || !addr_ptr || mem_size < 24) - return -EINVAL; - - if (obj_name) { - ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, - &obj_addr, &obj_size); - if (ret) - return ret; - } else { - obj_addr = addr_ptr; - obj_size = mem_size; - } - - return (handle->chip_info->fw_auth) ? - qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : - qat_uclo_map_uof_obj(handle, obj_addr, obj_size); -} - -void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned int a; - - if (handle->mobj_handle) - qat_uclo_del_mof(handle); - if (handle->sobj_handle) - qat_uclo_del_suof(handle); - if (!obj_handle) - return; - - kfree(obj_handle->uword_buf); - for (a = 0; a < obj_handle->uimage_num; a++) - kfree(obj_handle->ae_uimage[a].page); - - for (a = 0; a < handle->hal_handle->ae_max_num; a++) - qat_uclo_free_ae_data(&obj_handle->ae_data[a]); - - kfree(obj_handle->obj_hdr); - kfree(obj_handle->obj_buf); - kfree(obj_handle); - handle->obj_handle = NULL; -} - -static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, - struct icp_qat_uclo_encap_page *encap_page, - u64 *uword, unsigned int addr_p, - unsigned int raddr, u64 fill) -{ - unsigned int i, addr; - u64 uwrd = 0; - - if (!encap_page) { - *uword = fill; - return; - } - addr = (encap_page->page_region) ? raddr : addr_p; - for (i = 0; i < encap_page->uwblock_num; i++) { - if (addr >= encap_page->uwblock[i].start_addr && - addr <= encap_page->uwblock[i].start_addr + - encap_page->uwblock[i].words_num - 1) { - addr -= encap_page->uwblock[i].start_addr; - addr *= obj_handle->uword_in_bytes; - memcpy(&uwrd, (void *)(((uintptr_t) - encap_page->uwblock[i].micro_words) + addr), - obj_handle->uword_in_bytes); - uwrd = uwrd & GENMASK_ULL(43, 0); - } - } - *uword = uwrd; - if (*uword == INVLD_UWORD) - *uword = fill; -} - -static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uclo_encap_page - *encap_page, unsigned int ae) -{ - unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - u64 fill_pat; - - /* load the page starting at appropriate ustore address */ - /* get fill-pattern from an image -- they are all the same */ - memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, - sizeof(u64)); - uw_physical_addr = encap_page->beg_addr_p; - uw_relative_addr = 0; - words_num = encap_page->micro_words_num; - while (words_num) { - if (words_num < UWORD_CPYBUF_SIZE) - cpylen = words_num; - else - cpylen = UWORD_CPYBUF_SIZE; - - /* load the buffer */ - for (i = 0; i < cpylen; i++) - qat_uclo_fill_uwords(obj_handle, encap_page, - &obj_handle->uword_buf[i], - uw_physical_addr + i, - uw_relative_addr + i, fill_pat); - - /* copy the buffer to ustore */ - qat_hal_wr_uwords(handle, (unsigned char)ae, - uw_physical_addr, cpylen, - obj_handle->uword_buf); - - uw_physical_addr += cpylen; - uw_relative_addr += cpylen; - words_num -= cpylen; - } -} - -static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, - struct icp_qat_uof_image *image) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned long cfg_ae_mask = handle->cfg_ae_mask; - unsigned long ae_assigned = image->ae_assigned; - struct icp_qat_uclo_aedata *aed; - unsigned int ctx_mask, s; - struct icp_qat_uclo_page *page; - unsigned char ae; - int ctx; - - if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) - ctx_mask = 0xff; - else - ctx_mask = 0x55; - /* load the default page and set assigned CTX PC - * to the entrypoint address */ - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - if (!test_bit(ae, &cfg_ae_mask)) - continue; - - if (!test_bit(ae, &ae_assigned)) - continue; - - aed = &obj_handle->ae_data[ae]; - /* find the slice to which this image is assigned */ - for (s = 0; s < aed->slice_num; s++) { - if (image->ctx_assigned & - aed->ae_slices[s].ctx_mask_assigned) - break; - } - if (s >= aed->slice_num) - continue; - page = aed->ae_slices[s].page; - if (!page->encap_page->def_page) - continue; - qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); - - page = aed->ae_slices[s].page; - for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) - aed->ae_slices[s].cur_page[ctx] = - (ctx_mask & (1 << ctx)) ? page : NULL; - qat_hal_set_live_ctx(handle, (unsigned char)ae, - image->ctx_assigned); - qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, - image->entry_address); - } -} - -static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) -{ - unsigned int i; - struct icp_qat_fw_auth_desc *desc = NULL; - struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; - struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; - int ret; - - for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { - ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, - simg_hdr[i].simg_len, - CSS_AE_FIRMWARE); - if (ret) - return ret; - - if (qat_uclo_map_auth_fw(handle, - (char *)simg_hdr[i].simg_buf, - (unsigned int) - simg_hdr[i].simg_len, - &desc)) - goto wr_err; - if (qat_uclo_auth_fw(handle, desc)) - goto wr_err; - if (qat_uclo_is_broadcast(handle, i)) { - if (qat_uclo_broadcast_load_fw(handle, desc)) - goto wr_err; - } else { - if (qat_uclo_load_fw(handle, desc)) - goto wr_err; - } - qat_uclo_ummap_auth_fw(handle, &desc); - } - return 0; -wr_err: - qat_uclo_ummap_auth_fw(handle, &desc); - return -EINVAL; -} - -static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) -{ - struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - unsigned int i; - - if (qat_uclo_init_globals(handle)) - return -EINVAL; - for (i = 0; i < obj_handle->uimage_num; i++) { - if (!obj_handle->ae_uimage[i].img_ptr) - return -EINVAL; - if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) - return -EINVAL; - qat_uclo_wr_uimage_page(handle, - obj_handle->ae_uimage[i].img_ptr); - } - return 0; -} - -int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) -{ - return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : - qat_uclo_wr_uof_img(handle); -} - -int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, - unsigned int cfg_ae_mask) -{ - if (!cfg_ae_mask) - return -EINVAL; - - handle->cfg_ae_mask = cfg_ae_mask; - return 0; -} diff --git a/drivers/crypto/qat/qat_dh895xcc/Makefile b/drivers/crypto/qat/qat_dh895xcc/Makefile deleted file mode 100644 index 38d6f8e1624a..000000000000 --- a/drivers/crypto/qat/qat_dh895xcc/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o -qat_dh895xcc-objs := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c deleted file mode 100644 index 1ebe0b351fae..000000000000 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ /dev/null @@ -1,252 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include "adf_dh895xcc_hw_data.h" -#include "icp_qat_hw.h" - -#define ADF_DH895XCC_VF_MSK 0xFFFFFFFF - -/* Worker thread to service arbiter mappings */ -static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { - 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, - 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, - 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222 -}; - -static struct adf_hw_device_class dh895xcc_class = { - .name = ADF_DH895XCC_DEVICE_NAME, - .type = DEV_DH895XCC, - .instances = 0 -}; - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - u32 fuses = self->fuses; - - return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & - ADF_DH895XCC_ACCELERATORS_MASK; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - u32 fuses = self->fuses; - - return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_DH895XCC_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_DH895XCC_ETR_BAR; -} - -static u32 get_sram_bar_id(struct adf_hw_device_data *self) -{ - return ADF_DH895XCC_SRAM_BAR; -} - -static u32 get_accel_cap(struct adf_accel_dev *accel_dev) -{ - struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; - u32 capabilities; - u32 legfuses; - - capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | - ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | - ICP_ACCEL_CAPABILITIES_AUTHENTICATION | - ICP_ACCEL_CAPABILITIES_CIPHER | - ICP_ACCEL_CAPABILITIES_COMPRESSION; - - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); - - /* A set bit in legfuses means the feature is OFF in this SKU */ - if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) - capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; - if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) { - capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; - capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; - } - if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) - capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; - - return capabilities; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) - >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; - - switch (sku) { - case ADF_DH895XCC_FUSECTL_SKU_1: - return DEV_SKU_1; - case ADF_DH895XCC_FUSECTL_SKU_2: - return DEV_SKU_2; - case ADF_DH895XCC_FUSECTL_SKU_3: - return DEV_SKU_3; - case ADF_DH895XCC_FUSECTL_SKU_4: - return DEV_SKU_4; - default: - return DEV_SKU_UNKNOWN; - } - return DEV_SKU_UNKNOWN; -} - -static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) -{ - return thrd_to_arb_map; -} - -static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) -{ - /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ - if (vf_mask & 0xFFFF) { - u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) - & ~ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); - } - - /* Enable VF2PF Messaging Ints - VFs 16 through 31 per vf_mask[31:16] */ - if (vf_mask >> 16) { - u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5) - & ~ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, val); - } -} - -static void disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) -{ - u32 val; - - /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ - val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) - | ADF_DH895XCC_ERR_MSK_VF2PF_L(ADF_DH895XCC_VF_MSK); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); - - /* Disable VF2PF interrupts for VFs 16 through 31 per vf_mask[31:16] */ - val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5) - | ADF_DH895XCC_ERR_MSK_VF2PF_U(ADF_DH895XCC_VF_MSK); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, val); -} - -static u32 disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) -{ - u32 sources, pending, disabled; - u32 errsou3, errmsk3; - u32 errsou5, errmsk5; - - /* Get the interrupt sources triggered by VFs */ - errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); - errsou5 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU5); - sources = ADF_DH895XCC_ERR_REG_VF2PF_L(errsou3) - | ADF_DH895XCC_ERR_REG_VF2PF_U(errsou5); - - if (!sources) - return 0; - - /* Get the already disabled interrupts */ - errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); - errmsk5 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5); - disabled = ADF_DH895XCC_ERR_REG_VF2PF_L(errmsk3) - | ADF_DH895XCC_ERR_REG_VF2PF_U(errmsk5); - - pending = sources & ~disabled; - if (!pending) - return 0; - - /* Due to HW limitations, when disabling the interrupts, we can't - * just disable the requested sources, as this would lead to missed - * interrupts if sources changes just before writing to ERRMSK3 and - * ERRMSK5. - * To work around it, disable all and re-enable only the sources that - * are not in vf_mask and were not already disabled. Re-enabling will - * trigger a new interrupt for the sources that have changed in the - * meantime, if any. - */ - errmsk3 |= ADF_DH895XCC_ERR_MSK_VF2PF_L(ADF_DH895XCC_VF_MSK); - errmsk5 |= ADF_DH895XCC_ERR_MSK_VF2PF_U(ADF_DH895XCC_VF_MSK); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, errmsk5); - - errmsk3 &= ADF_DH895XCC_ERR_MSK_VF2PF_L(sources | disabled); - errmsk5 &= ADF_DH895XCC_ERR_MSK_VF2PF_U(sources | disabled); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); - ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, errmsk5); - - /* Return the sources of the (new) interrupt(s) */ - return pending; -} - -static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) -{ - adf_gen2_cfg_iov_thds(accel_dev, enable, - ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS, - ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS); -} - -void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class = &dh895xcc_class; - hw_data->instance_id = dh895xcc_class.instances++; - hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; - hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; - hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; - hw_data->num_logical_accel = 1; - hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_isr_resource_alloc; - hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_gen2_enable_error_correction; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_accel_cap = get_accel_cap; - hw_data->get_num_accels = adf_gen2_get_num_accels; - hw_data->get_num_aes = adf_gen2_get_num_aes; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_admin_info = adf_gen2_get_admin_info; - hw_data->get_arb_info = adf_gen2_get_arb_info; - hw_data->get_sram_bar_id = get_sram_bar_id; - hw_data->get_sku = get_sku; - hw_data->fw_name = ADF_DH895XCC_FW; - hw_data->fw_mmp_name = ADF_DH895XCC_MMP; - hw_data->init_admin_comms = adf_init_admin_comms; - hw_data->exit_admin_comms = adf_exit_admin_comms; - hw_data->configure_iov_threads = configure_iov_threads; - hw_data->send_admin_init = adf_send_admin_init; - hw_data->init_arb = adf_init_arb; - hw_data->exit_arb = adf_exit_arb; - hw_data->get_arb_mapping = adf_get_arbiter_mapping; - hw_data->enable_ints = adf_gen2_enable_ints; - hw_data->reset_device = adf_reset_sbr; - hw_data->disable_iov = adf_disable_sriov; - hw_data->dev_config = adf_gen2_dev_config; - - adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); - hw_data->pfvf_ops.enable_vf2pf_interrupts = enable_vf2pf_interrupts; - hw_data->pfvf_ops.disable_all_vf2pf_interrupts = disable_all_vf2pf_interrupts; - hw_data->pfvf_ops.disable_pending_vf2pf_interrupts = disable_pending_vf2pf_interrupts; - adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen2_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; -} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h deleted file mode 100644 index 7b674bbe4192..000000000000 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_DH895x_HW_DATA_H_ -#define ADF_DH895x_HW_DATA_H_ - -/* PCIe configuration space */ -#define ADF_DH895XCC_SRAM_BAR 0 -#define ADF_DH895XCC_PMISC_BAR 1 -#define ADF_DH895XCC_ETR_BAR 2 -#define ADF_DH895XCC_FUSECTL_SKU_MASK 0x300000 -#define ADF_DH895XCC_FUSECTL_SKU_SHIFT 20 -#define ADF_DH895XCC_FUSECTL_SKU_1 0x0 -#define ADF_DH895XCC_FUSECTL_SKU_2 0x1 -#define ADF_DH895XCC_FUSECTL_SKU_3 0x2 -#define ADF_DH895XCC_FUSECTL_SKU_4 0x3 -#define ADF_DH895XCC_MAX_ACCELERATORS 6 -#define ADF_DH895XCC_MAX_ACCELENGINES 12 -#define ADF_DH895XCC_ACCELERATORS_REG_OFFSET 13 -#define ADF_DH895XCC_ACCELERATORS_MASK 0x3F -#define ADF_DH895XCC_ACCELENGINES_MASK 0xFFF -#define ADF_DH895XCC_ETR_MAX_BANKS 32 - -/* Masks for VF2PF interrupts */ -#define ADF_DH895XCC_ERR_REG_VF2PF_L(vf_src) (((vf_src) & 0x01FFFE00) >> 9) -#define ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask) (((vf_mask) & 0xFFFF) << 9) -#define ADF_DH895XCC_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16) -#define ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask) ((vf_mask) >> 16) - -/* AE to function mapping */ -#define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 -#define ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS 12 - -/* FW names */ -#define ADF_DH895XCC_FW "qat_895xcc.bin" -#define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" - -void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); -#endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c deleted file mode 100644 index e18860ab5c8e..000000000000 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_dh895xcc_hw_data.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_DH895XCC_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, - .sriov_configure = adf_sriov_configure, - .err_handler = &adf_err_handler, -}; - -static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) -{ - pci_release_regions(accel_dev->accel_pci_dev.pci_dev); - pci_disable_device(accel_dev->accel_pci_dev.pci_dev); -} - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; - int i; - - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; - - if (bar->virt_addr) - pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); - } - - if (accel_dev->hw_device) { - switch (accel_pci_dev->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: - adf_clean_hw_data_dh895xcc(accel_dev->hw_device); - break; - default: - break; - } - kfree(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev, NULL); -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - int ret; - - switch (ent->device) { - case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: - break; - default: - dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); - return -ENODEV; - } - - if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { - /* If the accelerator is connected to a node with no memory - * there is no point in using the accelerator since the remote - * memory transaction will be very slow. */ - dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); - return -EINVAL; - } - - accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!accel_dev) - return -ENOMEM; - - INIT_LIST_HEAD(&accel_dev->crypto_list); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* Add accel device to accel table. - * This should be called before adf_cleanup_accel is called */ - if (adf_devmgr_add_dev(accel_dev, NULL)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - kfree(accel_dev); - return -EFAULT; - } - - accel_dev->owner = THIS_MODULE; - /* Allocate and configure device configuration structure */ - hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - - accel_dev->hw_device = hw_data; - adf_init_hw_data_dh895xcc(accel_dev->hw_device); - pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - /* If the device has no acceleration engines then ignore it. */ - if (!hw_data->accel_mask || !hw_data->ae_mask || - ((~hw_data->ae_mask) & 0x01)) { - dev_err(&pdev->dev, "No acceleration units found"); - ret = -EFAULT; - goto out_err; - } - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - pcie_set_readrq(pdev, 1024); - - /* enable PCI device */ - if (pci_enable_device(pdev)) { - ret = -EFAULT; - goto out_err; - } - - /* set dma identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration\n"); - goto out_err_disable; - } - - if (pci_request_regions(pdev, ADF_DH895XCC_DEVICE_NAME)) { - ret = -EFAULT; - goto out_err_disable; - } - - /* Get accelerator capabilities mask */ - hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); - - /* Find and map all the device's BARS */ - i = 0; - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - - bar->base_addr = pci_resource_start(pdev, bar_nr); - if (!bar->base_addr) - break; - bar->size = pci_resource_len(pdev, bar_nr); - bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); - if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); - ret = -EFAULT; - goto out_err_free_reg; - } - } - pci_set_master(pdev); - - if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state\n"); - ret = -ENOMEM; - goto out_err_free_reg; - } - - ret = adf_dev_up(accel_dev, true); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err_free_reg: - pci_release_regions(accel_pci_dev->pci_dev); -out_err_disable: - pci_disable_device(accel_pci_dev->pci_dev); -out_err: - adf_cleanup_accel(accel_dev); - kfree(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); - adf_cleanup_pci_dev(accel_dev); - kfree(accel_dev); -} - -static int __init adfdrv_init(void) -{ - request_module("intel_qat"); - - if (pci_register_driver(&adf_driver)) { - pr_err("QAT: Driver initialization failed\n"); - return -EFAULT; - } - return 0; -} - -static void __exit adfdrv_release(void) -{ - pci_unregister_driver(&adf_driver); -} - -module_init(adfdrv_init); -module_exit(adfdrv_release); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_FIRMWARE(ADF_DH895XCC_FW); -MODULE_FIRMWARE(ADF_DH895XCC_MMP); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_dh895xccvf/Makefile b/drivers/crypto/qat/qat_dh895xccvf/Makefile deleted file mode 100644 index 0153c85ce743..000000000000 --- a/drivers/crypto/qat/qat_dh895xccvf/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common -obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o -qat_dh895xccvf-objs := adf_drv.o adf_dh895xccvf_hw_data.o diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c deleted file mode 100644 index 70e56cc16ece..000000000000 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2015 - 2021 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include "adf_dh895xccvf_hw_data.h" - -static struct adf_hw_device_class dh895xcciov_class = { - .name = ADF_DH895XCCVF_DEVICE_NAME, - .type = DEV_DH895XCCVF, - .instances = 0 -}; - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - return ADF_DH895XCCIOV_ACCELERATORS_MASK; -} - -static u32 get_ae_mask(struct adf_hw_device_data *self) -{ - return ADF_DH895XCCIOV_ACCELENGINES_MASK; -} - -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - return ADF_DH895XCCIOV_MAX_ACCELERATORS; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - return ADF_DH895XCCIOV_MAX_ACCELENGINES; -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_DH895XCCIOV_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_DH895XCCIOV_ETR_BAR; -} - -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - return DEV_SKU_VF; -} - -static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) -{ - return 0; -} - -static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) -{ -} - -void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class = &dh895xcciov_class; - hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; - hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; - hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; - hw_data->num_logical_accel = 1; - hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_DH895XCCIOV_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_DH895XCCIOV_TX_RINGS_MASK; - hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; - hw_data->alloc_irq = adf_vf_isr_resource_alloc; - hw_data->free_irq = adf_vf_isr_resource_free; - hw_data->enable_error_correction = adf_vf_void_noop; - hw_data->init_admin_comms = adf_vf_int_noop; - hw_data->exit_admin_comms = adf_vf_void_noop; - hw_data->send_admin_init = adf_vf2pf_notify_init; - hw_data->init_arb = adf_vf_int_noop; - hw_data->exit_arb = adf_vf_void_noop; - hw_data->disable_iov = adf_vf2pf_notify_shutdown; - hw_data->get_accel_mask = get_accel_mask; - hw_data->get_ae_mask = get_ae_mask; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_sku = get_sku; - hw_data->enable_ints = adf_vf_void_noop; - hw_data->dev_class->instances++; - hw_data->dev_config = adf_gen2_dev_config; - adf_devmgr_update_class_index(hw_data); - adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); - adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); - adf_gen2_init_dc_ops(&hw_data->dc_ops); -} - -void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) -{ - hw_data->dev_class->instances--; - adf_devmgr_update_class_index(hw_data); -} diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h deleted file mode 100644 index 6973fa967bc8..000000000000 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2015 - 2020 Intel Corporation */ -#ifndef ADF_DH895XVF_HW_DATA_H_ -#define ADF_DH895XVF_HW_DATA_H_ - -#define ADF_DH895XCCIOV_PMISC_BAR 1 -#define ADF_DH895XCCIOV_ACCELERATORS_MASK 0x1 -#define ADF_DH895XCCIOV_ACCELENGINES_MASK 0x1 -#define ADF_DH895XCCIOV_MAX_ACCELERATORS 1 -#define ADF_DH895XCCIOV_MAX_ACCELENGINES 1 -#define ADF_DH895XCCIOV_RX_RINGS_OFFSET 8 -#define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF -#define ADF_DH895XCCIOV_ETR_BAR 0 -#define ADF_DH895XCCIOV_ETR_MAX_BANKS 1 - -void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); -#endif diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c deleted file mode 100644 index 96854a1cd87e..000000000000 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "adf_dh895xccvf_hw_data.h" - -static const struct pci_device_id adf_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF), }, - { } -}; -MODULE_DEVICE_TABLE(pci, adf_pci_tbl); - -static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); -static void adf_remove(struct pci_dev *dev); - -static struct pci_driver adf_driver = { - .id_table = adf_pci_tbl, - .name = ADF_DH895XCCVF_DEVICE_NAME, - .probe = adf_probe, - .remove = adf_remove, -}; - -static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) -{ - pci_release_regions(accel_dev->accel_pci_dev.pci_dev); - pci_disable_device(accel_dev->accel_pci_dev.pci_dev); -} - -static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; - struct adf_accel_dev *pf; - int i; - - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; - - if (bar->virt_addr) - pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); - } - - if (accel_dev->hw_device) { - switch (accel_pci_dev->pci_dev->device) { - case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: - adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); - break; - default: - break; - } - kfree(accel_dev->hw_device); - accel_dev->hw_device = NULL; - } - adf_cfg_dev_remove(accel_dev); - debugfs_remove(accel_dev->debugfs_dir); - pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); - adf_devmgr_rm_dev(accel_dev, pf); -} - -static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct adf_accel_dev *accel_dev; - struct adf_accel_dev *pf; - struct adf_accel_pci *accel_pci_dev; - struct adf_hw_device_data *hw_data; - char name[ADF_DEVICE_NAME_LENGTH]; - unsigned int i, bar_nr; - unsigned long bar_mask; - int ret; - - switch (ent->device) { - case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: - break; - default: - dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); - return -ENODEV; - } - - accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!accel_dev) - return -ENOMEM; - - accel_dev->is_vf = true; - pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); - accel_pci_dev = &accel_dev->accel_pci_dev; - accel_pci_dev->pci_dev = pdev; - - /* Add accel device to accel table */ - if (adf_devmgr_add_dev(accel_dev, pf)) { - dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); - kfree(accel_dev); - return -EFAULT; - } - INIT_LIST_HEAD(&accel_dev->crypto_list); - - accel_dev->owner = THIS_MODULE; - /* Allocate and configure device configuration structure */ - hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, - dev_to_node(&pdev->dev)); - if (!hw_data) { - ret = -ENOMEM; - goto out_err; - } - accel_dev->hw_device = hw_data; - adf_init_hw_data_dh895xcciov(accel_dev->hw_device); - - /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data); - accel_pci_dev->sku = hw_data->get_sku(hw_data); - - /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, pci_name(pdev)); - - accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); - - /* Create device configuration table */ - ret = adf_cfg_dev_add(accel_dev); - if (ret) - goto out_err; - - /* enable PCI device */ - if (pci_enable_device(pdev)) { - ret = -EFAULT; - goto out_err; - } - - /* set dma identifier */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(&pdev->dev, "No usable DMA configuration\n"); - goto out_err_disable; - } - - if (pci_request_regions(pdev, ADF_DH895XCCVF_DEVICE_NAME)) { - ret = -EFAULT; - goto out_err_disable; - } - - /* Find and map all the device's BARS */ - i = 0; - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - - bar->base_addr = pci_resource_start(pdev, bar_nr); - if (!bar->base_addr) - break; - bar->size = pci_resource_len(pdev, bar_nr); - bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); - if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); - ret = -EFAULT; - goto out_err_free_reg; - } - } - pci_set_master(pdev); - /* Completion for VF2PF request/response message exchange */ - init_completion(&accel_dev->vf.msg_received); - - ret = adf_dev_up(accel_dev, false); - if (ret) - goto out_err_dev_stop; - - return ret; - -out_err_dev_stop: - adf_dev_down(accel_dev, false); -out_err_free_reg: - pci_release_regions(accel_pci_dev->pci_dev); -out_err_disable: - pci_disable_device(accel_pci_dev->pci_dev); -out_err: - adf_cleanup_accel(accel_dev); - kfree(accel_dev); - return ret; -} - -static void adf_remove(struct pci_dev *pdev) -{ - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - - if (!accel_dev) { - pr_err("QAT: Driver removal failed\n"); - return; - } - adf_flush_vf_wq(accel_dev); - adf_dev_down(accel_dev, false); - adf_cleanup_accel(accel_dev); - adf_cleanup_pci_dev(accel_dev); - kfree(accel_dev); -} - -static int __init adfdrv_init(void) -{ - request_module("intel_qat"); - - if (pci_register_driver(&adf_driver)) { - pr_err("QAT: Driver initialization failed\n"); - return -EFAULT; - } - return 0; -} - -static void __exit adfdrv_release(void) -{ - pci_unregister_driver(&adf_driver); - adf_clean_vf_map(true); -} - -module_init(adfdrv_init); -module_exit(adfdrv_release); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); -- cgit v1.2.3