From 6ff265fc5ef660499e0edc4641647e99eed3f519 Mon Sep 17 00:00:00 2001 From: Bean Huo Date: Thu, 1 Dec 2022 15:04:37 +0100 Subject: scsi: ufs: core: bsg: Add advanced RPMB support in ufs_bsg Add advanced RPMB support in ufs_bsg: 1. According to the UFS specification, only one RPMB operation can be performed at any time. We can ensure this by using reserved slot and its dev_cmd sync operation protection mechanism. 2. For Advanced RPMB, RPMB metadata is packaged in an EHS (Extra Header Segment) of a command UPIU, and the corresponding reply EHS (from the device) should also be returned to the user space. bsg_job->request and bsg_job->reply allow us to pass and return EHS from/back to userspace. Compared to normal/legacy RPMB, the advantages of advanced RPMB are: 1. The data length in the Advanced RPMB data read/write command can be larger than 4KB. For the legacy RPMB, the data length in a single RPMB data transfer is 256 bytes. 2. All of the advanced RPMB operations will be a single command. For legacy RPMB, take the read write-counter value as an example, you need two commands (first SECURITY PROTOCOL OUT, then second SECURITY PROTOCOL IN). Signed-off-by: Bean Huo Reviewed-by: Avri Altman Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 5cf81dff60aa..c3dfa8084b5c 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -30,6 +30,7 @@ struct ufs_hba; enum dev_cmd_type { DEV_CMD_TYPE_NOP = 0x0, DEV_CMD_TYPE_QUERY = 0x1, + DEV_CMD_TYPE_RPMB = 0x2, }; enum ufs_event_type { @@ -1201,7 +1202,10 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, int msgcode, u8 *desc_buff, int *buff_len, enum query_opcode desc_op); - +int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, + struct utp_upiu_req *rsp_upiu, struct ufs_ehs *ehs_req, + struct ufs_ehs *ehs_rsp, int sg_cnt, + struct scatterlist *sg_list, enum dma_data_direction dir); int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable); int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable); int ufshcd_suspend_prepare(struct device *dev); -- cgit v1.2.3 From ada1e653a5eae7361d95781ed812caa0c8e07dbb Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 8 Dec 2022 15:43:58 -0800 Subject: scsi: ufs: core: Allow UFS host drivers to override the sg entry size Modify the UFSHCD core to allow 'struct ufshcd_sg_entry' to be variable-length. The default is the standard length, but variants can override ufs_hba::sg_entry_size with a larger value if there are vendor-specific fields following the standard ones. This is needed to support inline encryption with ufs-exynos (FMP). Cc: Eric Biggers Reviewed-by: Avri Altman Signed-off-by: Eric Biggers [ bvanassche: edited commit message and introduced CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE ] Signed-off-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 5cf81dff60aa..e03f111947b6 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -754,6 +754,7 @@ struct ufs_hba_monitor { * @vops: pointer to variant specific operations * @vps: pointer to variant specific parameters * @priv: pointer to variant specific private data + * @sg_entry_size: size of struct ufshcd_sg_entry (may include variant fields) * @irq: Irq number of the controller * @is_irq_enabled: whether or not the UFS controller interrupt is enabled. * @dev_ref_clk_freq: reference clock frequency @@ -877,6 +878,9 @@ struct ufs_hba { const struct ufs_hba_variant_ops *vops; struct ufs_hba_variant_params *vps; void *priv; +#ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE + size_t sg_entry_size; +#endif unsigned int irq; bool is_irq_enabled; enum ufs_ref_clk_freq dev_ref_clk_freq; @@ -980,6 +984,32 @@ struct ufs_hba { bool complete_put; }; +#ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE +static inline size_t ufshcd_sg_entry_size(const struct ufs_hba *hba) +{ + return hba->sg_entry_size; +} + +static inline void ufshcd_set_sg_entry_size(struct ufs_hba *hba, size_t sg_entry_size) +{ + WARN_ON_ONCE(sg_entry_size < sizeof(struct ufshcd_sg_entry)); + hba->sg_entry_size = sg_entry_size; +} +#else +static inline size_t ufshcd_sg_entry_size(const struct ufs_hba *hba) +{ + return sizeof(struct ufshcd_sg_entry); +} + +#define ufshcd_set_sg_entry_size(hba, sg_entry_size) \ + ({ (void)(hba); BUILD_BUG_ON(sg_entry_size != sizeof(struct ufshcd_sg_entry)); }) +#endif + +static inline size_t sizeof_utp_transfer_cmd_desc(const struct ufs_hba *hba) +{ + return sizeof(struct utp_transfer_cmd_desc) + SG_ALL * ufshcd_sg_entry_size(hba); +} + /* Returns true if clocks can be gated. Otherwise false */ static inline bool ufshcd_is_clkgating_allowed(struct ufs_hba *hba) { -- cgit v1.2.3 From f2a89b071b26b79abbe892ce88c4d674d1f21f63 Mon Sep 17 00:00:00 2001 From: Arthur Simchaev Date: Sun, 11 Dec 2022 15:05:09 +0200 Subject: scsi: ufs: core: Remove redundant desc_size variable from hba Always read the descriptor with QUERY_DESC_MAX_SIZE. According to the spec, the device returns the actual size. Signed-off-by: Arthur Simchaev Reviewed-by: Bean Huo Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 5cf81dff60aa..830ababe9932 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -952,7 +952,6 @@ struct ufs_hba { bool is_urgent_bkops_lvl_checked; struct rw_semaphore clk_scaling_lock; - unsigned char desc_size[QUERY_DESC_IDN_MAX]; atomic_t scsi_block_reqs_cnt; struct device bsg_dev; @@ -1186,9 +1185,6 @@ void ufshcd_release(struct ufs_hba *hba); void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value); -void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id, - int *desc_length); - u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba); int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg); -- cgit v1.2.3 From c2c38c573a2e6184f9acd10e98305d3cb5f1c62b Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Thu, 22 Dec 2022 19:39:56 +0530 Subject: scsi: ufs: core: Add reinit_notify() callback reinit_notify() callback can be used by the UFS controller drivers to perform changes required for UFSHCD reinit that can happen during max gear switch. Tested-by: Andrew Halaney # Qdrive3/sa8540p-ride Signed-off-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index dd5912b4db77..97f007d3b851 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -298,6 +298,7 @@ struct ufs_pwr_mode_info { * @config_scaling_param: called to configure clock scaling parameters * @program_key: program or evict an inline encryption key * @event_notify: called to notify important events + * @reinit_notify: called to notify reinit of UFSHCD during max gear switch */ struct ufs_hba_variant_ops { const char *name; @@ -336,6 +337,7 @@ struct ufs_hba_variant_ops { const union ufs_crypto_cfg_entry *cfg, int slot); void (*event_notify)(struct ufs_hba *hba, enum ufs_event_type evt, void *data); + void (*reinit_notify)(struct ufs_hba *); }; /* clock gating state */ -- cgit v1.2.3 From 96a7141da33207672d7a354c885d65af4f0f9b6c Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Thu, 22 Dec 2022 19:39:57 +0530 Subject: scsi: ufs: core: Add support for reinitializing the UFS device Some platforms like Qcom, requires the UFS device to be reinitialized after switching to maximum gear speed. So add support for that in UFS core by introducing a new quirk (UFSHCD_CAP_REINIT_AFTER_MAX_GEAR_SWITCH) and doing the reinitialization, if the quirk is enabled by the controller driver. Suggested-by: Can Guo Tested-by: Andrew Halaney # Qdrive3/sa8540p-ride Signed-off-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 97f007d3b851..ff138927676b 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -596,6 +596,12 @@ enum ufshcd_quirks { * auto-hibernate capability but it's FASTAUTO only. */ UFSHCD_QUIRK_HIBERN_FASTAUTO = 1 << 18, + + /* + * This quirk needs to be enabled if the host controller needs + * to reinit the device after switching to maximum gear. + */ + UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH = 1 << 19, }; enum ufshcd_caps { -- cgit v1.2.3 From f3e57da528127febae7eb03d9c87408d572b0fd8 Mon Sep 17 00:00:00 2001 From: Bean Huo Date: Sun, 8 Jan 2023 23:40:56 +0100 Subject: scsi: core: Fix invisible definition compilation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 'include/ufs/ufshcd.h' file, 'enum dma_data_direction' will be used, which is defined in linux/dma-direction.h, however, this header file is not included in ufshcd.h, thus causing the following compilation warning: "warning: ‘enum dma_data_direction’ declared inside parameter list will not be visible outside of this definition or declaration" Fix this warning by including 'linux/dma-direction.h'. Fixes: 6ff265fc5ef6 ("scsi: ufs: core: bsg: Add advanced RPMB support in ufs_bsg") Reported-by: Xiaosen He Reported-by: Bart Van Assche Signed-off-by: Bean Huo Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index ff138927676b..fc7373a1a15e 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 6e1d850acff9477ae4c18a73c19ef52841ac2010 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:38 -0800 Subject: scsi: ufs: core: Probe for EXT_IID support Task Tag is limited to 8 bits and this restricts the number of active I/Os to 255. In multi-circular queue mode, this may not be enough. The specification provides EXT_IID which can be used to increase the number of I/Os if the UFS device and UFSHC support it. This patch adds support to probe for EXT_IID support in UFS device and UFSHC. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Avri Altman Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index fc7373a1a15e..8f2080f9fdb5 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -757,6 +757,7 @@ struct ufs_hba_monitor { * @outstanding_lock: Protects @outstanding_reqs. * @outstanding_reqs: Bits representing outstanding transfer requests * @capabilities: UFS Controller Capabilities + * @mcq_capabilities: UFS Multi Circular Queue capabilities * @nutrs: Transfer Request Queue depth supported by controller * @nutmrs: Task Management Queue depth supported by controller * @reserved_slot: Used to submit device commands. Protected by @dev_cmd.lock. @@ -841,6 +842,7 @@ struct ufs_hba_monitor { * device * @complete_put: whether or not to call ufshcd_rpm_put() from inside * ufshcd_resume_complete() + * @ext_iid_sup: is EXT_IID is supported by UFSHC */ struct ufs_hba { void __iomem *mmio_base; @@ -882,6 +884,7 @@ struct ufs_hba { u32 capabilities; int nutrs; + u32 mcq_capabilities; int nutmrs; u32 reserved_slot; u32 ufs_version; @@ -991,6 +994,7 @@ struct ufs_hba { #endif u32 luns_avail; bool complete_put; + bool ext_iid_sup; }; #ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE -- cgit v1.2.3 From 305a357d3595d39be7c001f72e135bc94cbd85da Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:39 -0800 Subject: scsi: ufs: core: Introduce multi-circular queue capability Add support to check for MCQ capability in the UFSHC. Add a module parameter to disable MCQ if needed. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 8f2080f9fdb5..514687d2de54 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -843,6 +843,7 @@ struct ufs_hba_monitor { * @complete_put: whether or not to call ufshcd_rpm_put() from inside * ufshcd_resume_complete() * @ext_iid_sup: is EXT_IID is supported by UFSHC + * @mcq_sup: is mcq supported by UFSHC */ struct ufs_hba { void __iomem *mmio_base; @@ -995,6 +996,7 @@ struct ufs_hba { u32 luns_avail; bool complete_put; bool ext_iid_sup; + bool mcq_sup; }; #ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE -- cgit v1.2.3 From 0cab4023ec7b49b18145f74ab8389678d6d58878 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:40 -0800 Subject: scsi: ufs: core: Defer adding host to SCSI if MCQ is supported If MCQ support is present, enabling it after MCQ support has been configured would require reallocating tags and memory. It would also free up the already allocated memory in Single Doorbell Mode. So defer invoking scsi_add_host() until MCQ is configured. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Manivannan Sadhasivam Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 514687d2de54..a343bd4bc0eb 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -996,6 +996,7 @@ struct ufs_hba { u32 luns_avail; bool complete_put; bool ext_iid_sup; + bool scsi_host_added; bool mcq_sup; }; -- cgit v1.2.3 From 57b1c0ef89ac9d9e7475df7843aeb7672ebcd197 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:41 -0800 Subject: scsi: ufs: core: mcq: Add support to allocate multiple queues Multi-circular queue (MCQ) has been added in UFSHC v4.0 standard in addition to the Single Doorbell mode. The MCQ mode supports multiple submission and completion queues. Add support to allocate and configure the queues. Add module parameters support to configure the queues. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Manivannan Sadhasivam Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index a343bd4bc0eb..91ca724c0294 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -840,6 +840,8 @@ struct ufs_hba_monitor { * ee_ctrl_mask * @luns_avail: number of regular and well known LUNs supported by the UFS * device + * @nr_hw_queues: number of hardware queues configured + * @nr_queues: number of Queues of different queue types * @complete_put: whether or not to call ufshcd_rpm_put() from inside * ufshcd_resume_complete() * @ext_iid_sup: is EXT_IID is supported by UFSHC @@ -994,6 +996,8 @@ struct ufs_hba { u32 debugfs_ee_rate_limit_ms; #endif u32 luns_avail; + unsigned int nr_hw_queues; + unsigned int nr_queues[HCTX_MAX_TYPES]; bool complete_put; bool ext_iid_sup; bool scsi_host_added; -- cgit v1.2.3 From c263b4ef737e622e2a908c58ca4bb68a89376387 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:42 -0800 Subject: scsi: ufs: core: mcq: Configure resource regions Define the MCQ resources and add support to ioremap the resource regions. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 91ca724c0294..81c7494cecad 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -300,6 +300,7 @@ struct ufs_pwr_mode_info { * @program_key: program or evict an inline encryption key * @event_notify: called to notify important events * @reinit_notify: called to notify reinit of UFSHCD during max gear switch + * @mcq_config_resource: called to configure MCQ platform resources */ struct ufs_hba_variant_ops { const char *name; @@ -339,6 +340,7 @@ struct ufs_hba_variant_ops { void (*event_notify)(struct ufs_hba *hba, enum ufs_event_type evt, void *data); void (*reinit_notify)(struct ufs_hba *); + int (*mcq_config_resource)(struct ufs_hba *hba); }; /* clock gating state */ @@ -733,6 +735,30 @@ struct ufs_hba_monitor { bool enabled; }; +/** + * struct ufshcd_res_info_t - MCQ related resource regions + * + * @name: resource name + * @resource: pointer to resource region + * @base: register base address + */ +struct ufshcd_res_info { + const char *name; + struct resource *resource; + void __iomem *base; +}; + +enum ufshcd_res { + RES_UFS, + RES_MCQ, + RES_MCQ_SQD, + RES_MCQ_SQIS, + RES_MCQ_CQD, + RES_MCQ_CQIS, + RES_MCQ_VS, + RES_MAX, +}; + /** * struct ufs_hba - per adapter private structure * @mmio_base: UFSHCI base register address @@ -846,6 +872,8 @@ struct ufs_hba_monitor { * ufshcd_resume_complete() * @ext_iid_sup: is EXT_IID is supported by UFSHC * @mcq_sup: is mcq supported by UFSHC + * @res: array of resource info of MCQ registers + * @mcq_base: Multi circular queue registers base address */ struct ufs_hba { void __iomem *mmio_base; @@ -1002,6 +1030,8 @@ struct ufs_hba { bool ext_iid_sup; bool scsi_host_added; bool mcq_sup; + struct ufshcd_res_info res[RES_MAX]; + void __iomem *mcq_base; }; #ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE -- cgit v1.2.3 From 7224c806876e46cfaf46b1c90da8d5c2e1f2108f Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:43 -0800 Subject: scsi: ufs: core: mcq: Calculate queue depth The UFS device defines the supported queuedepth by bqueuedepth which has a max value of 256. The HC defines MAC (Max Active Commands) that defines the max number of commands that in flight to the UFS device. Calculate and configure the nutrs based on both these values. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 81c7494cecad..57854bb0395e 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -301,6 +301,7 @@ struct ufs_pwr_mode_info { * @event_notify: called to notify important events * @reinit_notify: called to notify reinit of UFSHCD during max gear switch * @mcq_config_resource: called to configure MCQ platform resources + * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode */ struct ufs_hba_variant_ops { const char *name; @@ -341,6 +342,7 @@ struct ufs_hba_variant_ops { enum ufs_event_type evt, void *data); void (*reinit_notify)(struct ufs_hba *); int (*mcq_config_resource)(struct ufs_hba *hba); + int (*get_hba_mac)(struct ufs_hba *hba); }; /* clock gating state */ -- cgit v1.2.3 From 4682abfae2eb3a1c138130cfd6d71411d81aaa00 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:44 -0800 Subject: scsi: ufs: core: mcq: Allocate memory for MCQ mode To read the bqueuedepth, the device descriptor is fetched in Single Doorbell Mode. This allocated memory may not be enough for MCQ mode because the number of tags supported in MCQ mode may be larger than in SDB mode. Hence, release the memory allocated in SDB mode and allocate memory for MCQ mode operation. Define the UFS hardware queue and Completion Queue Entry. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 57854bb0395e..311113c8e92d 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -876,6 +876,8 @@ enum ufshcd_res { * @mcq_sup: is mcq supported by UFSHC * @res: array of resource info of MCQ registers * @mcq_base: Multi circular queue registers base address + * @uhq: array of supported hardware queues + * @dev_cmd_queue: Queue for issuing device management commands */ struct ufs_hba { void __iomem *mmio_base; @@ -1034,6 +1036,24 @@ struct ufs_hba { bool mcq_sup; struct ufshcd_res_info res[RES_MAX]; void __iomem *mcq_base; + struct ufs_hw_queue *uhq; + struct ufs_hw_queue *dev_cmd_queue; +}; + +/** + * struct ufs_hw_queue - per hardware queue structure + * @sqe_base_addr: submission queue entry base address + * @sqe_dma_addr: submission queue dma address + * @cqe_base_addr: completion queue base address + * @cqe_dma_addr: completion queue dma address + * @max_entries: max number of slots in this hardware queue + */ +struct ufs_hw_queue { + void *sqe_base_addr; + dma_addr_t sqe_dma_addr; + struct cq_entry *cqe_base_addr; + dma_addr_t cqe_dma_addr; + u32 max_entries; }; #ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE -- cgit v1.2.3 From 2468da61ea095162067ed408824298ba9c3661c8 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:45 -0800 Subject: scsi: ufs: core: mcq: Configure operation and runtime interface Runtime and operation registers are defined per Submission and Completion queue. The location of these registers is not defined in the spec; meaning the offsets and stride may vary for different HC vendors. Establish the stride, base address, and doorbell address offsets from vendor host driver and program it. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 311113c8e92d..13a2f17daa8c 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -302,6 +302,7 @@ struct ufs_pwr_mode_info { * @reinit_notify: called to notify reinit of UFSHCD during max gear switch * @mcq_config_resource: called to configure MCQ platform resources * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode + * @op_runtime_config: called to config Operation and runtime regs Pointers */ struct ufs_hba_variant_ops { const char *name; @@ -343,6 +344,7 @@ struct ufs_hba_variant_ops { void (*reinit_notify)(struct ufs_hba *); int (*mcq_config_resource)(struct ufs_hba *hba); int (*get_hba_mac)(struct ufs_hba *hba); + int (*op_runtime_config)(struct ufs_hba *hba); }; /* clock gating state */ @@ -761,6 +763,27 @@ enum ufshcd_res { RES_MAX, }; +/** + * struct ufshcd_mcq_opr_info_t - Operation and Runtime registers + * + * @offset: Doorbell Address Offset + * @stride: Steps proportional to queue [0...31] + * @base: base address + */ +struct ufshcd_mcq_opr_info_t { + unsigned long offset; + unsigned long stride; + void __iomem *base; +}; + +enum ufshcd_mcq_opr { + OPR_SQD, + OPR_SQIS, + OPR_CQD, + OPR_CQIS, + OPR_MAX, +}; + /** * struct ufs_hba - per adapter private structure * @mmio_base: UFSHCI base register address @@ -874,6 +897,7 @@ enum ufshcd_res { * ufshcd_resume_complete() * @ext_iid_sup: is EXT_IID is supported by UFSHC * @mcq_sup: is mcq supported by UFSHC + * @mcq_enabled: is mcq ready to accept requests * @res: array of resource info of MCQ registers * @mcq_base: Multi circular queue registers base address * @uhq: array of supported hardware queues @@ -1034,28 +1058,46 @@ struct ufs_hba { bool ext_iid_sup; bool scsi_host_added; bool mcq_sup; + bool mcq_enabled; struct ufshcd_res_info res[RES_MAX]; void __iomem *mcq_base; struct ufs_hw_queue *uhq; struct ufs_hw_queue *dev_cmd_queue; + struct ufshcd_mcq_opr_info_t mcq_opr[OPR_MAX]; }; /** * struct ufs_hw_queue - per hardware queue structure + * @mcq_sq_head: base address of submission queue head pointer + * @mcq_sq_tail: base address of submission queue tail pointer + * @mcq_cq_head: base address of completion queue head pointer + * @mcq_cq_tail: base address of completion queue tail pointer * @sqe_base_addr: submission queue entry base address * @sqe_dma_addr: submission queue dma address * @cqe_base_addr: completion queue base address * @cqe_dma_addr: completion queue dma address * @max_entries: max number of slots in this hardware queue + * @id: hardware queue ID */ struct ufs_hw_queue { + void __iomem *mcq_sq_head; + void __iomem *mcq_sq_tail; + void __iomem *mcq_cq_head; + void __iomem *mcq_cq_tail; + void *sqe_base_addr; dma_addr_t sqe_dma_addr; struct cq_entry *cqe_base_addr; dma_addr_t cqe_dma_addr; u32 max_entries; + u32 id; }; +static inline bool is_mcq_enabled(struct ufs_hba *hba) +{ + return hba->mcq_enabled; +} + #ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE static inline size_t ufshcd_sg_entry_size(const struct ufs_hba *hba) { @@ -1137,6 +1179,16 @@ static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba) return hba->caps & UFSHCD_CAP_WB_WITH_CLK_SCALING; } +#define ufsmcq_writel(hba, val, reg) \ + writel((val), (hba)->mcq_base + (reg)) +#define ufsmcq_readl(hba, reg) \ + readl((hba)->mcq_base + (reg)) + +#define ufsmcq_writelx(hba, val, reg) \ + writel_relaxed((val), (hba)->mcq_base + (reg)) +#define ufsmcq_readlx(hba, reg) \ + readl_relaxed((hba)->mcq_base + (reg)) + #define ufshcd_writel(hba, val, reg) \ writel((val), (hba)->mmio_base + (reg)) #define ufshcd_readl(hba, reg) \ -- cgit v1.2.3 From 22a2d563de1425ea294e9abfa104dbf20c83a28a Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:47 -0800 Subject: scsi: ufs: core: Prepare ufshcd_send_command() for MCQ Add support to send commands using multiple submission queues in MCQ mode. Modify the functions that use ufshcd_send_command(). Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 13a2f17daa8c..019b8cf23b29 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -224,6 +224,7 @@ struct ufs_dev_cmd { struct mutex lock; struct completion *complete; struct ufs_query query; + struct cq_entry *cqe; }; /** @@ -1078,6 +1079,8 @@ struct ufs_hba { * @cqe_dma_addr: completion queue dma address * @max_entries: max number of slots in this hardware queue * @id: hardware queue ID + * @sq_tp_slot: current slot to which SQ tail pointer is pointing + * @sq_lock: serialize submission queue access */ struct ufs_hw_queue { void __iomem *mcq_sq_head; @@ -1091,6 +1094,8 @@ struct ufs_hw_queue { dma_addr_t cqe_dma_addr; u32 max_entries; u32 id; + u32 sq_tail_slot; + spinlock_t sq_lock; }; static inline bool is_mcq_enabled(struct ufs_hba *hba) -- cgit v1.2.3 From f87b2c41822aad09aadac31b8ba22c0c0e639eee Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:50 -0800 Subject: scsi: ufs: mcq: Add completion support of a CQE Add support for completing requests from Completion Queue. Some host controllers support vendor specific registers that provide a bitmap of all CQs which have at least one completed CQE. Add this support. The MCQ specification doesn't provide the Task Tag or its equivalent in the Completion Queue Entry. So use an indirect method to find the Task Tag from the Completion Queue Entry. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 019b8cf23b29..0dcb104a6713 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -304,6 +304,7 @@ struct ufs_pwr_mode_info { * @mcq_config_resource: called to configure MCQ platform resources * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode * @op_runtime_config: called to config Operation and runtime regs Pointers + * @get_outstanding_cqs: called to get outstanding completion queues */ struct ufs_hba_variant_ops { const char *name; @@ -346,6 +347,8 @@ struct ufs_hba_variant_ops { int (*mcq_config_resource)(struct ufs_hba *hba); int (*get_hba_mac)(struct ufs_hba *hba); int (*op_runtime_config)(struct ufs_hba *hba); + int (*get_outstanding_cqs)(struct ufs_hba *hba, + unsigned long *ocqs); }; /* clock gating state */ @@ -1081,6 +1084,8 @@ struct ufs_hba { * @id: hardware queue ID * @sq_tp_slot: current slot to which SQ tail pointer is pointing * @sq_lock: serialize submission queue access + * @cq_tail_slot: current slot to which CQ tail pointer is pointing + * @cq_head_slot: current slot to which CQ head pointer is pointing */ struct ufs_hw_queue { void __iomem *mcq_sq_head; @@ -1096,6 +1101,8 @@ struct ufs_hw_queue { u32 id; u32 sq_tail_slot; spinlock_t sq_lock; + u32 cq_tail_slot; + u32 cq_head_slot; }; static inline bool is_mcq_enabled(struct ufs_hba *hba) -- cgit v1.2.3 From ed975065c31c2a0372e13c19e8140b69814a98ba Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Fri, 13 Jan 2023 12:48:51 -0800 Subject: scsi: ufs: core: mcq: Add completion support in poll Complete CQE requests in poll. Assumption is that several poll completion may happen in different CPUs for the same completion queue. Hence a spin lock protection is added. Co-developed-by: Can Guo Signed-off-by: Can Guo Signed-off-by: Asutosh Das Reviewed-by: Bart Van Assche Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanley Chu Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 0dcb104a6713..33973e9f6d6a 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1086,6 +1086,7 @@ struct ufs_hba { * @sq_lock: serialize submission queue access * @cq_tail_slot: current slot to which CQ tail pointer is pointing * @cq_head_slot: current slot to which CQ head pointer is pointing + * @cq_lock: Synchronize between multiple polling instances */ struct ufs_hw_queue { void __iomem *mcq_sq_head; @@ -1103,6 +1104,7 @@ struct ufs_hw_queue { spinlock_t sq_lock; u32 cq_tail_slot; u32 cq_head_slot; + spinlock_t cq_lock; }; static inline bool is_mcq_enabled(struct ufs_hba *hba) -- cgit v1.2.3 From edb0db05607ce05a5e0df00518b58a811e9f548e Mon Sep 17 00:00:00 2001 From: Can Guo Date: Wed, 14 Dec 2022 19:06:20 -0800 Subject: scsi: ufs: core: Add Event Specific Interrupt configuration vendor specific ops As Event Specific Interrupt message format is not defined in UFSHCI JEDEC specs, and the ESI handling highly depends on how the format is designed, hence add a vendor specific ops such that SoC vendors can configure their own ESI handlers. If ESI vops is not provided or returning error, go with the legacy (central) interrupt way. Signed-off-by: Can Guo Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 33973e9f6d6a..1285787e2628 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -305,6 +305,7 @@ struct ufs_pwr_mode_info { * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode * @op_runtime_config: called to config Operation and runtime regs Pointers * @get_outstanding_cqs: called to get outstanding completion queues + * @config_esi: called to config Event Specific Interrupt */ struct ufs_hba_variant_ops { const char *name; @@ -349,6 +350,7 @@ struct ufs_hba_variant_ops { int (*op_runtime_config)(struct ufs_hba *hba); int (*get_outstanding_cqs)(struct ufs_hba *hba, unsigned long *ocqs); + int (*config_esi)(struct ufs_hba *hba); }; /* clock gating state */ -- cgit v1.2.3 From e02288e0265fe316a16d48ec6dd7b7fd54d66e3e Mon Sep 17 00:00:00 2001 From: Can Guo Date: Wed, 14 Dec 2022 19:06:21 -0800 Subject: scsi: ufs: core: mcq: Add Event Specific Interrupt enable and config functions Add and export two functions to enable ESI and config ESI base addresses. The calls to these exported functions will be added by the next patch in this series. Signed-off-by: Can Guo Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 1285787e2628..1779238d8a56 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1241,6 +1242,11 @@ void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk); void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val); void ufshcd_hba_stop(struct ufs_hba *hba); void ufshcd_schedule_eh_work(struct ufs_hba *hba); +void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i); +unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba, + struct ufs_hw_queue *hwq); +void ufshcd_mcq_enable_esi(struct ufs_hba *hba); +void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg); /** * ufshcd_set_variant - set variant specific data to the hba -- cgit v1.2.3 From 86bd0c4a2a5dc4265884648cb92c681646509692 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 12 Jan 2023 15:42:13 -0800 Subject: scsi: ufs: exynos: Fix DMA alignment for PAGE_SIZE != 4096 The Exynos UFS controller only supports scatter/gather list elements that are aligned on a 4 KiB boundary. Fix DMA alignment in case PAGE_SIZE != 4096. Rename UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE into UFSHCD_QUIRK_4KB_DMA_ALIGNMENT. Cc: Kiwoong Kim Fixes: 2b2bfc8aa519 ("scsi: ufs: Introduce a quirk to allow only page-aligned sg entries") Signed-off-by: Bart Van Assche Reviewed-by: Alim Akhtar Tested-by: Alim Akhtar Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 1779238d8a56..57a5af27522a 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -583,9 +583,9 @@ enum ufshcd_quirks { UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING = 1 << 13, /* - * This quirk allows only sg entries aligned with page size. + * Align DMA SG entries on a 4 KiB boundary. */ - UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE = 1 << 14, + UFSHCD_QUIRK_4KB_DMA_ALIGNMENT = 1 << 14, /* * This quirk needs to be enabled if the host controller does not -- cgit v1.2.3 From 88441a8d355dcbb86aa69f82934ae1ff0fccfa83 Mon Sep 17 00:00:00 2001 From: Anjana Hari Date: Thu, 2 Feb 2023 21:40:45 +0530 Subject: scsi: ufs: core: Add hibernation callbacks Add freeze, thaw, and restore callbacks for hibernate and restore functionality. Link: https://lore.kernel.org/r/20230202161045.3956-2-quic_ahari@quicinc.com Signed-off-by: Anjana Hari Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/ufs/ufshcd.h') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 57a5af27522a..ed9e3d5addb3 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1276,8 +1276,12 @@ extern int ufshcd_runtime_resume(struct device *dev); #ifdef CONFIG_PM_SLEEP extern int ufshcd_system_suspend(struct device *dev); extern int ufshcd_system_resume(struct device *dev); +extern int ufshcd_system_freeze(struct device *dev); +extern int ufshcd_system_thaw(struct device *dev); +extern int ufshcd_system_restore(struct device *dev); #endif extern int ufshcd_shutdown(struct ufs_hba *hba); + extern int ufshcd_dme_configure_adapt(struct ufs_hba *hba, int agreed_gear, int adapt_val); -- cgit v1.2.3