summaryrefslogtreecommitdiffstats
path: root/drivers/accel
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/accel')
-rw-r--r--drivers/accel/Makefile4
-rw-r--r--drivers/accel/ivpu/ivpu_drv.c18
-rw-r--r--drivers/accel/ivpu/ivpu_drv.h7
-rw-r--r--drivers/accel/ivpu/ivpu_hw_mtl.c113
-rw-r--r--drivers/accel/ivpu/ivpu_ipc.h2
-rw-r--r--drivers/accel/ivpu/ivpu_job.c29
-rw-r--r--drivers/accel/ivpu/ivpu_pm.c43
-rw-r--r--drivers/accel/ivpu/ivpu_pm.h1
8 files changed, 100 insertions, 117 deletions
diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile
index 07aa77aed1c8..f22fd44d586b 100644
--- a/drivers/accel/Makefile
+++ b/drivers/accel/Makefile
@@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-y += habanalabs/
-obj-y += ivpu/
+obj-$(CONFIG_DRM_ACCEL_HABANALABS) += habanalabs/
+obj-$(CONFIG_DRM_ACCEL_IVPU) += ivpu/
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 231f29bb5025..6a320a73e3cc 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -8,7 +8,6 @@
#include <linux/pci.h>
#include <drm/drm_accel.h>
-#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_gem.h>
#include <drm/drm_ioctl.h>
@@ -118,6 +117,10 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f
struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
struct drm_ivpu_param *args = data;
int ret = 0;
+ int idx;
+
+ if (!drm_dev_enter(dev, &idx))
+ return -ENODEV;
switch (args->param) {
case DRM_IVPU_PARAM_DEVICE_ID:
@@ -171,6 +174,7 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f
break;
}
+ drm_dev_exit(idx);
return ret;
}
@@ -470,8 +474,8 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
vdev->hw->ops = &ivpu_hw_mtl_ops;
vdev->platform = IVPU_PLATFORM_INVALID;
- vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1;
- vdev->context_xa_limit.max = IVPU_CONTEXT_LIMIT;
+ vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID;
+ vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID;
atomic64_set(&vdev->unique_id_counter, 0);
xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC);
xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
@@ -565,6 +569,8 @@ err_mmu_gctx_fini:
ivpu_mmu_global_context_fini(vdev);
err_power_down:
ivpu_hw_power_down(vdev);
+ if (IVPU_WA(d3hot_after_power_off))
+ pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);
err_xa_destroy:
xa_destroy(&vdev->submitted_jobs_xa);
xa_destroy(&vdev->context_xa);
@@ -575,7 +581,11 @@ static void ivpu_dev_fini(struct ivpu_device *vdev)
{
ivpu_pm_disable(vdev);
ivpu_shutdown(vdev);
+ if (IVPU_WA(d3hot_after_power_off))
+ pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);
ivpu_job_done_thread_fini(vdev);
+ ivpu_pm_cancel_recovery(vdev);
+
ivpu_ipc_fini(vdev);
ivpu_fw_fini(vdev);
ivpu_mmu_global_context_fini(vdev);
@@ -622,7 +632,7 @@ static void ivpu_remove(struct pci_dev *pdev)
{
struct ivpu_device *vdev = pci_get_drvdata(pdev);
- drm_dev_unregister(&vdev->drm);
+ drm_dev_unplug(&vdev->drm);
ivpu_dev_fini(vdev);
}
diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h
index f47b4965db2e..d3013fbd13b3 100644
--- a/drivers/accel/ivpu/ivpu_drv.h
+++ b/drivers/accel/ivpu/ivpu_drv.h
@@ -7,6 +7,7 @@
#define __IVPU_DRV_H__
#include <drm/drm_device.h>
+#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/drm_mm.h>
#include <drm/drm_print.h>
@@ -24,7 +25,10 @@
#define PCI_DEVICE_ID_MTL 0x7d1d
#define IVPU_GLOBAL_CONTEXT_MMU_SSID 0
-#define IVPU_CONTEXT_LIMIT 64
+/* SSID 1 is used by the VPU to represent invalid context */
+#define IVPU_USER_CONTEXT_MIN_SSID 2
+#define IVPU_USER_CONTEXT_MAX_SSID (IVPU_USER_CONTEXT_MIN_SSID + 63)
+
#define IVPU_NUM_ENGINES 2
#define IVPU_PLATFORM_SILICON 0
@@ -70,6 +74,7 @@
struct ivpu_wa_table {
bool punit_disabled;
bool clear_runtime_mem;
+ bool d3hot_after_power_off;
};
struct ivpu_hw_info;
diff --git a/drivers/accel/ivpu/ivpu_hw_mtl.c b/drivers/accel/ivpu/ivpu_hw_mtl.c
index 62bfaa9081c4..382ec127be8e 100644
--- a/drivers/accel/ivpu/ivpu_hw_mtl.c
+++ b/drivers/accel/ivpu/ivpu_hw_mtl.c
@@ -12,24 +12,23 @@
#include "ivpu_mmu.h"
#include "ivpu_pm.h"
-#define TILE_FUSE_ENABLE_BOTH 0x0
-#define TILE_FUSE_ENABLE_UPPER 0x1
-#define TILE_FUSE_ENABLE_LOWER 0x2
-
-#define TILE_SKU_BOTH_MTL 0x3630
-#define TILE_SKU_LOWER_MTL 0x3631
-#define TILE_SKU_UPPER_MTL 0x3632
+#define TILE_FUSE_ENABLE_BOTH 0x0
+#define TILE_SKU_BOTH_MTL 0x3630
/* Work point configuration values */
-#define WP_CONFIG_1_TILE_5_3_RATIO 0x0101
-#define WP_CONFIG_1_TILE_4_3_RATIO 0x0102
-#define WP_CONFIG_2_TILE_5_3_RATIO 0x0201
-#define WP_CONFIG_2_TILE_4_3_RATIO 0x0202
-#define WP_CONFIG_0_TILE_PLL_OFF 0x0000
+#define CONFIG_1_TILE 0x01
+#define CONFIG_2_TILE 0x02
+#define PLL_RATIO_5_3 0x01
+#define PLL_RATIO_4_3 0x02
+#define WP_CONFIG(tile, ratio) (((tile) << 8) | (ratio))
+#define WP_CONFIG_1_TILE_5_3_RATIO WP_CONFIG(CONFIG_1_TILE, PLL_RATIO_5_3)
+#define WP_CONFIG_1_TILE_4_3_RATIO WP_CONFIG(CONFIG_1_TILE, PLL_RATIO_4_3)
+#define WP_CONFIG_2_TILE_5_3_RATIO WP_CONFIG(CONFIG_2_TILE, PLL_RATIO_5_3)
+#define WP_CONFIG_2_TILE_4_3_RATIO WP_CONFIG(CONFIG_2_TILE, PLL_RATIO_4_3)
+#define WP_CONFIG_0_TILE_PLL_OFF WP_CONFIG(0, 0)
#define PLL_REF_CLK_FREQ (50 * 1000000)
#define PLL_SIMULATION_FREQ (10 * 1000000)
-#define PLL_RATIO_TO_FREQ(x) ((x) * PLL_REF_CLK_FREQ)
#define PLL_DEFAULT_EPP_VALUE 0x80
#define TIM_SAFE_ENABLE 0xf1d0dead
@@ -101,6 +100,7 @@ static void ivpu_hw_wa_init(struct ivpu_device *vdev)
{
vdev->wa.punit_disabled = ivpu_is_fpga(vdev);
vdev->wa.clear_runtime_mem = false;
+ vdev->wa.d3hot_after_power_off = true;
}
static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
@@ -218,7 +218,8 @@ static int ivpu_pll_drive(struct ivpu_device *vdev, bool enable)
config = 0;
}
- ivpu_dbg(vdev, PM, "PLL workpoint request: %d Hz\n", PLL_RATIO_TO_FREQ(target_ratio));
+ ivpu_dbg(vdev, PM, "PLL workpoint request: config 0x%04x pll ratio 0x%x\n",
+ config, target_ratio);
ret = ivpu_pll_cmd_send(vdev, hw->pll.min_ratio, hw->pll.max_ratio, target_ratio, config);
if (ret) {
@@ -403,11 +404,6 @@ static int ivpu_boot_host_ss_axi_enable(struct ivpu_device *vdev)
return ivpu_boot_host_ss_axi_drive(vdev, true);
}
-static int ivpu_boot_host_ss_axi_disable(struct ivpu_device *vdev)
-{
- return ivpu_boot_host_ss_axi_drive(vdev, false);
-}
-
static int ivpu_boot_host_ss_top_noc_drive(struct ivpu_device *vdev, bool enable)
{
int ret;
@@ -441,11 +437,6 @@ static int ivpu_boot_host_ss_top_noc_enable(struct ivpu_device *vdev)
return ivpu_boot_host_ss_top_noc_drive(vdev, true);
}
-static int ivpu_boot_host_ss_top_noc_disable(struct ivpu_device *vdev)
-{
- return ivpu_boot_host_ss_top_noc_drive(vdev, false);
-}
-
static void ivpu_boot_pwr_island_trickle_drive(struct ivpu_device *vdev, bool enable)
{
u32 val = REGV_RD32(MTL_VPU_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
@@ -504,16 +495,6 @@ static void ivpu_boot_dpu_active_drive(struct ivpu_device *vdev, bool enable)
REGV_WR32(MTL_VPU_HOST_SS_AON_DPU_ACTIVE, val);
}
-static int ivpu_boot_pwr_domain_disable(struct ivpu_device *vdev)
-{
- ivpu_boot_dpu_active_drive(vdev, false);
- ivpu_boot_pwr_island_isolation_drive(vdev, true);
- ivpu_boot_pwr_island_trickle_drive(vdev, false);
- ivpu_boot_pwr_island_drive(vdev, false);
-
- return ivpu_boot_wait_for_pwr_island_status(vdev, 0x0);
-}
-
static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev)
{
int ret;
@@ -629,34 +610,10 @@ static int ivpu_boot_d0i3_drive(struct ivpu_device *vdev, bool enable)
static int ivpu_hw_mtl_info_init(struct ivpu_device *vdev)
{
struct ivpu_hw_info *hw = vdev->hw;
- u32 tile_fuse;
-
- tile_fuse = REGB_RD32(MTL_BUTTRESS_TILE_FUSE);
- if (!REG_TEST_FLD(MTL_BUTTRESS_TILE_FUSE, VALID, tile_fuse))
- ivpu_warn(vdev, "Tile Fuse: Invalid (0x%x)\n", tile_fuse);
-
- hw->tile_fuse = REG_GET_FLD(MTL_BUTTRESS_TILE_FUSE, SKU, tile_fuse);
- switch (hw->tile_fuse) {
- case TILE_FUSE_ENABLE_LOWER:
- hw->sku = TILE_SKU_LOWER_MTL;
- hw->config = WP_CONFIG_1_TILE_5_3_RATIO;
- ivpu_dbg(vdev, MISC, "Tile Fuse: Enable Lower\n");
- break;
- case TILE_FUSE_ENABLE_UPPER:
- hw->sku = TILE_SKU_UPPER_MTL;
- hw->config = WP_CONFIG_1_TILE_4_3_RATIO;
- ivpu_dbg(vdev, MISC, "Tile Fuse: Enable Upper\n");
- break;
- case TILE_FUSE_ENABLE_BOTH:
- hw->sku = TILE_SKU_BOTH_MTL;
- hw->config = WP_CONFIG_2_TILE_5_3_RATIO;
- ivpu_dbg(vdev, MISC, "Tile Fuse: Enable Both\n");
- break;
- default:
- hw->config = WP_CONFIG_0_TILE_PLL_OFF;
- ivpu_dbg(vdev, MISC, "Tile Fuse: Disable\n");
- break;
- }
+
+ hw->tile_fuse = TILE_FUSE_ENABLE_BOTH;
+ hw->sku = TILE_SKU_BOTH_MTL;
+ hw->config = WP_CONFIG_2_TILE_4_3_RATIO;
ivpu_pll_init_frequency_ratios(vdev);
@@ -797,21 +754,8 @@ static int ivpu_hw_mtl_power_down(struct ivpu_device *vdev)
{
int ret = 0;
- /* FPGA requires manual clearing of IP_Reset bit by enabling quiescent state */
- if (ivpu_is_fpga(vdev)) {
- if (ivpu_boot_host_ss_top_noc_disable(vdev)) {
- ivpu_err(vdev, "Failed to disable TOP NOC\n");
- ret = -EIO;
- }
-
- if (ivpu_boot_host_ss_axi_disable(vdev)) {
- ivpu_err(vdev, "Failed to disable AXI\n");
- ret = -EIO;
- }
- }
-
- if (ivpu_boot_pwr_domain_disable(vdev)) {
- ivpu_err(vdev, "Failed to disable power domain\n");
+ if (ivpu_hw_mtl_reset(vdev)) {
+ ivpu_err(vdev, "Failed to reset the VPU\n");
ret = -EIO;
}
@@ -844,6 +788,19 @@ static void ivpu_hw_mtl_wdt_disable(struct ivpu_device *vdev)
REGV_WR32(MTL_VPU_CPU_SS_TIM_GEN_CONFIG, val);
}
+static u32 ivpu_hw_mtl_pll_to_freq(u32 ratio, u32 config)
+{
+ u32 pll_clock = PLL_REF_CLK_FREQ * ratio;
+ u32 cpu_clock;
+
+ if ((config & 0xff) == PLL_RATIO_4_3)
+ cpu_clock = pll_clock * 2 / 4;
+ else
+ cpu_clock = pll_clock * 2 / 5;
+
+ return cpu_clock;
+}
+
/* Register indirect accesses */
static u32 ivpu_hw_mtl_reg_pll_freq_get(struct ivpu_device *vdev)
{
@@ -855,7 +812,7 @@ static u32 ivpu_hw_mtl_reg_pll_freq_get(struct ivpu_device *vdev)
if (!ivpu_is_silicon(vdev))
return PLL_SIMULATION_FREQ;
- return PLL_RATIO_TO_FREQ(pll_curr_ratio);
+ return ivpu_hw_mtl_pll_to_freq(pll_curr_ratio, vdev->hw->config);
}
static u32 ivpu_hw_mtl_reg_telemetry_offset_get(struct ivpu_device *vdev)
diff --git a/drivers/accel/ivpu/ivpu_ipc.h b/drivers/accel/ivpu/ivpu_ipc.h
index 9838202ecfad..68f5b6668e00 100644
--- a/drivers/accel/ivpu/ivpu_ipc.h
+++ b/drivers/accel/ivpu/ivpu_ipc.h
@@ -21,7 +21,7 @@ struct ivpu_bo;
#define IVPU_IPC_ALIGNMENT 64
#define IVPU_IPC_HDR_FREE 0
-#define IVPU_IPC_HDR_ALLOCATED 0
+#define IVPU_IPC_HDR_ALLOCATED 1
/**
* struct ivpu_ipc_hdr - The IPC message header structure, exchanged
diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index 94068aedf97c..3c6f1e16cf2f 100644
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -461,26 +461,22 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
- ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
- &acquire_ctx);
+ ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, 1, &acquire_ctx);
if (ret) {
ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
return ret;
}
- for (i = 0; i < buf_count; i++) {
- ret = dma_resv_reserve_fences(job->bos[i]->base.resv, 1);
- if (ret) {
- ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
- goto unlock_reservations;
- }
+ ret = dma_resv_reserve_fences(bo->base.resv, 1);
+ if (ret) {
+ ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
+ goto unlock_reservations;
}
- for (i = 0; i < buf_count; i++)
- dma_resv_add_fence(job->bos[i]->base.resv, job->done_fence, DMA_RESV_USAGE_WRITE);
+ dma_resv_add_fence(bo->base.resv, job->done_fence, DMA_RESV_USAGE_WRITE);
unlock_reservations:
- drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
+ drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, 1, &acquire_ctx);
wmb(); /* Flush write combining buffers */
@@ -489,12 +485,12 @@ unlock_reservations:
int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
- int ret = 0;
struct ivpu_file_priv *file_priv = file->driver_priv;
struct ivpu_device *vdev = file_priv->vdev;
struct drm_ivpu_submit *params = data;
struct ivpu_job *job;
u32 *buf_handles;
+ int idx, ret;
if (params->engine > DRM_IVPU_ENGINE_COPY)
return -EINVAL;
@@ -523,6 +519,11 @@ int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
goto free_handles;
}
+ if (!drm_dev_enter(&vdev->drm, &idx)) {
+ ret = -ENODEV;
+ goto free_handles;
+ }
+
ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
file_priv->ctx.id, params->buffer_count);
@@ -530,7 +531,7 @@ int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (!job) {
ivpu_err(vdev, "Failed to create job\n");
ret = -ENOMEM;
- goto free_handles;
+ goto dev_exit;
}
ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
@@ -548,6 +549,8 @@ int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
job_put:
job_put(job);
+dev_exit:
+ drm_dev_exit(idx);
free_handles:
kfree(buf_handles);
diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index 553bcbd787b3..bde42d6383da 100644
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -98,12 +98,18 @@ retry:
static void ivpu_pm_recovery_work(struct work_struct *work)
{
struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, recovery_work);
- struct ivpu_device *vdev = pm->vdev;
+ struct ivpu_device *vdev = pm->vdev;
char *evt[2] = {"IVPU_PM_EVENT=IVPU_RECOVER", NULL};
int ret;
- ret = pci_reset_function(to_pci_dev(vdev->drm.dev));
- if (ret)
+retry:
+ ret = pci_try_reset_function(to_pci_dev(vdev->drm.dev));
+ if (ret == -EAGAIN && !drm_dev_is_unplugged(&vdev->drm)) {
+ cond_resched();
+ goto retry;
+ }
+
+ if (ret && ret != -EAGAIN)
ivpu_err(vdev, "Failed to reset VPU: %d\n", ret);
kobject_uevent_env(&vdev->drm.dev->kobj, KOBJ_CHANGE, evt);
@@ -134,32 +140,28 @@ int ivpu_pm_suspend_cb(struct device *dev)
{
struct drm_device *drm = dev_get_drvdata(dev);
struct ivpu_device *vdev = to_ivpu_device(drm);
- int ret;
+ unsigned long timeout;
ivpu_dbg(vdev, PM, "Suspend..\n");
- ret = ivpu_suspend(vdev);
- if (ret && vdev->pm->suspend_reschedule_counter) {
- ivpu_dbg(vdev, PM, "Failed to enter idle, rescheduling suspend, retries left %d\n",
- vdev->pm->suspend_reschedule_counter);
- pm_schedule_suspend(dev, vdev->timeout.reschedule_suspend);
- vdev->pm->suspend_reschedule_counter--;
- return -EBUSY;
- } else if (!vdev->pm->suspend_reschedule_counter) {
- ivpu_warn(vdev, "Failed to enter idle, force suspend\n");
- ivpu_pm_prepare_cold_boot(vdev);
- } else {
- ivpu_pm_prepare_warm_boot(vdev);
+ timeout = jiffies + msecs_to_jiffies(vdev->timeout.tdr);
+ while (!ivpu_hw_is_idle(vdev)) {
+ cond_resched();
+ if (time_after_eq(jiffies, timeout)) {
+ ivpu_err(vdev, "Failed to enter idle on system suspend\n");
+ return -EBUSY;
+ }
}
- vdev->pm->suspend_reschedule_counter = PM_RESCHEDULE_LIMIT;
+ ivpu_suspend(vdev);
+ ivpu_pm_prepare_warm_boot(vdev);
pci_save_state(to_pci_dev(dev));
pci_set_power_state(to_pci_dev(dev), PCI_D3hot);
ivpu_dbg(vdev, PM, "Suspend done.\n");
- return ret;
+ return 0;
}
int ivpu_pm_resume_cb(struct device *dev)
@@ -306,6 +308,11 @@ int ivpu_pm_init(struct ivpu_device *vdev)
return 0;
}
+void ivpu_pm_cancel_recovery(struct ivpu_device *vdev)
+{
+ cancel_work_sync(&vdev->pm->recovery_work);
+}
+
void ivpu_pm_enable(struct ivpu_device *vdev)
{
struct device *dev = vdev->drm.dev;
diff --git a/drivers/accel/ivpu/ivpu_pm.h b/drivers/accel/ivpu/ivpu_pm.h
index dc1b3758e13f..baca98187255 100644
--- a/drivers/accel/ivpu/ivpu_pm.h
+++ b/drivers/accel/ivpu/ivpu_pm.h
@@ -21,6 +21,7 @@ struct ivpu_pm_info {
int ivpu_pm_init(struct ivpu_device *vdev);
void ivpu_pm_enable(struct ivpu_device *vdev);
void ivpu_pm_disable(struct ivpu_device *vdev);
+void ivpu_pm_cancel_recovery(struct ivpu_device *vdev);
int ivpu_pm_suspend_cb(struct device *dev);
int ivpu_pm_resume_cb(struct device *dev);
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-20 09:18:49 +0000