diff options
Diffstat (limited to 'drivers/gpu/drm')
229 files changed, 3269 insertions, 1546 deletions
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index d3a9ca4b1cec..594ac6c3a1f4 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -99,7 +99,6 @@ config DRM_FBDEV_EMULATION bool "Enable legacy fbdev support for your modesetting driver" depends on DRM select DRM_KMS_HELPER - select DRM_KMS_FB_HELPER select FB select FB_CFB_FILLRECT select FB_CFB_COPYAREA diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 93972dcd94ca..267e9e147a6c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1068,7 +1068,7 @@ struct amdgpu_device { uint32_t ras_hw_enabled; uint32_t ras_enabled; - bool in_pci_err_recovery; + bool no_hw_access; struct pci_saved_state *pci_state; struct amdgpu_reset_control *reset_cntl; @@ -1091,7 +1091,9 @@ static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_device *bdev) int amdgpu_device_init(struct amdgpu_device *adev, uint32_t flags); -void amdgpu_device_fini(struct amdgpu_device *adev); +void amdgpu_device_fini_hw(struct amdgpu_device *adev); +void amdgpu_device_fini_sw(struct amdgpu_device *adev); + int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev); void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, @@ -1313,6 +1315,8 @@ void amdgpu_driver_lastclose_kms(struct drm_device *dev); int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv); void amdgpu_driver_postclose_kms(struct drm_device *dev, struct drm_file *file_priv); +void amdgpu_driver_release_kms(struct drm_device *dev); + int amdgpu_device_ip_suspend(struct amdgpu_device *adev); int amdgpu_device_suspend(struct drm_device *dev, bool fbcon); int amdgpu_device_resume(struct drm_device *dev, bool fbcon); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index bfab2f9fdd17..f9c01bdc3d4c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -170,7 +170,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) } } -void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev) +void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev) { if (adev->kfd.dev) { kgd2kfd_device_exit(adev->kfd.dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 81264517d532..cf62f43a03da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -142,7 +142,7 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, const void *ih_ring_entry); void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev); void amdgpu_amdkfd_device_init(struct amdgpu_device *adev); -void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev); +void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev); int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, uint32_t vmid, uint64_t gpu_addr, uint32_t *ib_cmd, uint32_t ib_len); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 18bd1b49ced6..96b7bb13a2dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -1768,6 +1768,15 @@ static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev, static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version, NULL); +static struct attribute *amdgpu_vbios_version_attrs[] = { + &dev_attr_vbios_version.attr, + NULL +}; + +const struct attribute_group amdgpu_vbios_version_attr_group = { + .attrs = amdgpu_vbios_version_attrs +}; + /** * amdgpu_atombios_fini - free the driver info and callbacks for atombios * @@ -1787,7 +1796,6 @@ void amdgpu_atombios_fini(struct amdgpu_device *adev) adev->mode_info.atom_context = NULL; kfree(adev->mode_info.atom_card_info); adev->mode_info.atom_card_info = NULL; - device_remove_file(adev->dev, &dev_attr_vbios_version); } /** @@ -1804,7 +1812,6 @@ int amdgpu_atombios_init(struct amdgpu_device *adev) { struct card_info *atom_card_info = kzalloc(sizeof(struct card_info), GFP_KERNEL); - int ret; if (!atom_card_info) return -ENOMEM; @@ -1836,12 +1843,6 @@ int amdgpu_atombios_init(struct amdgpu_device *adev) amdgpu_atombios_allocate_fb_scratch(adev); } - ret = device_create_file(adev->dev, &dev_attr_vbios_version); - if (ret) { - DRM_ERROR("Failed to create device file for VBIOS version\n"); - return ret; - } - return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index f364d5afb2ad..0585442b000e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -71,6 +71,8 @@ #include <drm/task_barrier.h> #include <linux/pm_runtime.h> +#include <drm/drm_drv.h> + MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); @@ -297,7 +299,10 @@ void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, unsigned long flags; uint32_t hi = ~0; uint64_t last; + int idx; + if (!drm_dev_enter(&adev->ddev, &idx)) + return; #ifdef CONFIG_64BIT last = min(pos + size, adev->gmc.visible_vram_size); @@ -316,7 +321,7 @@ void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, } if (count == size) - return; + goto exit; pos += count; buf += count / 4; @@ -339,6 +344,11 @@ void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, *buf++ = RREG32_NO_KIQ(mmMM_DATA); } spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + +#ifdef CONFIG_64BIT +exit: +#endif + drm_dev_exit(idx); } /* @@ -348,7 +358,7 @@ void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, /* Check if hw access should be skipped because of hotplug or device error */ bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) { - if (adev->in_pci_err_recovery) + if (adev->no_hw_access) return true; #ifdef CONFIG_LOCKDEP @@ -2589,34 +2599,26 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) return 0; } -/** - * amdgpu_device_ip_fini - run fini for hardware IPs - * - * @adev: amdgpu_device pointer - * - * Main teardown pass for hardware IPs. The list of all the hardware - * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks - * are run. hw_fini tears down the hardware associated with each IP - * and sw_fini tears down any software state associated with each IP. - * Returns 0 on success, negative error code on failure. - */ -static int amdgpu_device_ip_fini(struct amdgpu_device *adev) +static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) { int i, r; - if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) - amdgpu_virt_release_ras_err_handler_data(adev); + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].version->funcs->early_fini) + continue; - amdgpu_ras_pre_fini(adev); + r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev); + if (r) { + DRM_DEBUG("early_fini of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + } - if (adev->gmc.xgmi.num_physical_nodes > 1) - amdgpu_xgmi_remove_device(adev); + amdgpu_amdkfd_suspend(adev, false); amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); - amdgpu_amdkfd_device_fini(adev); - /* need to disable SMC first */ for (i = 0; i < adev->num_ip_blocks; i++) { if (!adev->ip_blocks[i].status.hw) @@ -2647,6 +2649,33 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) adev->ip_blocks[i].status.hw = false; } + return 0; +} + +/** + * amdgpu_device_ip_fini - run fini for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main teardown pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks + * are run. hw_fini tears down the hardware associated with each IP + * and sw_fini tears down any software state associated with each IP. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_fini(struct amdgpu_device *adev) +{ + int i, r; + + if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) + amdgpu_virt_release_ras_err_handler_data(adev); + + amdgpu_ras_pre_fini(adev); + + if (adev->gmc.xgmi.num_physical_nodes > 1) + amdgpu_xgmi_remove_device(adev); + + amdgpu_amdkfd_device_fini_sw(adev); for (i = adev->num_ip_blocks - 1; i >= 0; i--) { if (!adev->ip_blocks[i].status.sw) @@ -3268,7 +3297,6 @@ static const struct attribute *amdgpu_dev_attributes[] = { NULL }; - /** * amdgpu_device_init - initialize the driver * @@ -3670,6 +3698,27 @@ failed_unmap: return r; } +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) +{ + /* Clear all CPU mappings pointing to this device */ + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); + + /* Unmap all mapped bars - Doorbell, registers and VRAM */ + amdgpu_device_doorbell_fini(adev); + + iounmap(adev->rmmio); + adev->rmmio = NULL; + if (adev->mman.aper_base_kaddr) + iounmap(adev->mman.aper_base_kaddr); + adev->mman.aper_base_kaddr = NULL; + + /* Memory manager related */ + if (!adev->gmc.xgmi.connected_to_cpu) { + arch_phys_wc_del(adev->gmc.vram_mtrr); + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); + } +} + /** * amdgpu_device_fini - tear down the driver * @@ -3678,15 +3727,13 @@ failed_unmap: * Tear down the driver info (all asics). * Called at driver shutdown. */ -void amdgpu_device_fini(struct amdgpu_device *adev) +void amdgpu_device_fini_hw(struct amdgpu_device *adev) { dev_info(adev->dev, "amdgpu: finishing device.\n"); flush_delayed_work(&adev->delayed_init_work); ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); adev->shutdown = true; - kfree(adev->pci_state); - /* make sure IB test finished before entering exclusive mode * to avoid preemption on IB test * */ @@ -3703,11 +3750,29 @@ void amdgpu_device_fini(struct amdgpu_device *adev) else drm_atomic_helper_shutdown(adev_to_drm(adev)); } - amdgpu_fence_driver_fini(adev); + amdgpu_fence_driver_fini_hw(adev); + if (adev->pm_sysfs_en) amdgpu_pm_sysfs_fini(adev); + if (adev->ucode_sysfs_en) + amdgpu_ucode_sysfs_fini(adev); + sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); + amdgpu_fbdev_fini(adev); + + amdgpu_irq_fini_hw(adev); + + amdgpu_device_ip_fini_early(adev); + + amdgpu_gart_dummy_page_fini(adev); + + amdgpu_device_unmap_mmio(adev); +} + +void amdgpu_device_fini_sw(struct amdgpu_device *adev) +{ amdgpu_device_ip_fini(adev); + amdgpu_fence_driver_fini_sw(adev); release_firmware(adev->firmware.gpu_info_fw); adev->firmware.gpu_info_fw = NULL; adev->accel_working = false; @@ -3729,18 +3794,14 @@ void amdgpu_device_fini(struct amdgpu_device *adev) } if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) vga_client_register(adev->pdev, NULL, NULL, NULL); - iounmap(adev->rmmio); - adev->rmmio = NULL; - amdgpu_device_doorbell_fini(adev); - if (adev->ucode_sysfs_en) - amdgpu_ucode_sysfs_fini(adev); - - sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); if (IS_ENABLED(CONFIG_PERF_EVENTS)) amdgpu_pmu_fini(adev); if (adev->mman.discovery_bin) amdgpu_discovery_fini(adev); + + kfree(adev->pci_state); + } @@ -5317,9 +5378,9 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); - adev->in_pci_err_recovery = true; + adev->no_hw_access = true; r = amdgpu_device_pre_asic_reset(adev, &reset_context); - adev->in_pci_err_recovery = false; + adev->no_hw_access = false; if (r) goto out; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index 8a1fb8b6606e..49f73b5b89b0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -1077,12 +1077,9 @@ int amdgpu_display_gem_fb_verify_and_init( /* Verify that the modifier is supported. */ if (!drm_any_plane_has_format(dev, mode_cmd->pixel_format, mode_cmd->modifier[0])) { - struct drm_format_name_buf format_name; drm_dbg_kms(dev, - "unsupported pixel format %s / modifier 0x%llx\n", - drm_get_format_name(mode_cmd->pixel_format, - &format_name), - mode_cmd->modifier[0]); + "unsupported pixel format %p4cc / modifier 0x%llx\n", + &mode_cmd->pixel_format, mode_cmd->modifier[0]); ret = -EINVAL; goto err; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index baa980a477d9..6ec1312b7389 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -42,52 +42,6 @@ #include <linux/pci-p2pdma.h> #include <linux/pm_runtime.h> -/** - * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation - * @obj: GEM BO - * @vma: Virtual memory area - * - * Sets up a userspace mapping of the BO's memory in the given - * virtual memory area. - * - * Returns: - * 0 on success or a negative error code on failure. - */ -int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, - struct vm_area_struct *vma) -{ - struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); - struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - unsigned asize = amdgpu_bo_size(bo); - int ret; - - if (!vma->vm_file) - return -ENODEV; - - if (adev == NULL) - return -ENODEV; - - /* Check for valid size. */ - if (asize < vma->vm_end - vma->vm_start) - return -EINVAL; - - if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || - (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { - return -EPERM; - } - vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT; - - /* prime mmap does not need to check access, so allow here */ - ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data); - if (ret) - return ret; - - ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev); - drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data); - - return ret; -} - static int __dma_resv_make_exclusive(struct dma_resv *obj) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h index 39b5b9616fd8..3e93b9b407a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h @@ -31,8 +31,6 @@ struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf); bool amdgpu_dmabuf_is_xgmi_accessible(struct amdgpu_device *adev, struct amdgpu_bo *bo); -int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, - struct vm_area_struct *vma); extern const struct dma_buf_ops amdgpu_dmabuf_ops; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index f576426e24fc..809aa7641d7e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -1314,14 +1314,16 @@ amdgpu_pci_remove(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); -#ifdef MODULE - if (THIS_MODULE->state != MODULE_STATE_GOING) -#endif - DRM_ERROR("Hotplug removal is not supported\n"); drm_dev_unplug(dev); amdgpu_driver_unload_kms(dev); + + /* + * Flush any in flight DMA operations from device. + * Clear the Bus Master Enable bit and then wait on the PCIe Device + * StatusTransactions Pending bit. + */ pci_disable_device(pdev); - pci_set_drvdata(pdev, NULL); + pci_wait_for_pending_transaction(pdev); } static void @@ -1556,6 +1558,10 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) if (!adev->runpm) return -EINVAL; + /* Avoids registers access if device is physically gone */ + if (!pci_device_is_present(adev->pdev)) + adev->no_hw_access = true; + if (amdgpu_device_supports_px(drm_dev)) { drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; @@ -1690,7 +1696,7 @@ static const struct file_operations amdgpu_driver_kms_fops = { .flush = amdgpu_flush, .release = drm_release, .unlocked_ioctl = amdgpu_drm_ioctl, - .mmap = amdgpu_mmap, + .mmap = drm_gem_mmap, .poll = drm_poll, .read = drm_read, #ifdef CONFIG_COMPAT @@ -1752,11 +1758,12 @@ static const struct drm_driver amdgpu_kms_driver = { .dumb_create = amdgpu_mode_dumb_create, .dumb_map_offset = amdgpu_mode_dumb_mmap, .fops = &amdgpu_driver_kms_fops, + .release = &amdgpu_driver_release_kms, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_import = amdgpu_gem_prime_import, - .gem_prime_mmap = amdgpu_gem_prime_mmap, + .gem_prime_mmap = drm_gem_prime_mmap, .name = DRIVER_NAME, .desc = DRIVER_DESC, @@ -1773,6 +1780,18 @@ static struct pci_error_handlers amdgpu_pci_err_handler = { .resume = amdgpu_pci_resume, }; +extern const struct attribute_group amdgpu_vram_mgr_attr_group; +extern const struct attribute_group amdgpu_gtt_mgr_attr_group; +extern const struct attribute_group amdgpu_vbios_version_attr_group; + +static const struct attribute_group *amdgpu_sysfs_groups[] = { + &amdgpu_vram_mgr_attr_group, + &amdgpu_gtt_mgr_attr_group, + &amdgpu_vbios_version_attr_group, + NULL, +}; + + static struct pci_driver amdgpu_kms_pci_driver = { .name = DRIVER_NAME, .id_table = pciidlist, @@ -1781,6 +1800,7 @@ static struct pci_driver amdgpu_kms_pci_driver = { .shutdown = amdgpu_pci_shutdown, .driver.pm = &amdgpu_pm_ops, .err_handler = &amdgpu_pci_err_handler, + .dev_groups = amdgpu_sysfs_groups, }; static int __init amdgpu_init(void) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 30772608eac6..72d9b92b1754 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -36,6 +36,7 @@ #include <linux/firmware.h> #include <linux/pm_runtime.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_trace.h" @@ -524,10 +525,9 @@ int amdgpu_fence_driver_init(struct amdgpu_device *adev) * * Tear down the fence driver for all possible rings (all asics). */ -void amdgpu_fence_driver_fini(struct amdgpu_device *adev) +void amdgpu_fence_driver_fini_hw(struct amdgpu_device *adev) { - unsigned i, j; - int r; + int i, r; for (i = 0; i < AMDGPU_MAX_RINGS; i++) { struct amdgpu_ring *ring = adev->rings[i]; @@ -536,16 +536,33 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev) continue; if (!ring->no_scheduler) drm_sched_fini(&ring->sched); - r = amdgpu_fence_wait_empty(ring); - if (r) { - /* no need to trigger GPU reset as we are unloading */ + /* You can't wait for HW to signal if it's gone */ + if (!drm_dev_is_unplugged(&adev->ddev)) + r = amdgpu_fence_wait_empty(ring); + else + r = -ENODEV; + /* no need to trigger GPU reset as we are unloading */ + if (r) amdgpu_fence_driver_force_completion(ring); - } + if (ring->fence_drv.irq_src) amdgpu_irq_put(adev, ring->fence_drv.irq_src, ring->fence_drv.irq_type); del_timer_sync(&ring->fence_drv.fallback_timer); + } +} + +void amdgpu_fence_driver_fini_sw(struct amdgpu_device *adev) +{ + unsigned int i, j; + + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + + if (!ring || !ring->fence_drv.initialized) + continue; + for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j) dma_fence_put(ring->fence_drv.fences[j]); kfree(ring->fence_drv.fences); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c index 35cc8009ac7a..34243e122d11 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c @@ -92,7 +92,7 @@ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) * * Frees the dummy page used by the driver (all asics). */ -static void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev) +void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev) { if (!adev->dummy_page_addr) return; @@ -373,15 +373,3 @@ int amdgpu_gart_init(struct amdgpu_device *adev) return 0; } - -/** - * amdgpu_gart_fini - tear down the driver info for managing the gart - * - * @adev: amdgpu_device pointer - * - * Tear down the gart driver info and free the dummy page (all asics). - */ -void amdgpu_gart_fini(struct amdgpu_device *adev) -{ - amdgpu_gart_dummy_page_fini(adev); -} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h index e104022197ae..f53f6a7b9b60 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h @@ -57,7 +57,7 @@ void amdgpu_gart_table_vram_free(struct amdgpu_device *adev); int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev); void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev); int amdgpu_gart_init(struct amdgpu_device *adev); -void amdgpu_gart_fini(struct amdgpu_device *adev); +void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev); int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, int pages); int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 18974bd081f0..73c76a3e2b12 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -32,6 +32,7 @@ #include <linux/dma-buf.h> #include <drm/amdgpu_drm.h> +#include <drm/drm_drv.h> #include <drm/drm_gem_ttm_helper.h> #include "amdgpu.h" @@ -41,6 +42,46 @@ static const struct drm_gem_object_funcs amdgpu_gem_object_funcs; +static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) +{ + struct ttm_buffer_object *bo = vmf->vma->vm_private_data; + struct drm_device *ddev = bo->base.dev; + vm_fault_t ret; + int idx; + + ret = ttm_bo_vm_reserve(bo, vmf); + if (ret) + return ret; + + if (drm_dev_enter(ddev, &idx)) { + ret = amdgpu_bo_fault_reserve_notify(bo); + if (ret) { + drm_dev_exit(idx); + goto unlock; + } + + ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, + TTM_BO_VM_NUM_PREFAULT, 1); + + drm_dev_exit(idx); + } else { + ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); + } + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return ret; + +unlock: + dma_resv_unlock(bo->base.resv); + return ret; +} + +static const struct vm_operations_struct amdgpu_gem_vm_ops = { + .fault = amdgpu_gem_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access +}; + static void amdgpu_gem_object_free(struct drm_gem_object *gobj) { struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj); @@ -205,6 +246,18 @@ out_unlock: ttm_eu_backoff_reservation(&ticket, &list); } +static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) + return -EPERM; + if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) + return -EPERM; + + return drm_gem_ttm_mmap(obj, vma); +} + static const struct drm_gem_object_funcs amdgpu_gem_object_funcs = { .free = amdgpu_gem_object_free, .open = amdgpu_gem_object_open, @@ -212,6 +265,8 @@ static const struct drm_gem_object_funcs amdgpu_gem_object_funcs = { .export = amdgpu_gem_prime_export, .vmap = drm_gem_ttm_vmap, .vunmap = drm_gem_ttm_vunmap, + .mmap = amdgpu_gem_object_mmap, + .vm_ops = &amdgpu_gem_vm_ops, }; /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 202e6f959ac4..7061c4a0e421 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -31,6 +31,8 @@ #include "amdgpu_ras.h" #include "amdgpu_xgmi.h" +#include <drm/drm_drv.h> + /** * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0 * @@ -151,6 +153,10 @@ int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, { void __iomem *ptr = (void *)cpu_pt_addr; uint64_t value; + int idx; + + if (!drm_dev_enter(&adev->ddev, &idx)) + return 0; /* * The following is for PTE only. GART does not have PDEs. @@ -158,6 +164,9 @@ int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, value = addr & 0x0000FFFFFFFFF000ULL; value |= flags; writeq(value, ptr + (gpu_page_idx * 8)); + + drm_dev_exit(idx); + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index 327da885eca9..9ab33048923e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -80,6 +80,16 @@ static DEVICE_ATTR(mem_info_gtt_total, S_IRUGO, static DEVICE_ATTR(mem_info_gtt_used, S_IRUGO, amdgpu_mem_info_gtt_used_show, NULL); +static struct attribute *amdgpu_gtt_mgr_attributes[] = { + &dev_attr_mem_info_gtt_total.attr, + &dev_attr_mem_info_gtt_used.attr, + NULL +}; + +const struct attribute_group amdgpu_gtt_mgr_attr_group = { + .attrs = amdgpu_gtt_mgr_attributes +}; + /** * amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space * @@ -267,7 +277,6 @@ int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size) struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr; struct ttm_resource_manager *man = &mgr->manager; uint64_t start, size; - int ret; man->use_tt = true; man->func = &amdgpu_gtt_mgr_func; @@ -280,17 +289,6 @@ int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size) spin_lock_init(&mgr->lock); atomic64_set(&mgr->available, gtt_size >> PAGE_SHIFT); - ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total); - if (ret) { - DRM_ERROR("Failed to create device file mem_info_gtt_total\n"); - return ret; - } - ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used); - if (ret) { - DRM_ERROR("Failed to create device file mem_info_gtt_used\n"); - return ret; - } - ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager); ttm_resource_manager_set_used(man, true); return 0; @@ -320,9 +318,6 @@ void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev) drm_mm_takedown(&mgr->mm); spin_unlock(&mgr->lock); - device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total); - device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used); - ttm_resource_manager_cleanup(man); ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index a36e191cf086..f3d62e196901 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -115,9 +115,11 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, */ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { + + if (!ih->ring) + return; + if (ih->use_bus_addr) { - if (!ih->ring) - return; /* add 8 bytes for the rptr/wptr shadows and * add them to the end of the ring allocation. diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 90f50561b43a..32ce0e679dc7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -49,6 +49,7 @@ #include <drm/drm_irq.h> #include <drm/drm_vblank.h> #include <drm/amdgpu_drm.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_ih.h" #include "atom.h" @@ -348,6 +349,25 @@ int amdgpu_irq_init(struct amdgpu_device *adev) return 0; } + +void amdgpu_irq_fini_hw(struct amdgpu_device *adev) +{ + if (adev->irq.installed) { + drm_irq_uninstall(&adev->ddev); + adev->irq.installed = false; + if (adev->irq.msi_enabled) + pci_free_irq_vectors(adev->pdev); + + if (!amdgpu_device_has_dc_support(adev)) + flush_work(&adev->hotplug_work); + } + + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); + amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_ih_ring_fini(adev, &adev->irq.ih1); + amdgpu_ih_ring_fini(adev, &adev->irq.ih2); +} + /** * amdgpu_irq_fini - shut down interrupt handling * @@ -357,19 +377,10 @@ int amdgpu_irq_init(struct amdgpu_device *adev) * functionality, shuts down vblank, hotplug and reset interrupt handling, * turns off interrupts from all sources (all ASICs). */ -void amdgpu_irq_fini(struct amdgpu_device *adev) +void amdgpu_irq_fini_sw(struct amdgpu_device *adev) { unsigned i, j; - if (adev->irq.installed) { - drm_irq_uninstall(adev_to_drm(adev)); - adev->irq.installed = false; - if (adev->irq.msi_enabled) - pci_free_irq_vectors(adev->pdev); - if (!amdgpu_device_has_dc_support(adev)) - flush_work(&adev->hotplug_work); - } - for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) { if (!adev->irq.client[i].sources) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h index cf6116648322..78ad4784cc74 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h @@ -103,7 +103,8 @@ void amdgpu_irq_disable_all(struct amdgpu_device *adev); irqreturn_t amdgpu_irq_handler(int irq, void *arg); int amdgpu_irq_init(struct amdgpu_device *adev); -void amdgpu_irq_fini(struct amdgpu_device *adev); +void amdgpu_irq_fini_sw(struct amdgpu_device *adev); +void amdgpu_irq_fini_hw(struct amdgpu_device *adev); int amdgpu_irq_add_id(struct amdgpu_device *adev, unsigned client_id, unsigned src_id, struct amdgpu_irq_src *source); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 759b34799221..d33e6d97cc89 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -25,6 +25,8 @@ #include <linux/wait.h> #include <linux/sched.h> +#include <drm/drm_drv.h> + #include "amdgpu.h" #include "amdgpu_trace.h" @@ -34,6 +36,15 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) struct amdgpu_job *job = to_amdgpu_job(s_job); struct amdgpu_task_info ti; struct amdgpu_device *adev = ring->adev; + int idx; + + if (!drm_dev_enter(&adev->ddev, &idx)) { + DRM_INFO("%s - device unplugged skipping recovery on scheduler:%s", + __func__, s_job->sched->name); + + /* Effectively the job is aborted as the device is gone */ + return DRM_GPU_SCHED_STAT_ENODEV; + } memset(&ti, 0, sizeof(struct amdgpu_task_info)); @@ -41,7 +52,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) { DRM_ERROR("ring %s timeout, but soft recovered\n", s_job->sched->name); - return DRM_GPU_SCHED_STAT_NOMINAL; + goto exit; } amdgpu_vm_get_task_info(ring->adev, job->pasid, &ti); @@ -53,13 +64,15 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) if (amdgpu_device_should_recover_gpu(ring->adev)) { amdgpu_device_gpu_recover(ring->adev, job); - return DRM_GPU_SCHED_STAT_NOMINAL; } else { drm_sched_suspend_timeout(&ring->sched); if (amdgpu_sriov_vf(adev)) adev->virt.tdr_debug = true; - return DRM_GPU_SCHED_STAT_NOMINAL; } + +exit: + drm_dev_exit(idx); + return DRM_GPU_SCHED_STAT_NOMINAL; } int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index c354ffa62483..425596c2f6f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -28,6 +28,7 @@ #include "amdgpu.h" #include <drm/amdgpu_drm.h> +#include <drm/drm_drv.h> #include "amdgpu_uvd.h" #include "amdgpu_vce.h" #include "atom.h" @@ -95,7 +96,7 @@ void amdgpu_driver_unload_kms(struct drm_device *dev) DRM_WARN("smart shift update failed\n"); amdgpu_acpi_fini(adev); - amdgpu_device_fini(adev); + amdgpu_device_fini_hw(adev); } void amdgpu_register_gpu_instance(struct amdgpu_device *adev) @@ -1241,6 +1242,15 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, pm_runtime_put_autosuspend(dev->dev); } + +void amdgpu_driver_release_kms(struct drm_device *dev) +{ + struct amdgpu_device *adev = drm_to_adev(dev); + + amdgpu_device_fini_sw(adev); + pci_set_drvdata(adev->pdev, NULL); +} + /* * VBlank related functions. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index df531f287283..a6fa3968463a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1093,10 +1093,6 @@ int amdgpu_bo_init(struct amdgpu_device *adev) void amdgpu_bo_fini(struct amdgpu_device *adev) { amdgpu_ttm_fini(adev); - if (!adev->gmc.xgmi.connected_to_cpu) { - arch_phys_wc_del(adev->gmc.vram_mtrr); - arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); - } } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 55378c6b9722..3ff76cbaec8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -25,6 +25,7 @@ #include <linux/firmware.h> #include <linux/dma-mapping.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_psp.h" @@ -40,6 +41,8 @@ #include "amdgpu_securedisplay.h" #include "amdgpu_atomfirmware.h" +#include <drm/drm_drv.h> + static int psp_sysfs_init(struct amdgpu_device *adev); static void psp_sysfs_fini(struct amdgpu_device *adev); @@ -231,7 +234,7 @@ int psp_wait_for(struct psp_context *psp, uint32_t reg_index, int i; struct amdgpu_device *adev = psp->adev; - if (psp->adev->in_pci_err_recovery) + if (psp->adev->no_hw_access) return 0; for (i = 0; i < adev->usec_timeout; i++) { @@ -255,12 +258,15 @@ psp_cmd_submit_buf(struct psp_context *psp, struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr) { int ret; - int index; + int index, idx; int timeout = 20000; bool ras_intr = false; bool skip_unsupport = false; - if (psp->adev->in_pci_err_recovery) + if (psp->adev->no_hw_access) + return 0; + + if (!drm_dev_enter(&psp->adev->ddev, &idx)) return 0; mutex_lock(&psp->mutex); @@ -273,8 +279,7 @@ psp_cmd_submit_buf(struct psp_context *psp, ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index); if (ret) { atomic_dec(&psp->fence_value); - mutex_unlock(&psp->mutex); - return ret; + goto exit; } amdgpu_asic_invalidate_hdp(psp->adev, NULL); @@ -314,8 +319,8 @@ psp_cmd_submit_buf(struct psp_context *psp, psp->cmd_buf_mem->cmd_id, psp->cmd_buf_mem->resp.status); if (!timeout) { - mutex_unlock(&psp->mutex); - return -EINVAL; + ret = -EINVAL; + goto exit; } } @@ -323,8 +328,10 @@ psp_cmd_submit_buf(struct psp_context *psp, ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo; ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi; } - mutex_unlock(&psp->mutex); +exit: + mutex_unlock(&psp->mutex); + drm_dev_exit(idx); return ret; } @@ -368,8 +375,7 @@ static int psp_load_toc(struct psp_context *psp, if (!cmd) return -ENOMEM; /* Copy toc to psp firmware private buffer */ - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size); + psp_copy_fw(psp, psp->toc_start_addr, psp->toc_bin_size); psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size); @@ -604,8 +610,7 @@ static int psp_asd_load(struct psp_context *psp) if (!cmd) return -ENOMEM; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size); + psp_copy_fw(psp, psp->asd_start_addr, psp->asd_ucode_size); psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_ucode_size); @@ -760,8 +765,7 @@ static int psp_xgmi_load(struct psp_context *psp) if (!cmd) return -ENOMEM; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size); + psp_copy_fw(psp, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, @@ -1017,8 +1021,7 @@ static int psp_ras_load(struct psp_context *psp) if (!cmd) return -ENOMEM; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size); + psp_copy_fw(psp, psp->ta_ras_start_addr, psp->ta_ras_ucode_size); ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; @@ -1284,9 +1287,8 @@ static int psp_hdcp_load(struct psp_context *psp) if (!cmd) return -ENOMEM; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr, - psp->ta_hdcp_ucode_size); + psp_copy_fw(psp, psp->ta_hdcp_start_addr, + psp->ta_hdcp_ucode_size); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, @@ -1436,8 +1438,7 @@ static int psp_dtm_load(struct psp_context *psp) if (!cmd) return -ENOMEM; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size); + psp_copy_fw(psp, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, @@ -1582,8 +1583,7 @@ static int psp_rap_load(struct psp_context *psp) if (!cmd) return -ENOMEM; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->ta_rap_start_addr, psp->ta_rap_ucode_size); + psp_copy_fw(psp, psp->ta_rap_start_addr, psp->ta_rap_ucode_size); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, @@ -3020,7 +3020,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, struct amdgpu_device *adev = drm_to_adev(ddev); void *cpu_addr; dma_addr_t dma_addr; - int ret; + int ret, idx; char fw_name[100]; const struct firmware *usbc_pd_fw; @@ -3029,6 +3029,9 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, return -EBUSY; } + if (!drm_dev_enter(ddev, &idx)) + return -ENODEV; + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf); ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev); if (ret) @@ -3060,16 +3063,29 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, rel_buf: dma_free_coherent(adev->dev, usbc_pd_fw->size, cpu_addr, dma_addr); release_firmware(usbc_pd_fw); - fail: if (ret) { DRM_ERROR("Failed to load USBC PD FW, err = %d", ret); - return ret; + count = ret; } + drm_dev_exit(idx); return count; } +void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) +{ + int idx; + + if (!drm_dev_enter(&psp->adev->ddev, &idx)) + return; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, start_addr, bin_size); + + drm_dev_exit(idx); +} + static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR, psp_usbc_pd_fw_sysfs_read, psp_usbc_pd_fw_sysfs_write); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 60aa99a39a74..e5023f1de7fd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -424,4 +424,6 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, int psp_load_fw_list(struct psp_context *psp, struct amdgpu_firmware_info **ucode_list, int ucode_count); +void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index ec936cde2726..9dfc1ebb41a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -2407,6 +2407,7 @@ int amdgpu_ras_pre_fini(struct amdgpu_device *adev) if (!adev->ras_enabled || !con) return 0; + /* Need disable ras on all IPs here before ip [hw/sw]fini */ amdgpu_ras_disable_all_features(adev, 0); amdgpu_ras_recovery_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index ca1622835296..e7d3d0dbdd96 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -107,7 +107,8 @@ struct amdgpu_fence_driver { }; int amdgpu_fence_driver_init(struct amdgpu_device *adev); -void amdgpu_fence_driver_fini(struct amdgpu_device *adev); +void amdgpu_fence_driver_fini_hw(struct amdgpu_device *adev); +void amdgpu_fence_driver_fini_sw(struct amdgpu_device *adev); void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring); int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 86259435803e..832970cff64c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -167,25 +167,6 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo, } /** - * amdgpu_verify_access - Verify access for a mmap call - * - * @bo: The buffer object to map - * @filp: The file pointer from the process performing the mmap - * - * This is called by ttm_bo_mmap() to verify whether a process - * has the right to mmap a BO to their process space. - */ -static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp) -{ - struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); - - if (amdgpu_ttm_tt_get_usermm(bo->ttm)) - return -EPERM; - return drm_vma_node_verify_access(&abo->tbo.base.vma_node, - filp->private_data); -} - -/** * amdgpu_ttm_map_buffer - Map memory into the GART windows * @bo: buffer object to map * @mem: memory object to map @@ -1492,7 +1473,6 @@ static struct ttm_device_funcs amdgpu_bo_driver = { .eviction_valuable = amdgpu_ttm_bo_eviction_valuable, .evict_flags = &amdgpu_evict_flags, .move = &amdgpu_bo_move, - .verify_access = &amdgpu_verify_access, .delete_mem_notify = &amdgpu_bo_delete_mem_notify, .release_notify = &amdgpu_bo_release_notify, .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, @@ -1809,10 +1789,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); amdgpu_ttm_fw_reserve_vram_fini(adev); - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); - adev->mman.aper_base_kaddr = NULL; - amdgpu_vram_mgr_fini(adev); amdgpu_gtt_mgr_fini(adev); amdgpu_preempt_mgr_fini(adev); @@ -1872,50 +1848,6 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) adev->mman.buffer_funcs_enabled = enable; } -static vm_fault_t amdgpu_ttm_fault(struct vm_fault *vmf) -{ - struct ttm_buffer_object *bo = vmf->vma->vm_private_data; - vm_fault_t ret; - - ret = ttm_bo_vm_reserve(bo, vmf); - if (ret) - return ret; - - ret = amdgpu_bo_fault_reserve_notify(bo); - if (ret) - goto unlock; - - ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, - TTM_BO_VM_NUM_PREFAULT, 1); - if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - return ret; - -unlock: - dma_resv_unlock(bo->base.resv); - return ret; -} - -static const struct vm_operations_struct amdgpu_ttm_vm_ops = { - .fault = amdgpu_ttm_fault, - .open = ttm_bo_vm_open, - .close = ttm_bo_vm_close, - .access = ttm_bo_vm_access -}; - -int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma) -{ - struct drm_file *file_priv = filp->private_data; - struct amdgpu_device *adev = drm_to_adev(file_priv->minor->dev); - int r; - - r = ttm_bo_mmap(filp, vma, &adev->mman.bdev); - if (unlikely(r != 0)) - return r; - - vma->vm_ops = &amdgpu_ttm_vm_ops; - return 0; -} - int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count, struct dma_resv *resv, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index 2877a924086f..74a7021592c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -153,7 +153,6 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, struct dma_resv *resv, struct dma_fence **fence); -int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma); int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo); int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo); uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 35f68748ad26..ce8f80a959c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -32,6 +32,7 @@ #include <linux/module.h> #include <drm/drm.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_pm.h" @@ -375,7 +376,7 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev) { unsigned size; void *ptr; - int i, j; + int i, j, idx; bool in_ras_intr = amdgpu_ras_intr_triggered(); cancel_delayed_work_sync(&adev->uvd.idle_work); @@ -403,11 +404,15 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev) if (!adev->uvd.inst[j].saved_bo) return -ENOMEM; - /* re-write 0 since err_event_athub will corrupt VCPU buffer */ - if (in_ras_intr) - memset(adev->uvd.inst[j].saved_bo, 0, size); - else - memcpy_fromio(adev->uvd.inst[j].saved_bo, ptr, size); + if (drm_dev_enter(&adev->ddev, &idx)) { + /* re-write 0 since err_event_athub will corrupt VCPU buffer */ + if (in_ras_intr) + memset(adev->uvd.inst[j].saved_bo, 0, size); + else + memcpy_fromio(adev->uvd.inst[j].saved_bo, ptr, size); + + drm_dev_exit(idx); + } } if (in_ras_intr) @@ -420,7 +425,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) { unsigned size; void *ptr; - int i; + int i, idx; for (i = 0; i < adev->uvd.num_uvd_inst; i++) { if (adev->uvd.harvest_config & (1 << i)) @@ -432,7 +437,10 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) ptr = adev->uvd.inst[i].cpu_addr; if (adev->uvd.inst[i].saved_bo != NULL) { - memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size); + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size); + drm_dev_exit(idx); + } kvfree(adev->uvd.inst[i].saved_bo); adev->uvd.inst[i].saved_bo = NULL; } else { @@ -442,8 +450,11 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) hdr = (const struct common_firmware_header *)adev->uvd.fw->data; if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { offset = le32_to_cpu(hdr->ucode_array_offset_bytes); - memcpy_toio(adev->uvd.inst[i].cpu_addr, adev->uvd.fw->data + offset, - le32_to_cpu(hdr->ucode_size_bytes)); + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_toio(adev->uvd.inst[i].cpu_addr, adev->uvd.fw->data + offset, + le32_to_cpu(hdr->ucode_size_bytes)); + drm_dev_exit(idx); + } size -= le32_to_cpu(hdr->ucode_size_bytes); ptr += le32_to_cpu(hdr->ucode_size_bytes); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 7ad83da613ed..1ae7f824adc7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -29,6 +29,7 @@ #include <linux/module.h> #include <drm/drm.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_pm.h" @@ -293,7 +294,7 @@ int amdgpu_vce_resume(struct amdgpu_device *adev) void *cpu_addr; const struct common_firmware_header *hdr; unsigned offset; - int r; + int r, idx; if (adev->vce.vcpu_bo == NULL) return -EINVAL; @@ -313,8 +314,12 @@ int amdgpu_vce_resume(struct amdgpu_device *adev) hdr = (const struct common_firmware_header *)adev->vce.fw->data; offset = le32_to_cpu(hdr->ucode_array_offset_bytes); - memcpy_toio(cpu_addr, adev->vce.fw->data + offset, - adev->vce.fw->size - offset); + + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_toio(cpu_addr, adev->vce.fw->data + offset, + adev->vce.fw->size - offset); + drm_dev_exit(idx); + } amdgpu_bo_kunmap(adev->vce.vcpu_bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index d101cb9697a2..b812938c9322 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -27,6 +27,7 @@ #include <linux/firmware.h> #include <linux/module.h> #include <linux/pci.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_pm.h" @@ -283,7 +284,7 @@ int amdgpu_vcn_suspend(struct amdgpu_device *adev) { unsigned size; void *ptr; - int i; + int i, idx; cancel_delayed_work_sync(&adev->vcn.idle_work); @@ -300,7 +301,10 @@ int amdgpu_vcn_suspend(struct amdgpu_device *adev) if (!adev->vcn.inst[i].saved_bo) return -ENOMEM; - memcpy_fromio(adev->vcn.inst[i].saved_bo, ptr, size); + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_fromio(adev->vcn.inst[i].saved_bo, ptr, size); + drm_dev_exit(idx); + } } return 0; } @@ -309,7 +313,7 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev) { unsigned size; void *ptr; - int i; + int i, idx; for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { if (adev->vcn.harvest_config & (1 << i)) @@ -321,7 +325,10 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev) ptr = adev->vcn.inst[i].cpu_addr; if (adev->vcn.inst[i].saved_bo != NULL) { - memcpy_toio(ptr, adev->vcn.inst[i].saved_bo, size); + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_toio(ptr, adev->vcn.inst[i].saved_bo, size); + drm_dev_exit(idx); + } kvfree(adev->vcn.inst[i].saved_bo); adev->vcn.inst[i].saved_bo = NULL; } else { @@ -331,8 +338,11 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev) hdr = (const struct common_firmware_header *)adev->vcn.fw->data; if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { offset = le32_to_cpu(hdr->ucode_array_offset_bytes); - memcpy_toio(adev->vcn.inst[i].cpu_addr, adev->vcn.fw->data + offset, - le32_to_cpu(hdr->ucode_size_bytes)); + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_toio(adev->vcn.inst[i].cpu_addr, adev->vcn.fw->data + offset, + le32_to_cpu(hdr->ucode_size_bytes)); + drm_dev_exit(idx); + } size -= le32_to_cpu(hdr->ucode_size_bytes); ptr += le32_to_cpu(hdr->ucode_size_bytes); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 1923f035713a..8d218c5cfee8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -32,6 +32,7 @@ #include <linux/dma-buf.h> #include <drm/amdgpu_drm.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_trace.h" #include "amdgpu_amdkfd.h" @@ -1673,7 +1674,10 @@ int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, struct amdgpu_vm_update_params params; struct amdgpu_res_cursor cursor; enum amdgpu_sync_mode sync_mode; - int r; + int r, idx; + + if (!drm_dev_enter(&adev->ddev, &idx)) + return -ENODEV; memset(¶ms, 0, sizeof(params)); params.adev = adev; @@ -1768,6 +1772,7 @@ int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, error_unlock: amdgpu_vm_eviction_unlock(vm); + drm_dev_exit(idx); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 0f897a343073..07e007dbff7c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -172,7 +172,7 @@ static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, amdgpu_mem_info_vram_vendor, NULL); -static const struct attribute *amdgpu_vram_mgr_attributes[] = { +static struct attribute *amdgpu_vram_mgr_attributes[] = { &dev_attr_mem_info_vram_total.attr, &dev_attr_mem_info_vis_vram_total.attr, &dev_attr_mem_info_vram_used.attr, @@ -181,6 +181,10 @@ static const struct attribute *amdgpu_vram_mgr_attributes[] = { NULL }; +const struct attribute_group amdgpu_vram_mgr_attr_group = { + .attrs = amdgpu_vram_mgr_attributes +}; + /** * amdgpu_vram_mgr_vis_size - Calculate visible node size * @@ -684,7 +688,6 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) { struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; struct ttm_resource_manager *man = &mgr->manager; - int ret; ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT); @@ -695,11 +698,6 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) INIT_LIST_HEAD(&mgr->reservations_pending); INIT_LIST_HEAD(&mgr->reserved_pages); - /* Add the two VRAM-related sysfs files */ - ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes); - if (ret) - DRM_ERROR("Failed to register sysfs\n"); - ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); ttm_resource_manager_set_used(man, true); return 0; @@ -737,8 +735,6 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) drm_mm_takedown(&mgr->mm); spin_unlock(&mgr->lock); - sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes); - ttm_resource_manager_cleanup(man); ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c index d3745711d55f..df385ffc9768 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c @@ -309,8 +309,7 @@ static int cik_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); amdgpu_irq_remove_domain(adev); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c index 307c01301c87..b8c47e0cf37a 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c @@ -301,8 +301,7 @@ static int cz_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); amdgpu_irq_remove_domain(adev); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 7fc546ff94c3..102f31526204 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -7391,7 +7391,7 @@ static int gfx_v10_0_hw_fini(void *handle) amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); - if (!adev->in_pci_err_recovery) { + if (!adev->no_hw_access) { #ifndef BRING_UP_DEBUG if (amdgpu_async_gfx_ring) { r = gfx_v10_0_kiq_disable_kgq(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index ceab5ef50790..db154bf083e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -965,7 +965,6 @@ static int gmc_v10_0_sw_init(void *handle) static void gmc_v10_0_gart_fini(struct amdgpu_device *adev) { amdgpu_gart_table_vram_free(adev); - amdgpu_gart_fini(adev); } static int gmc_v10_0_sw_fini(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index 405d6ad09022..0e81e03e9b49 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -898,7 +898,6 @@ static int gmc_v6_0_sw_fini(void *handle) amdgpu_vm_manager_fini(adev); amdgpu_gart_table_vram_free(adev); amdgpu_bo_fini(adev); - amdgpu_gart_fini(adev); release_firmware(adev->gmc.fw); adev->gmc.fw = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 8e282169f99e..0a50fdaced7e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -1085,7 +1085,6 @@ static int gmc_v7_0_sw_fini(void *handle) kfree(adev->gmc.vm_fault_info); amdgpu_gart_table_vram_free(adev); amdgpu_bo_fini(adev); - amdgpu_gart_fini(adev); release_firmware(adev->gmc.fw); adev->gmc.fw = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index e4f27b3f28fb..492ebed2915b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -1201,7 +1201,6 @@ static int gmc_v8_0_sw_fini(void *handle) kfree(adev->gmc.vm_fault_info); amdgpu_gart_table_vram_free(adev); amdgpu_bo_fini(adev); - amdgpu_gart_fini(adev); release_firmware(adev->gmc.fw); adev->gmc.fw = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index fb3c7588dde3..68e6b6b8989a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -1615,7 +1615,6 @@ static int gmc_v9_0_sw_fini(void *handle) amdgpu_gart_table_vram_free(adev); amdgpu_bo_unref(&adev->gmc.pdb0_bo); amdgpu_bo_fini(adev); - amdgpu_gart_fini(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c index cc957471f31e..ddfe4eaeea05 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c @@ -300,8 +300,7 @@ static int iceland_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); amdgpu_irq_remove_domain(adev); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c index 17ed61a34fbf..baa2fd119e6e 100644 --- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c @@ -586,11 +586,7 @@ static int navi10_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); - amdgpu_ih_ring_fini(adev, &adev->irq.ih2); - amdgpu_ih_ring_fini(adev, &adev->irq.ih1); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index 0fd1ed918627..fc400d95b83b 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -23,6 +23,7 @@ #include <linux/firmware.h> #include <linux/module.h> #include <linux/vmalloc.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_psp.h" @@ -282,10 +283,8 @@ static int psp_v11_0_bootloader_load_kdb(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, psp->kdb_start_addr, psp->kdb_bin_size); + psp_copy_fw(psp, psp->kdb_start_addr, psp->kdb_bin_size); /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -315,10 +314,8 @@ static int psp_v11_0_bootloader_load_spl(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy PSP SPL binary to memory */ - memcpy(psp->fw_pri_buf, psp->spl_start_addr, psp->spl_bin_size); + psp_copy_fw(psp, psp->spl_start_addr, psp->spl_bin_size); /* Provide the PSP SPL to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -348,10 +345,8 @@ static int psp_v11_0_bootloader_load_sysdrv(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy PSP System Driver binary to memory */ - memcpy(psp->fw_pri_buf, psp->sys_start_addr, psp->sys_bin_size); + psp_copy_fw(psp, psp->sys_start_addr, psp->sys_bin_size); /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -384,10 +379,8 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos_start_addr, psp->sos_bin_size); + psp_copy_fw(psp, psp->sos_start_addr, psp->sos_bin_size); /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -621,7 +614,7 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) uint32_t p2c_header[4]; uint32_t sz; void *buf; - int ret; + int ret, idx; if (ctx->init == PSP_MEM_TRAIN_NOT_SUPPORT) { DRM_DEBUG("Memory training is not supported.\n"); @@ -694,17 +687,24 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) return -ENOMEM; } - memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz); - ret = psp_v11_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN); - if (ret) { - DRM_ERROR("Send long training msg failed.\n"); + if (drm_dev_enter(&adev->ddev, &idx)) { + memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz); + ret = psp_v11_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN); + if (ret) { + DRM_ERROR("Send long training msg failed.\n"); + vfree(buf); + drm_dev_exit(idx); + return ret; + } + + memcpy_toio(adev->mman.aper_base_kaddr, buf, sz); + adev->hdp.funcs->flush_hdp(adev, NULL); vfree(buf); - return ret; + drm_dev_exit(idx); + } else { + vfree(buf); + return -ENODEV; } - - memcpy_toio(adev->mman.aper_base_kaddr, buf, sz); - adev->hdp.funcs->flush_hdp(adev, NULL); - vfree(buf); } if (ops & PSP_MEM_TRAIN_SAVE) { diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c index c4828bd3264b..618e5b6b85d9 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c @@ -138,10 +138,8 @@ static int psp_v12_0_bootloader_load_sysdrv(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy PSP System Driver binary to memory */ - memcpy(psp->fw_pri_buf, psp->sys_start_addr, psp->sys_bin_size); + psp_copy_fw(psp, psp->sys_start_addr, psp->sys_bin_size); /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -179,10 +177,8 @@ static int psp_v12_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos_start_addr, psp->sos_bin_size); + psp_copy_fw(psp, psp->sos_start_addr, psp->sos_bin_size); /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index 908664a5774b..ce7377d2368f 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -102,10 +102,8 @@ static int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy PSP System Driver binary to memory */ - memcpy(psp->fw_pri_buf, psp->sys_start_addr, psp->sys_bin_size); + psp_copy_fw(psp, psp->sys_start_addr, psp->sys_bin_size); /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -143,10 +141,8 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos_start_addr, psp->sos_bin_size); + psp_copy_fw(psp, psp->sos_start_addr, psp->sos_bin_size); /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c index 51880f6ef634..9a24f17a5750 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c @@ -175,8 +175,7 @@ static int si_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c index 249fcbee7871..b08905d1c00f 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c @@ -312,8 +312,7 @@ static int tonga_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); amdgpu_irq_remove_domain(adev); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c index 8e238dea7bef..90910d19db12 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c @@ -25,6 +25,7 @@ */ #include <linux/firmware.h> +#include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_vce.h" @@ -555,16 +556,19 @@ static int vce_v4_0_hw_fini(void *handle) static int vce_v4_0_suspend(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; + int r, idx; if (adev->vce.vcpu_bo == NULL) return 0; - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo); - void *ptr = adev->vce.cpu_addr; + if (drm_dev_enter(&adev->ddev, &idx)) { + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo); + void *ptr = adev->vce.cpu_addr; - memcpy_fromio(adev->vce.saved_bo, ptr, size); + memcpy_fromio(adev->vce.saved_bo, ptr, size); + } + drm_dev_exit(idx); } r = vce_v4_0_hw_fini(adev); @@ -577,16 +581,20 @@ static int vce_v4_0_suspend(void *handle) static int vce_v4_0_resume(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; + int r, idx; if (adev->vce.vcpu_bo == NULL) return -EINVAL; if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo); - void *ptr = adev->vce.cpu_addr; - memcpy_toio(ptr, adev->vce.saved_bo, size); + if (drm_dev_enter(&adev->ddev, &idx)) { + unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo); + void *ptr = adev->vce.cpu_addr; + + memcpy_toio(ptr, adev->vce.saved_bo, size); + drm_dev_exit(idx); + } } else { r = amdgpu_vce_resume(adev); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index ce3c794c176f..4c36fc5c9738 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -34,6 +34,8 @@ #include "vcn/vcn_3_0_0_sh_mask.h" #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h" +#include <drm/drm_drv.h> + #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10 @@ -276,16 +278,20 @@ static int vcn_v3_0_sw_init(void *handle) static int vcn_v3_0_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, r; + int i, r, idx; - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_fw_shared *fw_shared; + if (drm_dev_enter(&adev->ddev, &idx)) { + for (i = 0; i < adev->vcn.num_vcn_inst; i++) { + volatile struct amdgpu_fw_shared *fw_shared; - if (adev->vcn.harvest_config & (1 << i)) - continue; - fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr; - fw_shared->present_flag_0 = 0; - fw_shared->sw_ring.is_enabled = false; + if (adev->vcn.harvest_config & (1 << i)) + continue; + fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr; + fw_shared->present_flag_0 = 0; + fw_shared->sw_ring.is_enabled = false; + } + + drm_dev_exit(idx); } if (amdgpu_sriov_vf(adev)) diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index 2f17c8a57015..a9ca6988009e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -514,11 +514,7 @@ static int vega10_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); - amdgpu_ih_ring_fini(adev, &adev->irq.ih2); - amdgpu_ih_ring_fini(adev, &adev->irq.ih1); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c index 8a122b413bf5..f51dfc38ac65 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c @@ -565,11 +565,7 @@ static int vega20_ih_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_fini(adev); - amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); - amdgpu_ih_ring_fini(adev, &adev->irq.ih2); - amdgpu_ih_ring_fini(adev, &adev->irq.ih1); - amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_fini_sw(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 80015e866498..dd1b1a464c28 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -891,7 +891,6 @@ out: void kgd2kfd_device_exit(struct kfd_dev *kfd) { if (kfd->init_complete) { - kgd2kfd_suspend(kfd, false); svm_migrate_fini((struct amdgpu_device *)kfd->kgd); device_queue_manager_uninit(kfd->dqm); kfd_interrupt_exit(kfd); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index d5908c413705..4f4c6438d245 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1290,6 +1290,15 @@ error: return -EINVAL; } +static int amdgpu_dm_early_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + amdgpu_dm_audio_fini(adev); + + return 0; +} + static void amdgpu_dm_fini(struct amdgpu_device *adev) { int i; @@ -1298,8 +1307,6 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) drm_encoder_cleanup(&adev->dm.mst_encoders[i].base); } - amdgpu_dm_audio_fini(adev); - amdgpu_dm_destroy_drm_device(&adev->dm); #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) @@ -2340,6 +2347,7 @@ static const struct amd_ip_funcs amdgpu_dm_funcs = { .late_init = dm_late_init, .sw_init = dm_sw_init, .sw_fini = dm_sw_fini, + .early_fini = amdgpu_dm_early_fini, .hw_init = dm_hw_init, .hw_fini = dm_hw_fini, .suspend = dm_suspend, @@ -3881,7 +3889,6 @@ fail: static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm) { - drm_mode_config_cleanup(dm->ddev); drm_atomic_private_obj_fini(&dm->atomic_obj); return; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 6f0a1fdccf7f..5568d4e518e6 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -467,8 +467,8 @@ void amdgpu_dm_initialize_dp_connector(struct amdgpu_display_manager *dm, &aconnector->dm_dp_aux.aux, 16, 4, - (u8)max_link_enc_cap.lane_count, - (u8)max_link_enc_cap.link_rate, + max_link_enc_cap.lane_count, + drm_dp_bw_code_to_link_rate(max_link_enc_cap.link_rate), aconnector->connector_id); drm_connector_attach_dp_subconnector_property(&aconnector->base); diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index 9ab706cd07ff..332b0df53e52 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -246,6 +246,7 @@ enum amd_dpm_forced_level; * @late_init: sets up late driver/hw state (post hw_init) - Optional * @sw_init: sets up driver state, does not configure hw * @sw_fini: tears down driver state, does not configure hw + * @early_fini: tears down stuff before dev detached from driver * @hw_init: sets up the hw state * @hw_fini: tears down the hw state * @late_fini: final cleanup @@ -274,6 +275,7 @@ struct amd_ip_funcs { int (*late_init)(void *handle); int (*sw_init)(void *handle); int (*sw_fini)(void *handle); + int (*early_fini)(void *handle); int (*hw_init)(void *handle); int (*hw_fini)(void *handle); void (*late_fini)(void *handle); diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h index 47eb84598b96..6a505d1b82a5 100644 --- a/drivers/gpu/drm/amd/include/atombios.h +++ b/drivers/gpu/drm/amd/include/atombios.h @@ -5178,11 +5178,11 @@ typedef struct _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 typedef struct _ATOM_SVID2_VOLTAGE_OBJECT_V3 { ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; // voltage mode = VOLTAGE_OBJ_SVID2 -// 14:7 � PSI0_VID -// 6 � PSI0_EN -// 5 � PSI1 -// 4:2 � load line slope trim. -// 1:0 � offset trim, +// 14:7 - PSI0_VID +// 6 - PSI0_EN +// 5 - PSI1 +// 4:2 - load line slope trim. +// 1:0 - offset trim, USHORT usLoadLine_PSI; // GPU GPIO pin Id to SVID2 regulator VRHot pin. possible value 0~31. 0 means GPIO0, 31 means GPIO31 UCHAR ucSVDGpioId; //0~31 indicate GPIO0~31 diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 0ceb7329838c..0882a1dd6797 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -126,7 +126,7 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, struct amdgpu_device *adev = smu->adev; int ret = 0, index = 0; - if (smu->adev->in_pci_err_recovery) + if (smu->adev->no_hw_access) return 0; index = smu_cmn_to_asic_specific_index(smu, diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig index d25e900f07ef..c96e4b38d1d3 100644 --- a/drivers/gpu/drm/bridge/Kconfig +++ b/drivers/gpu/drm/bridge/Kconfig @@ -50,6 +50,19 @@ config DRM_CHRONTEL_CH7033 If in doubt, say "N". +config DRM_CROS_EC_ANX7688 + tristate "ChromeOS EC ANX7688 bridge" + depends on OF + depends on I2C_CROS_EC_TUNNEL || COMPILE_TEST + select DRM_KMS_HELPER + select REGMAP_I2C + help + ChromeOS EC ANX7688 is an ultra-low power + 4K Ultra-HD (4096x2160p60) mobile HD transmitter + designed for ChromeOS devices. It converts HDMI + 2.0 to DisplayPort 1.3 Ultra-HD. It is connected + to the ChromeOS Embedded Controller. + config DRM_DISPLAY_CONNECTOR tristate "Display connector support" depends on OF diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile index 965b54dccfe5..dff2b2897529 100644 --- a/drivers/gpu/drm/bridge/Makefile +++ b/drivers/gpu/drm/bridge/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_DRM_CDNS_DSI) += cdns-dsi.o obj-$(CONFIG_DRM_CHIPONE_ICN6211) += chipone-icn6211.o obj-$(CONFIG_DRM_CHRONTEL_CH7033) += chrontel-ch7033.o +obj-$(CONFIG_DRM_CROS_EC_ANX7688) += cros-ec-anx7688.o obj-$(CONFIG_DRM_DISPLAY_CONNECTOR) += display-connector.o obj-$(CONFIG_DRM_LONTIUM_LT8912B) += lontium-lt8912b.o obj-$(CONFIG_DRM_LONTIUM_LT9611) += lontium-lt9611.o diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c index b4e349ca38fe..7519b7a0f29d 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.c +++ b/drivers/gpu/drm/bridge/analogix/anx7625.c @@ -11,6 +11,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/pm_runtime.h> #include <linux/regulator/consumer.h> #include <linux/slab.h> #include <linux/types.h> @@ -1005,33 +1006,6 @@ static void anx7625_power_on_init(struct anx7625_data *ctx) } } -static void anx7625_chip_control(struct anx7625_data *ctx, int state) -{ - struct device *dev = &ctx->client->dev; - - DRM_DEV_DEBUG_DRIVER(dev, "before set, power_state(%d).\n", - atomic_read(&ctx->power_status)); - - if (!ctx->pdata.low_power_mode) - return; - - if (state) { - atomic_inc(&ctx->power_status); - if (atomic_read(&ctx->power_status) == 1) - anx7625_power_on_init(ctx); - } else { - if (atomic_read(&ctx->power_status)) { - atomic_dec(&ctx->power_status); - - if (atomic_read(&ctx->power_status) == 0) - anx7625_power_standby(ctx); - } - } - - DRM_DEV_DEBUG_DRIVER(dev, "after set, power_state(%d).\n", - atomic_read(&ctx->power_status)); -} - static void anx7625_init_gpio(struct anx7625_data *platform) { struct device *dev = &platform->client->dev; @@ -1061,9 +1035,6 @@ static void anx7625_stop_dp_work(struct anx7625_data *ctx) ctx->hpd_status = 0; ctx->hpd_high_cnt = 0; ctx->display_timing_valid = 0; - - if (ctx->pdata.low_power_mode == 0) - anx7625_disable_pd_protocol(ctx); } static void anx7625_start_dp_work(struct anx7625_data *ctx) @@ -1105,49 +1076,26 @@ static void anx7625_hpd_polling(struct anx7625_data *ctx) int ret, val; struct device *dev = &ctx->client->dev; - if (atomic_read(&ctx->power_status) != 1) { - DRM_DEV_DEBUG_DRIVER(dev, "No need to poling HPD status.\n"); - return; - } - ret = readx_poll_timeout(anx7625_read_hpd_status_p0, ctx, val, ((val & HPD_STATUS) || (val < 0)), 5000, 5000 * 100); if (ret) { - DRM_DEV_ERROR(dev, "HPD polling timeout!\n"); - } else { - DRM_DEV_DEBUG_DRIVER(dev, "HPD raise up.\n"); - anx7625_reg_write(ctx, ctx->i2c.tcpc_client, - INTR_ALERT_1, 0xFF); - anx7625_reg_write(ctx, ctx->i2c.rx_p0_client, - INTERFACE_CHANGE_INT, 0); + DRM_DEV_ERROR(dev, "no hpd.\n"); + return; } - anx7625_start_dp_work(ctx); -} - -static void anx7625_disconnect_check(struct anx7625_data *ctx) -{ - if (atomic_read(&ctx->power_status) == 0) - anx7625_stop_dp_work(ctx); -} - -static void anx7625_low_power_mode_check(struct anx7625_data *ctx, - int state) -{ - struct device *dev = &ctx->client->dev; + DRM_DEV_DEBUG_DRIVER(dev, "system status: 0x%x. HPD raise up.\n", val); + anx7625_reg_write(ctx, ctx->i2c.tcpc_client, + INTR_ALERT_1, 0xFF); + anx7625_reg_write(ctx, ctx->i2c.rx_p0_client, + INTERFACE_CHANGE_INT, 0); - DRM_DEV_DEBUG_DRIVER(dev, "low power mode check, state(%d).\n", state); + anx7625_start_dp_work(ctx); - if (ctx->pdata.low_power_mode) { - anx7625_chip_control(ctx, state); - if (state) - anx7625_hpd_polling(ctx); - else - anx7625_disconnect_check(ctx); - } + if (!ctx->pdata.panel_bridge && ctx->bridge_attached) + drm_helper_hpd_irq_event(ctx->bridge.dev); } static void anx7625_remove_edid(struct anx7625_data *ctx) @@ -1180,9 +1128,6 @@ static int anx7625_hpd_change_detect(struct anx7625_data *ctx) int intr_vector, status; struct device *dev = &ctx->client->dev; - DRM_DEV_DEBUG_DRIVER(dev, "power_status=%d\n", - (u32)atomic_read(&ctx->power_status)); - status = anx7625_reg_write(ctx, ctx->i2c.tcpc_client, INTR_ALERT_1, 0xFF); if (status < 0) { @@ -1228,22 +1173,25 @@ static void anx7625_work_func(struct work_struct *work) struct anx7625_data, work); mutex_lock(&ctx->lock); + + if (pm_runtime_suspended(&ctx->client->dev)) + goto unlock; + event = anx7625_hpd_change_detect(ctx); - mutex_unlock(&ctx->lock); if (event < 0) - return; + goto unlock; if (ctx->bridge_attached) drm_helper_hpd_irq_event(ctx->bridge.dev); + +unlock: + mutex_unlock(&ctx->lock); } static irqreturn_t anx7625_intr_hpd_isr(int irq, void *data) { struct anx7625_data *ctx = (struct anx7625_data *)data; - if (atomic_read(&ctx->power_status) != 1) - return IRQ_NONE; - queue_work(ctx->workqueue, &ctx->work); return IRQ_HANDLED; @@ -1305,9 +1253,9 @@ static struct edid *anx7625_get_edid(struct anx7625_data *ctx) return (struct edid *)edid; } - anx7625_low_power_mode_check(ctx, 1); + pm_runtime_get_sync(dev); edid_num = sp_tx_edid_read(ctx, p_edid->edid_raw_data); - anx7625_low_power_mode_check(ctx, 0); + pm_runtime_put_sync(dev); if (edid_num < 1) { DRM_DEV_ERROR(dev, "Fail to read EDID: %d\n", edid_num); @@ -1611,10 +1559,7 @@ static void anx7625_bridge_enable(struct drm_bridge *bridge) DRM_DEV_DEBUG_DRIVER(dev, "drm enable\n"); - anx7625_low_power_mode_check(ctx, 1); - - if (WARN_ON(!atomic_read(&ctx->power_status))) - return; + pm_runtime_get_sync(dev); anx7625_dp_start(ctx); } @@ -1624,14 +1569,11 @@ static void anx7625_bridge_disable(struct drm_bridge *bridge) struct anx7625_data *ctx = bridge_to_anx7625(bridge); struct device *dev = &ctx->client->dev; - if (WARN_ON(!atomic_read(&ctx->power_status))) - return; - DRM_DEV_DEBUG_DRIVER(dev, "drm disable\n"); anx7625_dp_stop(ctx); - anx7625_low_power_mode_check(ctx, 0); + pm_runtime_put_sync(dev); } static enum drm_connector_status @@ -1735,6 +1677,71 @@ static void anx7625_unregister_i2c_dummy_clients(struct anx7625_data *ctx) i2c_unregister_device(ctx->i2c.tcpc_client); } +static int __maybe_unused anx7625_runtime_pm_suspend(struct device *dev) +{ + struct anx7625_data *ctx = dev_get_drvdata(dev); + + mutex_lock(&ctx->lock); + + anx7625_stop_dp_work(ctx); + anx7625_power_standby(ctx); + + mutex_unlock(&ctx->lock); + + return 0; +} + +static int __maybe_unused anx7625_runtime_pm_resume(struct device *dev) +{ + struct anx7625_data *ctx = dev_get_drvdata(dev); + + mutex_lock(&ctx->lock); + + anx7625_power_on_init(ctx); + anx7625_hpd_polling(ctx); + + mutex_unlock(&ctx->lock); + + return 0; +} + +static int __maybe_unused anx7625_resume(struct device *dev) +{ + struct anx7625_data *ctx = dev_get_drvdata(dev); + + if (!ctx->pdata.intp_irq) + return 0; + + if (!pm_runtime_enabled(dev) || !pm_runtime_suspended(dev)) { + enable_irq(ctx->pdata.intp_irq); + anx7625_runtime_pm_resume(dev); + } + + return 0; +} + +static int __maybe_unused anx7625_suspend(struct device *dev) +{ + struct anx7625_data *ctx = dev_get_drvdata(dev); + + if (!ctx->pdata.intp_irq) + return 0; + + if (!pm_runtime_enabled(dev) || !pm_runtime_suspended(dev)) { + anx7625_runtime_pm_suspend(dev); + disable_irq(ctx->pdata.intp_irq); + flush_workqueue(ctx->workqueue); + } + + return 0; +} + +static const struct dev_pm_ops anx7625_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(anx7625_suspend, anx7625_resume) + SET_RUNTIME_PM_OPS(anx7625_runtime_pm_suspend, + anx7625_runtime_pm_resume, NULL) +}; + static int anx7625_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -1778,8 +1785,6 @@ static int anx7625_i2c_probe(struct i2c_client *client, } anx7625_init_gpio(platform); - atomic_set(&platform->power_status, 0); - mutex_init(&platform->lock); platform->pdata.intp_irq = client->irq; @@ -1809,9 +1814,11 @@ static int anx7625_i2c_probe(struct i2c_client *client, goto free_wq; } - if (platform->pdata.low_power_mode == 0) { + pm_runtime_enable(dev); + + if (!platform->pdata.low_power_mode) { anx7625_disable_pd_protocol(platform); - atomic_set(&platform->power_status, 1); + pm_runtime_get_sync(dev); } /* Add work function */ @@ -1847,6 +1854,9 @@ static int anx7625_i2c_remove(struct i2c_client *client) if (platform->pdata.intp_irq) destroy_workqueue(platform->workqueue); + if (!platform->pdata.low_power_mode) + pm_runtime_put_sync_suspend(&client->dev); + anx7625_unregister_i2c_dummy_clients(platform); kfree(platform); @@ -1869,6 +1879,7 @@ static struct i2c_driver anx7625_driver = { .driver = { .name = "anx7625", .of_match_table = anx_match_table, + .pm = &anx7625_pm_ops, }, .probe = anx7625_i2c_probe, .remove = anx7625_i2c_remove, diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.h b/drivers/gpu/drm/bridge/analogix/anx7625.h index e4a086b3a3d7..034c3840028f 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.h +++ b/drivers/gpu/drm/bridge/analogix/anx7625.h @@ -369,7 +369,6 @@ struct anx7625_i2c_client { struct anx7625_data { struct anx7625_platform_data pdata; - atomic_t power_status; int hpd_status; int hpd_high_cnt; /* Lock for work queue */ diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c index 0cd8f40fb690..5530fbf64f1e 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c @@ -2140,7 +2140,7 @@ cdns_mhdp_bridge_atomic_reset(struct drm_bridge *bridge) if (!cdns_mhdp_state) return NULL; - __drm_atomic_helper_bridge_reset(bridge, &cdns_mhdp_state->base); + __drm_atomic_helper_bridge_reset(bridge, &cdns_mhdp_state->base); return &cdns_mhdp_state->base; } @@ -2478,9 +2478,9 @@ static int cdns_mhdp_probe(struct platform_device *pdev) clk_prepare_enable(clk); pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { - dev_err(dev, "pm_runtime_get_sync failed\n"); + dev_err(dev, "pm_runtime_resume_and_get failed\n"); pm_runtime_disable(dev); goto clk_disable; } diff --git a/drivers/gpu/drm/bridge/cdns-dsi.c b/drivers/gpu/drm/bridge/cdns-dsi.c index 76373e31df92..b31281f76117 100644 --- a/drivers/gpu/drm/bridge/cdns-dsi.c +++ b/drivers/gpu/drm/bridge/cdns-dsi.c @@ -1028,7 +1028,7 @@ static ssize_t cdns_dsi_transfer(struct mipi_dsi_host *host, struct mipi_dsi_packet packet; int ret, i, tx_len, rx_len; - ret = pm_runtime_get_sync(host->dev); + ret = pm_runtime_resume_and_get(host->dev); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/bridge/cros-ec-anx7688.c b/drivers/gpu/drm/bridge/cros-ec-anx7688.c new file mode 100644 index 000000000000..0f6d907432e3 --- /dev/null +++ b/drivers/gpu/drm/bridge/cros-ec-anx7688.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CrOS EC ANX7688 HDMI->DP bridge driver + * + * Copyright 2020 Google LLC + */ + +#include <drm/drm_bridge.h> +#include <drm/drm_print.h> +#include <linux/i2c.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/types.h> + +/* Register addresses */ +#define ANX7688_VENDOR_ID_REG 0x00 +#define ANX7688_DEVICE_ID_REG 0x02 + +#define ANX7688_FW_VERSION_REG 0x80 + +#define ANX7688_DP_BANDWIDTH_REG 0x85 +#define ANX7688_DP_LANE_COUNT_REG 0x86 + +#define ANX7688_VENDOR_ID 0x1f29 +#define ANX7688_DEVICE_ID 0x7688 + +/* First supported firmware version (0.85) */ +#define ANX7688_MINIMUM_FW_VERSION 0x0085 + +static const struct regmap_config cros_ec_anx7688_regmap_config = { + .reg_bits = 8, + .val_bits = 8, +}; + +struct cros_ec_anx7688 { + struct i2c_client *client; + struct regmap *regmap; + struct drm_bridge bridge; + bool filter; +}; + +static inline struct cros_ec_anx7688 * +bridge_to_cros_ec_anx7688(struct drm_bridge *bridge) +{ + return container_of(bridge, struct cros_ec_anx7688, bridge); +} + +static bool cros_ec_anx7688_bridge_mode_fixup(struct drm_bridge *bridge, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct cros_ec_anx7688 *anx = bridge_to_cros_ec_anx7688(bridge); + int totalbw, requiredbw; + u8 dpbw, lanecount; + u8 regs[2]; + int ret; + + if (!anx->filter) + return true; + + /* Read both regs 0x85 (bandwidth) and 0x86 (lane count). */ + ret = regmap_bulk_read(anx->regmap, ANX7688_DP_BANDWIDTH_REG, regs, 2); + if (ret < 0) { + DRM_ERROR("Failed to read bandwidth/lane count\n"); + return false; + } + dpbw = regs[0]; + lanecount = regs[1]; + + /* Maximum 0x19 bandwidth (6.75 Gbps Turbo mode), 2 lanes */ + if (dpbw > 0x19 || lanecount > 2) { + DRM_ERROR("Invalid bandwidth/lane count (%02x/%d)\n", dpbw, + lanecount); + return false; + } + + /* Compute available bandwidth (kHz) */ + totalbw = dpbw * lanecount * 270000 * 8 / 10; + + /* Required bandwidth (8 bpc, kHz) */ + requiredbw = mode->clock * 8 * 3; + + DRM_DEBUG_KMS("DP bandwidth: %d kHz (%02x/%d); mode requires %d Khz\n", + totalbw, dpbw, lanecount, requiredbw); + + if (totalbw == 0) { + DRM_ERROR("Bandwidth/lane count are 0, not rejecting modes\n"); + return true; + } + + return totalbw >= requiredbw; +} + +static const struct drm_bridge_funcs cros_ec_anx7688_bridge_funcs = { + .mode_fixup = cros_ec_anx7688_bridge_mode_fixup, +}; + +static int cros_ec_anx7688_bridge_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct cros_ec_anx7688 *anx7688; + u16 vendor, device, fw_version; + u8 buffer[4]; + int ret; + + anx7688 = devm_kzalloc(dev, sizeof(*anx7688), GFP_KERNEL); + if (!anx7688) + return -ENOMEM; + + anx7688->client = client; + i2c_set_clientdata(client, anx7688); + + anx7688->regmap = devm_regmap_init_i2c(client, &cros_ec_anx7688_regmap_config); + if (IS_ERR(anx7688->regmap)) { + ret = PTR_ERR(anx7688->regmap); + dev_err(dev, "regmap i2c init failed: %d\n", ret); + return ret; + } + + /* Read both vendor and device id (4 bytes). */ + ret = regmap_bulk_read(anx7688->regmap, ANX7688_VENDOR_ID_REG, + buffer, 4); + if (ret) { + dev_err(dev, "Failed to read chip vendor/device id\n"); + return ret; + } + + vendor = (u16)buffer[1] << 8 | buffer[0]; + device = (u16)buffer[3] << 8 | buffer[2]; + if (vendor != ANX7688_VENDOR_ID || device != ANX7688_DEVICE_ID) { + dev_err(dev, "Invalid vendor/device id %04x/%04x\n", + vendor, device); + return -ENODEV; + } + + ret = regmap_bulk_read(anx7688->regmap, ANX7688_FW_VERSION_REG, + buffer, 2); + if (ret) { + dev_err(dev, "Failed to read firmware version\n"); + return ret; + } + + fw_version = (u16)buffer[0] << 8 | buffer[1]; + dev_info(dev, "ANX7688 firmware version 0x%04x\n", fw_version); + + anx7688->bridge.of_node = dev->of_node; + + /* FW version >= 0.85 supports bandwidth/lane count registers */ + if (fw_version >= ANX7688_MINIMUM_FW_VERSION) + anx7688->filter = true; + else + /* Warn, but not fail, for backwards compatibility */ + DRM_WARN("Old ANX7688 FW version (0x%04x), not filtering\n", + fw_version); + + anx7688->bridge.funcs = &cros_ec_anx7688_bridge_funcs; + drm_bridge_add(&anx7688->bridge); + + return 0; +} + +static int cros_ec_anx7688_bridge_remove(struct i2c_client *client) +{ + struct cros_ec_anx7688 *anx7688 = i2c_get_clientdata(client); + + drm_bridge_remove(&anx7688->bridge); + + return 0; +} + +static const struct of_device_id cros_ec_anx7688_bridge_match_table[] = { + { .compatible = "google,cros-ec-anx7688" }, + { } +}; +MODULE_DEVICE_TABLE(of, cros_ec_anx7688_bridge_match_table); + +static struct i2c_driver cros_ec_anx7688_bridge_driver = { + .probe_new = cros_ec_anx7688_bridge_probe, + .remove = cros_ec_anx7688_bridge_remove, + .driver = { + .name = "cros-ec-anx7688-bridge", + .of_match_table = cros_ec_anx7688_bridge_match_table, + }, +}; + +module_i2c_driver(cros_ec_anx7688_bridge_driver); + +MODULE_DESCRIPTION("ChromeOS EC ANX7688 HDMI->DP bridge driver"); +MODULE_AUTHOR("Nicolas Boichat <drinkcat@chromium.org>"); +MODULE_AUTHOR("Enric Balletbo i Serra <enric.balletbo@collabora.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c index d8a60691fd32..7149ed40af83 100644 --- a/drivers/gpu/drm/bridge/ite-it66121.c +++ b/drivers/gpu/drm/bridge/ite-it66121.c @@ -898,14 +898,14 @@ static int it66121_probe(struct i2c_client *client, return -ENXIO; } - ep = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); - if (!ep) - return -EINVAL; - ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; + ep = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); + if (!ep) + return -EINVAL; + ctx->dev = dev; ctx->client = client; @@ -943,7 +943,7 @@ static int it66121_probe(struct i2c_client *client, ctx->regmap = devm_regmap_init_i2c(client, &it66121_regmap_config); if (IS_ERR(ctx->regmap)) { ite66121_power_off(ctx); - return PTR_ERR(ctx); + return PTR_ERR(ctx->regmap); } regmap_read(ctx->regmap, IT66121_VENDOR_ID0_REG, &vendor_ids[0]); diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c index 443f1b47e031..76c720b535fb 100644 --- a/drivers/gpu/drm/bridge/lontium-lt8912b.c +++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c @@ -7,8 +7,6 @@ #include <linux/delay.h> #include <linux/gpio/consumer.h> #include <linux/i2c.h> -#include <linux/gpio.h> -#include <linux/of_gpio.h> #include <linux/regmap.h> #include <drm/drm_probe_helper.h> @@ -755,7 +753,6 @@ static struct i2c_driver lt8912_i2c_driver = { .driver = { .name = "lt8912", .of_match_table = lt8912_dt_match, - .owner = THIS_MODULE, }, .probe = lt8912_probe, .remove = lt8912_remove, diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index bb0a0e1c6341..45a2969afb2b 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -1042,7 +1042,9 @@ exit: pm_runtime_mark_last_busy(pdata->dev); pm_runtime_put_autosuspend(pdata->dev); - return ret ? ret : len; + if (ret) + return ret; + return len; } static int ti_sn_bridge_parse_dsi_host(struct ti_sn65dsi86 *pdata) diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c index 4805726b34ac..ae8e4d76209c 100644 --- a/drivers/gpu/drm/drm_bufs.c +++ b/drivers/gpu/drm/drm_bufs.c @@ -685,6 +685,7 @@ static void drm_cleanup_buf_error(struct drm_device *dev, dmah->size, dmah->vaddr, dmah->busaddr); + kfree(dmah); } } kfree(entry->seglist); @@ -984,8 +985,16 @@ int drm_legacy_addbufs_pci(struct drm_device *dev, while (entry->buf_count < count) { dmah = kmalloc(sizeof(drm_dma_handle_t), GFP_KERNEL); - if (!dmah) + if (!dmah) { + /* Set count correctly so we free the proper amount. */ + entry->buf_count = count; + entry->seg_count = count; + drm_cleanup_buf_error(dev, entry); + kfree(temp_pagelist); + mutex_unlock(&dev->struct_mutex); + atomic_dec(&dev->buf_alloc); return -ENOMEM; + } dmah->size = total; dmah->vaddr = dma_alloc_coherent(dev->dev, diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index 54604633e65c..32b7f8983b94 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -3722,9 +3722,9 @@ int drm_dp_mst_topology_mgr_set_mst(struct drm_dp_mst_topology_mgr *mgr, bool ms } lane_count = min_t(int, mgr->dpcd[2] & DP_MAX_LANE_COUNT_MASK, mgr->max_lane_count); - link_rate = min_t(int, mgr->dpcd[1], mgr->max_link_rate); + link_rate = min_t(int, drm_dp_bw_code_to_link_rate(mgr->dpcd[1]), mgr->max_link_rate); mgr->pbn_div = drm_dp_get_vc_payload_bw(mgr, - drm_dp_bw_code_to_link_rate(link_rate), + link_rate, lane_count); if (mgr->pbn_div == 0) { ret = -EINVAL; @@ -5454,7 +5454,7 @@ EXPORT_SYMBOL(drm_atomic_get_mst_topology_state); * @max_dpcd_transaction_bytes: hw specific DPCD transaction limit * @max_payloads: maximum number of payloads this GPU can source * @max_lane_count: maximum number of lanes this GPU supports - * @max_link_rate: maximum link rate this GPU supports, units as in DPCD + * @max_link_rate: maximum link rate per lane this GPU supports in kHz * @conn_base_id: the connector object ID the MST device is connected to. * * Return 0 for success, or negative error code on failure @@ -5462,7 +5462,7 @@ EXPORT_SYMBOL(drm_atomic_get_mst_topology_state); int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr, struct drm_device *dev, struct drm_dp_aux *aux, int max_dpcd_transaction_bytes, int max_payloads, - u8 max_lane_count, u8 max_link_rate, + int max_lane_count, int max_link_rate, int conn_base_id) { struct drm_dp_mst_topology_state *mst_state; diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c index cb2349ad338d..69c57273b184 100644 --- a/drivers/gpu/drm/drm_fb_cma_helper.c +++ b/drivers/gpu/drm/drm_fb_cma_helper.c @@ -9,12 +9,14 @@ * Copyright (C) 2012 Red Hat */ +#include <drm/drm_damage_helper.h> #include <drm/drm_fb_cma_helper.h> #include <drm/drm_fourcc.h> #include <drm/drm_framebuffer.h> #include <drm/drm_gem_cma_helper.h> #include <drm/drm_gem_framebuffer_helper.h> #include <drm/drm_plane.h> +#include <linux/dma-mapping.h> #include <linux/module.h> /** @@ -97,3 +99,47 @@ dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb, return paddr; } EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_addr); + +/** + * drm_fb_cma_sync_non_coherent - Sync GEM object to non-coherent backing + * memory + * @drm: DRM device + * @old_state: Old plane state + * @state: New plane state + * + * This function can be used by drivers that use damage clips and have + * CMA GEM objects backed by non-coherent memory. Calling this function + * in a plane's .atomic_update ensures that all the data in the backing + * memory have been written to RAM. + */ +void drm_fb_cma_sync_non_coherent(struct drm_device *drm, + struct drm_plane_state *old_state, + struct drm_plane_state *state) +{ + const struct drm_format_info *finfo = state->fb->format; + struct drm_atomic_helper_damage_iter iter; + const struct drm_gem_cma_object *cma_obj; + unsigned int offset, i; + struct drm_rect clip; + dma_addr_t daddr; + size_t nb_bytes; + + for (i = 0; i < finfo->num_planes; i++) { + cma_obj = drm_fb_cma_get_gem_obj(state->fb, i); + if (!cma_obj->map_noncoherent) + continue; + + daddr = drm_fb_cma_get_gem_addr(state->fb, state, i); + drm_atomic_helper_damage_iter_init(&iter, old_state, state); + + drm_atomic_for_each_plane_damage(&iter, &clip) { + /* Ignore x1/x2 values, invalidate complete lines */ + offset = clip.y1 * state->fb->pitches[i]; + + nb_bytes = (clip.y2 - clip.y1) * state->fb->pitches[i]; + dma_sync_single_for_device(drm->dev, daddr + offset, + nb_bytes, DMA_TO_DEVICE); + } + } +} +EXPORT_SYMBOL_GPL(drm_fb_cma_sync_non_coherent); diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index f6baa2046124..d77a24507d30 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -1737,7 +1737,7 @@ void drm_fb_helper_fill_info(struct fb_info *info, sizes->fb_width, sizes->fb_height); info->par = fb_helper; - snprintf(info->fix.id, sizeof(info->fix.id), "%sdrmfb", + snprintf(info->fix.id, sizeof(info->fix.id), "%s", fb_helper->dev->driver->name); } diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c index ce13d2be5d7b..eda832f9200d 100644 --- a/drivers/gpu/drm/drm_fourcc.c +++ b/drivers/gpu/drm/drm_fourcc.c @@ -30,11 +30,6 @@ #include <drm/drm_device.h> #include <drm/drm_fourcc.h> -static char printable_char(int c) -{ - return isascii(c) && isprint(c) ? c : '?'; -} - /** * drm_mode_legacy_fb_format - compute drm fourcc code from legacy description * @bpp: bits per pixels @@ -130,26 +125,6 @@ uint32_t drm_driver_legacy_fb_format(struct drm_device *dev, } EXPORT_SYMBOL(drm_driver_legacy_fb_format); -/** - * drm_get_format_name - fill a string with a drm fourcc format's name - * @format: format to compute name of - * @buf: caller-supplied buffer - */ -const char *drm_get_format_name(uint32_t format, struct drm_format_name_buf *buf) -{ - snprintf(buf->str, sizeof(buf->str), - "%c%c%c%c %s-endian (0x%08x)", - printable_char(format & 0xff), - printable_char((format >> 8) & 0xff), - printable_char((format >> 16) & 0xff), - printable_char((format >> 24) & 0x7f), - format & DRM_FORMAT_BIG_ENDIAN ? "big" : "little", - format); - - return buf->str; -} -EXPORT_SYMBOL(drm_get_format_name); - /* * Internal function to query information for a given format. See * drm_format_info() for the public API. diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c index 7942cf05cd93..4a24af2c5c43 100644 --- a/drivers/gpu/drm/drm_gem_cma_helper.c +++ b/drivers/gpu/drm/drm_gem_cma_helper.c @@ -46,6 +46,7 @@ static const struct drm_gem_object_funcs drm_gem_cma_default_funcs = { * __drm_gem_cma_create - Create a GEM CMA object without allocating memory * @drm: DRM device * @size: size of the object to allocate + * @private: true if used for internal purposes * * This function creates and initializes a GEM CMA object of the given size, * but doesn't allocate any memory to back the object. @@ -55,11 +56,11 @@ static const struct drm_gem_object_funcs drm_gem_cma_default_funcs = { * error code on failure. */ static struct drm_gem_cma_object * -__drm_gem_cma_create(struct drm_device *drm, size_t size) +__drm_gem_cma_create(struct drm_device *drm, size_t size, bool private) { struct drm_gem_cma_object *cma_obj; struct drm_gem_object *gem_obj; - int ret; + int ret = 0; if (drm->driver->gem_create_object) gem_obj = drm->driver->gem_create_object(drm, size); @@ -73,7 +74,14 @@ __drm_gem_cma_create(struct drm_device *drm, size_t size) cma_obj = container_of(gem_obj, struct drm_gem_cma_object, base); - ret = drm_gem_object_init(drm, gem_obj, size); + if (private) { + drm_gem_private_object_init(drm, gem_obj, size); + + /* Always use writecombine for dma-buf mappings */ + cma_obj->map_noncoherent = false; + } else { + ret = drm_gem_object_init(drm, gem_obj, size); + } if (ret) goto error; @@ -111,12 +119,19 @@ struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm, size = round_up(size, PAGE_SIZE); - cma_obj = __drm_gem_cma_create(drm, size); + cma_obj = __drm_gem_cma_create(drm, size, false); if (IS_ERR(cma_obj)) return cma_obj; - cma_obj->vaddr = dma_alloc_wc(drm->dev, size, &cma_obj->paddr, - GFP_KERNEL | __GFP_NOWARN); + if (cma_obj->map_noncoherent) { + cma_obj->vaddr = dma_alloc_noncoherent(drm->dev, size, + &cma_obj->paddr, + DMA_TO_DEVICE, + GFP_KERNEL | __GFP_NOWARN); + } else { + cma_obj->vaddr = dma_alloc_wc(drm->dev, size, &cma_obj->paddr, + GFP_KERNEL | __GFP_NOWARN); + } if (!cma_obj->vaddr) { drm_dbg(drm, "failed to allocate buffer with size %zu\n", size); @@ -432,7 +447,7 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev, return ERR_PTR(-EINVAL); /* Create a CMA GEM buffer. */ - cma_obj = __drm_gem_cma_create(dev, attach->dmabuf->size); + cma_obj = __drm_gem_cma_create(dev, attach->dmabuf->size, true); if (IS_ERR(cma_obj)) return ERR_CAST(cma_obj); @@ -499,8 +514,16 @@ int drm_gem_cma_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) cma_obj = to_drm_gem_cma_obj(obj); - ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr, - cma_obj->paddr, vma->vm_end - vma->vm_start); + if (cma_obj->map_noncoherent) { + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + + ret = dma_mmap_pages(cma_obj->base.dev->dev, + vma, vma->vm_end - vma->vm_start, + virt_to_page(cma_obj->vaddr)); + } else { + ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr, + cma_obj->paddr, vma->vm_end - vma->vm_start); + } if (ret) drm_gem_vm_close(vma); diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c index 5ed2067cebb6..e2c68822e05c 100644 --- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c +++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c @@ -149,8 +149,10 @@ int drm_gem_fb_init_with_funcs(struct drm_device *dev, int ret, i; info = drm_get_format_info(dev, mode_cmd); - if (!info) + if (!info) { + drm_dbg_kms(dev, "Failed to get FB format info\n"); return -EINVAL; + } for (i = 0; i < info->num_planes; i++) { unsigned int width = mode_cmd->width / (i ? info->hsub : 1); @@ -169,6 +171,9 @@ int drm_gem_fb_init_with_funcs(struct drm_device *dev, + mode_cmd->offsets[i]; if (objs[i]->size < min_size) { + drm_dbg_kms(dev, + "GEM object size (%zu) smaller than minimum size (%u) for plane %d\n", + objs[i]->size, min_size, i); drm_gem_object_put(objs[i]); ret = -EINVAL; goto err_gem_object_put; diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h index 1dcb5797a3bb..17f3548c8ed2 100644 --- a/drivers/gpu/drm/drm_internal.h +++ b/drivers/gpu/drm/drm_internal.h @@ -54,18 +54,10 @@ void drm_lastclose(struct drm_device *dev); #ifdef CONFIG_PCI /* drm_pci.c */ -int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, - struct drm_file *file_priv); int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master); #else -static inline int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, - struct drm_file *file_priv) -{ - return -EINVAL; -} - static inline int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master) { diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index b0856c139693..53d314103a37 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -577,7 +577,8 @@ static const struct drm_ioctl_desc drm_ioctls[] = { DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version, DRM_RENDER_ALLOW), DRM_IOCTL_DEF(DRM_IOCTL_GET_UNIQUE, drm_getunique, 0), DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, 0), - DRM_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_legacy_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY), + DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_legacy_irq_by_busid, + DRM_MASTER|DRM_ROOT_ONLY), DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_GET_MAP, drm_legacy_getmap_ioctl, 0), diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c index da4f085fc09e..00fb433bcef1 100644 --- a/drivers/gpu/drm/drm_lease.c +++ b/drivers/gpu/drm/drm_lease.c @@ -14,7 +14,6 @@ #include "drm_crtc_internal.h" #include "drm_internal.h" -#include "drm_legacy.h" #define drm_for_each_lessee(lessee, lessor) \ list_for_each_entry((lessee), &(lessor)->lessees, lessee_list) diff --git a/drivers/gpu/drm/drm_legacy.h b/drivers/gpu/drm/drm_legacy.h index 7080d2538421..c9206840c87f 100644 --- a/drivers/gpu/drm/drm_legacy.h +++ b/drivers/gpu/drm/drm_legacy.h @@ -235,9 +235,17 @@ void drm_master_legacy_init(struct drm_master *master); static inline void drm_master_legacy_init(struct drm_master *master) {} #endif +/* drm_pci.c */ #if IS_ENABLED(CONFIG_DRM_LEGACY) && IS_ENABLED(CONFIG_PCI) +int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, struct drm_file *file_priv); void drm_legacy_pci_agp_destroy(struct drm_device *dev); #else +static inline int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + return -EINVAL; +} + static inline void drm_legacy_pci_agp_destroy(struct drm_device *dev) {} #endif diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c index 38c3cb72e7e6..39d35fc3a43b 100644 --- a/drivers/gpu/drm/drm_pci.c +++ b/drivers/gpu/drm/drm_pci.c @@ -72,7 +72,9 @@ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master) return 0; } -static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p) +#ifdef CONFIG_DRM_LEGACY + +static int drm_legacy_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p) { struct pci_dev *pdev = to_pci_dev(dev->dev); @@ -115,11 +117,9 @@ int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) return -EOPNOTSUPP; - return drm_pci_irq_by_busid(dev, p); + return drm_legacy_pci_irq_by_busid(dev, p); } -#ifdef CONFIG_DRM_LEGACY - void drm_legacy_pci_agp_destroy(struct drm_device *dev) { if (dev->agp) { @@ -165,9 +165,6 @@ static int drm_legacy_get_pci_dev(struct pci_dev *pdev, dev->hose = pdev->sysdata; #endif - if (drm_core_check_feature(dev, DRIVER_MODESET)) - pci_set_drvdata(pdev, dev); - drm_legacy_pci_agp_init(dev); ret = drm_dev_register(dev, ent->driver_data); diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c index b9a4b7670a89..197b97341cad 100644 --- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c +++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c @@ -815,10 +815,8 @@ static int exynos5433_decon_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ctx->addr = devm_ioremap_resource(dev, res); - if (IS_ERR(ctx->addr)) { - dev_err(dev, "ioremap failed\n"); + if (IS_ERR(ctx->addr)) return PTR_ERR(ctx->addr); - } ret = decon_conf_irq(ctx, "vsync", decon_irq_handler, 0); if (ret < 0) diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c index 44e402b7cdfb..2d2fe5ab26e7 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c @@ -1786,10 +1786,8 @@ static int exynos_dsi_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); dsi->reg_base = devm_ioremap_resource(dev, res); - if (IS_ERR(dsi->reg_base)) { - dev_err(dev, "failed to remap io region\n"); + if (IS_ERR(dsi->reg_base)) return PTR_ERR(dsi->reg_base); - } dsi->phy = devm_phy_get(dev, "dsim"); if (IS_ERR(dsi->phy)) { diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c index 49a2e0c53918..ae576122873e 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c @@ -723,7 +723,7 @@ static void fimd_win_set_colkey(struct fimd_context *ctx, unsigned int win) } /** - * shadow_protect_win() - disable updating values from shadow registers at vsync + * fimd_shadow_protect_win() - disable updating values from shadow registers at vsync * * @ctx: local driver data * @win: window to protect registers for diff --git a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c index 00e87c290796..9b565a057340 100644 --- a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c +++ b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c @@ -3,7 +3,7 @@ * DesignWare MIPI DSI Host Controller v1.02 driver * * Copyright (c) 2016 Linaro Limited. - * Copyright (c) 2014-2016 Hisilicon Limited. + * Copyright (c) 2014-2016 HiSilicon Limited. * * Author: * Xinliang Liu <z.liuxinliang@hisilicon.com> diff --git a/drivers/gpu/drm/hisilicon/kirin/dw_dsi_reg.h b/drivers/gpu/drm/hisilicon/kirin/dw_dsi_reg.h index 19e81ff64fac..d79fc031e53d 100644 --- a/drivers/gpu/drm/hisilicon/kirin/dw_dsi_reg.h +++ b/drivers/gpu/drm/hisilicon/kirin/dw_dsi_reg.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2016 Linaro Limited. - * Copyright (c) 2014-2016 Hisilicon Limited. + * Copyright (c) 2014-2016 HiSilicon Limited. */ #ifndef __DW_DSI_REG_H__ diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_ade_reg.h b/drivers/gpu/drm/hisilicon/kirin/kirin_ade_reg.h index e2ac09894a6d..be9e789c2d04 100644 --- a/drivers/gpu/drm/hisilicon/kirin/kirin_ade_reg.h +++ b/drivers/gpu/drm/hisilicon/kirin/kirin_ade_reg.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2016 Linaro Limited. - * Copyright (c) 2014-2016 Hisilicon Limited. + * Copyright (c) 2014-2016 HiSilicon Limited. */ #ifndef __KIRIN_ADE_REG_H__ diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c index 6dcf9ec05eec..1ab94620776f 100644 --- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c +++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c @@ -3,7 +3,7 @@ * Hisilicon Hi6220 SoC ADE(Advanced Display Engine)'s crtc&plane driver * * Copyright (c) 2016 Linaro Limited. - * Copyright (c) 2014-2016 Hisilicon Limited. + * Copyright (c) 2014-2016 HiSilicon Limited. * * Author: * Xinliang Liu <z.liuxinliang@hisilicon.com> diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c index 4349da3e2379..e590e19db657 100644 --- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c +++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c @@ -3,7 +3,7 @@ * Hisilicon Kirin SoCs drm master driver * * Copyright (c) 2016 Linaro Limited. - * Copyright (c) 2014-2016 Hisilicon Limited. + * Copyright (c) 2014-2016 HiSilicon Limited. * * Author: * Xinliang Liu <z.liuxinliang@hisilicon.com> diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h index 386d137f29e5..db0dc7bdf72c 100644 --- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h +++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2016 Linaro Limited. - * Copyright (c) 2014-2016 Hisilicon Limited. + * Copyright (c) 2014-2016 HiSilicon Limited. */ #ifndef __KIRIN_DRM_DRV_H__ diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 69f57ca9c68d..93f4d059fc89 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -102,7 +102,6 @@ config DRM_I915_GVT bool "Enable Intel GVT-g graphics virtualization host support" depends on DRM_I915 depends on 64BIT - depends on VFIO_MDEV=y || VFIO_MDEV=DRM_I915 default n help Choose this option if you want to enable Intel GVT-g graphics diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 384ff0bb6e19..422b59ebf6dc 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -11660,11 +11660,20 @@ intel_user_framebuffer_create(struct drm_device *dev, struct drm_framebuffer *fb; struct drm_i915_gem_object *obj; struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd; + struct drm_i915_private *i915; obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]); if (!obj) return ERR_PTR(-ENOENT); + /* object is backed with LMEM for discrete */ + i915 = to_i915(obj->base.dev); + if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj)) { + /* object is "remote", not in local memory */ + i915_gem_object_put(obj); + return ERR_PTR(-EREMOTE); + } + fb = intel_framebuffer_create(obj, &mode_cmd); i915_gem_object_put(obj); diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 332d2f9fda5c..b170e272bdee 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -963,8 +963,8 @@ intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id) intel_dp_create_fake_mst_encoders(dig_port); ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, &i915->drm, &intel_dp->aux, 16, 3, - (u8)dig_port->max_lanes, - drm_dp_link_rate_to_bw_code(max_source_rate), + dig_port->max_lanes, + max_source_rate, conn_base_id); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index ccd00e65a5fe..4af40229f5ec 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -41,6 +41,8 @@ #include <drm/drm_fb_helper.h> #include <drm/drm_fourcc.h> +#include "gem/i915_gem_lmem.h" + #include "i915_drv.h" #include "intel_display_types.h" #include "intel_fbdev.h" @@ -137,14 +139,22 @@ static int intelfb_alloc(struct drm_fb_helper *helper, size = mode_cmd.pitches[0] * mode_cmd.height; size = PAGE_ALIGN(size); - /* If the FB is too big, just don't use it since fbdev is not very - * important and we should probably use that space with FBC or other - * features. */ obj = ERR_PTR(-ENODEV); - if (size * 2 < dev_priv->stolen_usable_size) - obj = i915_gem_object_create_stolen(dev_priv, size); - if (IS_ERR(obj)) - obj = i915_gem_object_create_shmem(dev_priv, size); + if (HAS_LMEM(dev_priv)) { + obj = i915_gem_object_create_lmem(dev_priv, size, + I915_BO_ALLOC_CONTIGUOUS); + } else { + /* + * If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. + */ + if (size * 2 < dev_priv->stolen_usable_size) + obj = i915_gem_object_create_stolen(dev_priv, size); + if (IS_ERR(obj)) + obj = i915_gem_object_create_shmem(dev_priv, size); + } + if (IS_ERR(obj)) { drm_err(&dev_priv->drm, "failed to allocate framebuffer\n"); return PTR_ERR(obj); @@ -178,6 +188,7 @@ static int intelfb_create(struct drm_fb_helper *helper, unsigned long flags = 0; bool prealloc = false; void __iomem *vaddr; + struct drm_i915_gem_object *obj; int ret; if (intel_fb && @@ -232,13 +243,27 @@ static int intelfb_create(struct drm_fb_helper *helper, info->fbops = &intelfb_ops; /* setup aperture base/size for vesafb takeover */ - info->apertures->ranges[0].base = ggtt->gmadr.start; - info->apertures->ranges[0].size = ggtt->mappable_end; + obj = intel_fb_obj(&intel_fb->base); + if (i915_gem_object_is_lmem(obj)) { + struct intel_memory_region *mem = obj->mm.region; + + info->apertures->ranges[0].base = mem->io_start; + info->apertures->ranges[0].size = mem->total; + + /* Use fbdev's framebuffer from lmem for discrete */ + info->fix.smem_start = + (unsigned long)(mem->io_start + + i915_gem_object_get_dma_address(obj, 0)); + info->fix.smem_len = obj->base.size; + } else { + info->apertures->ranges[0].base = ggtt->gmadr.start; + info->apertures->ranges[0].size = ggtt->mappable_end; - /* Our framebuffer is the entirety of fbdev's system memory */ - info->fix.smem_start = - (unsigned long)(ggtt->gmadr.start + vma->node.start); - info->fix.smem_len = vma->node.size; + /* Our framebuffer is the entirety of fbdev's system memory */ + info->fix.smem_start = + (unsigned long)(ggtt->gmadr.start + vma->node.start); + info->fix.smem_len = vma->node.size; + } vaddr = i915_vma_pin_iomap(vma); if (IS_ERR(vaddr)) { diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c index 8161d49e78ba..8e75debcce1a 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c @@ -211,7 +211,6 @@ static int frontbuffer_active(struct i915_active *ref) return 0; } -__i915_active_call static void frontbuffer_retire(struct i915_active *ref) { struct intel_frontbuffer *front = @@ -266,7 +265,8 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj) atomic_set(&front->bits, 0); i915_active_init(&front->write, frontbuffer_active, - i915_active_may_sleep(frontbuffer_retire)); + frontbuffer_retire, + I915_ACTIVE_RETIRE_SLEEPS); spin_lock(&i915->fb_tracking.lock); if (rcu_access_pointer(obj->frontbuffer)) { diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c index 46cba12be888..7e3f5c6ca484 100644 --- a/drivers/gpu/drm/i915/display/intel_overlay.c +++ b/drivers/gpu/drm/i915/display/intel_overlay.c @@ -384,8 +384,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay) i830_overlay_clock_gating(dev_priv, true); } -__i915_active_call static void -intel_overlay_last_flip_retire(struct i915_active *active) +static void intel_overlay_last_flip_retire(struct i915_active *active) { struct intel_overlay *overlay = container_of(active, typeof(*overlay), last_flip); @@ -1402,7 +1401,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv) overlay->saturation = 146; i915_active_init(&overlay->last_flip, - NULL, intel_overlay_last_flip_retire); + NULL, intel_overlay_last_flip_retire, 0); ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv)); if (ret) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index fd8ee52e17a4..188dee13e017 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -1046,7 +1046,6 @@ struct context_barrier_task { void *data; }; -__i915_active_call static void cb_retire(struct i915_active *base) { struct context_barrier_task *cb = container_of(base, typeof(*cb), base); @@ -1080,7 +1079,7 @@ static int context_barrier_task(struct i915_gem_context *ctx, if (!cb) return -ENOMEM; - i915_active_init(&cb->base, NULL, cb_retire); + i915_active_init(&cb->base, NULL, cb_retire, 0); err = i915_active_acquire(&cb->base); if (err) { kfree(cb); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.c b/drivers/gpu/drm/i915/gem/i915_gem_create.c index 45d60e3d98e3..548ddf39d853 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_create.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_create.c @@ -4,44 +4,102 @@ */ #include "gem/i915_gem_ioctls.h" +#include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" #include "i915_drv.h" +#include "i915_trace.h" +#include "i915_user_extensions.h" + +static u32 object_max_page_size(struct drm_i915_gem_object *obj) +{ + u32 max_page_size = 0; + int i; + + for (i = 0; i < obj->mm.n_placements; i++) { + struct intel_memory_region *mr = obj->mm.placements[i]; + + GEM_BUG_ON(!is_power_of_2(mr->min_page_size)); + max_page_size = max_t(u32, max_page_size, mr->min_page_size); + } + + GEM_BUG_ON(!max_page_size); + return max_page_size; +} + +static void object_set_placements(struct drm_i915_gem_object *obj, + struct intel_memory_region **placements, + unsigned int n_placements) +{ + GEM_BUG_ON(!n_placements); + + /* + * For the common case of one memory region, skip storing an + * allocated array and just point at the region directly. + */ + if (n_placements == 1) { + struct intel_memory_region *mr = placements[0]; + struct drm_i915_private *i915 = mr->i915; + + obj->mm.placements = &i915->mm.regions[mr->id]; + obj->mm.n_placements = 1; + } else { + obj->mm.placements = placements; + obj->mm.n_placements = n_placements; + } +} + +static int i915_gem_publish(struct drm_i915_gem_object *obj, + struct drm_file *file, + u64 *size_p, + u32 *handle_p) +{ + u64 size = obj->base.size; + int ret; + + ret = drm_gem_handle_create(file, &obj->base, handle_p); + /* drop reference from allocate - handle holds it now */ + i915_gem_object_put(obj); + if (ret) + return ret; + + *size_p = size; + return 0; +} static int -i915_gem_create(struct drm_file *file, - struct intel_memory_region *mr, - u64 *size_p, - u32 *handle_p) +i915_gem_setup(struct drm_i915_gem_object *obj, u64 size) { - struct drm_i915_gem_object *obj; - u32 handle; - u64 size; + struct intel_memory_region *mr = obj->mm.placements[0]; + unsigned int flags; int ret; - GEM_BUG_ON(!is_power_of_2(mr->min_page_size)); - size = round_up(*size_p, mr->min_page_size); + size = round_up(size, object_max_page_size(obj)); if (size == 0) return -EINVAL; /* For most of the ABI (e.g. mmap) we think in system pages */ GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); - /* Allocate the new object */ - obj = i915_gem_object_create_region(mr, size, 0); - if (IS_ERR(obj)) - return PTR_ERR(obj); + if (i915_gem_object_size_2big(size)) + return -E2BIG; - GEM_BUG_ON(size != obj->base.size); + /* + * For now resort to CPU based clearing for device local-memory, in the + * near future this will use the blitter engine for accelerated, GPU + * based clearing. + */ + flags = 0; + if (mr->type == INTEL_MEMORY_LOCAL) + flags = I915_BO_ALLOC_CPU_CLEAR; - ret = drm_gem_handle_create(file, &obj->base, &handle); - /* drop reference from allocate - handle holds it now */ - i915_gem_object_put(obj); + ret = mr->ops->init_object(mr, obj, size, flags); if (ret) return ret; - *handle_p = handle; - *size_p = size; + GEM_BUG_ON(size != obj->base.size); + + trace_i915_gem_object_create(obj); return 0; } @@ -50,9 +108,12 @@ i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, struct drm_mode_create_dumb *args) { + struct drm_i915_gem_object *obj; + struct intel_memory_region *mr; enum intel_memory_type mem_type; int cpp = DIV_ROUND_UP(args->bpp, 8); u32 format; + int ret; switch (cpp) { case 1: @@ -85,10 +146,22 @@ i915_gem_dumb_create(struct drm_file *file, if (HAS_LMEM(to_i915(dev))) mem_type = INTEL_MEMORY_LOCAL; - return i915_gem_create(file, - intel_memory_region_by_type(to_i915(dev), - mem_type), - &args->size, &args->handle); + obj = i915_gem_object_alloc(); + if (!obj) + return -ENOMEM; + + mr = intel_memory_region_by_type(to_i915(dev), mem_type); + object_set_placements(obj, &mr, 1); + + ret = i915_gem_setup(obj, args->size); + if (ret) + goto object_free; + + return i915_gem_publish(obj, file, &args->size, &args->handle); + +object_free: + i915_gem_object_free(obj); + return ret; } /** @@ -103,11 +176,229 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_create *args = data; + struct drm_i915_gem_object *obj; + struct intel_memory_region *mr; + int ret; i915_gem_flush_free_objects(i915); - return i915_gem_create(file, - intel_memory_region_by_type(i915, - INTEL_MEMORY_SYSTEM), - &args->size, &args->handle); + obj = i915_gem_object_alloc(); + if (!obj) + return -ENOMEM; + + mr = intel_memory_region_by_type(i915, INTEL_MEMORY_SYSTEM); + object_set_placements(obj, &mr, 1); + + ret = i915_gem_setup(obj, args->size); + if (ret) + goto object_free; + + return i915_gem_publish(obj, file, &args->size, &args->handle); + +object_free: + i915_gem_object_free(obj); + return ret; +} + +struct create_ext { + struct drm_i915_private *i915; + struct drm_i915_gem_object *vanilla_object; +}; + +static void repr_placements(char *buf, size_t size, + struct intel_memory_region **placements, + int n_placements) +{ + int i; + + buf[0] = '\0'; + + for (i = 0; i < n_placements; i++) { + struct intel_memory_region *mr = placements[i]; + int r; + + r = snprintf(buf, size, "\n %s -> { class: %d, inst: %d }", + mr->name, mr->type, mr->instance); + if (r >= size) + return; + + buf += r; + size -= r; + } +} + +static int set_placements(struct drm_i915_gem_create_ext_memory_regions *args, + struct create_ext *ext_data) +{ + struct drm_i915_private *i915 = ext_data->i915; + struct drm_i915_gem_memory_class_instance __user *uregions = + u64_to_user_ptr(args->regions); + struct drm_i915_gem_object *obj = ext_data->vanilla_object; + struct intel_memory_region **placements; + u32 mask; + int i, ret = 0; + + if (args->pad) { + drm_dbg(&i915->drm, "pad should be zero\n"); + ret = -EINVAL; + } + + if (!args->num_regions) { + drm_dbg(&i915->drm, "num_regions is zero\n"); + ret = -EINVAL; + } + + if (args->num_regions > ARRAY_SIZE(i915->mm.regions)) { + drm_dbg(&i915->drm, "num_regions is too large\n"); + ret = -EINVAL; + } + + if (ret) + return ret; + + placements = kmalloc_array(args->num_regions, + sizeof(struct intel_memory_region *), + GFP_KERNEL); + if (!placements) + return -ENOMEM; + + mask = 0; + for (i = 0; i < args->num_regions; i++) { + struct drm_i915_gem_memory_class_instance region; + struct intel_memory_region *mr; + + if (copy_from_user(®ion, uregions, sizeof(region))) { + ret = -EFAULT; + goto out_free; + } + + mr = intel_memory_region_lookup(i915, + region.memory_class, + region.memory_instance); + if (!mr || mr->private) { + drm_dbg(&i915->drm, "Device is missing region { class: %d, inst: %d } at index = %d\n", + region.memory_class, region.memory_instance, i); + ret = -EINVAL; + goto out_dump; + } + + if (mask & BIT(mr->id)) { + drm_dbg(&i915->drm, "Found duplicate placement %s -> { class: %d, inst: %d } at index = %d\n", + mr->name, region.memory_class, + region.memory_instance, i); + ret = -EINVAL; + goto out_dump; + } + + placements[i] = mr; + mask |= BIT(mr->id); + + ++uregions; + } + + if (obj->mm.placements) { + ret = -EINVAL; + goto out_dump; + } + + object_set_placements(obj, placements, args->num_regions); + if (args->num_regions == 1) + kfree(placements); + + return 0; + +out_dump: + if (1) { + char buf[256]; + + if (obj->mm.placements) { + repr_placements(buf, + sizeof(buf), + obj->mm.placements, + obj->mm.n_placements); + drm_dbg(&i915->drm, + "Placements were already set in previous EXT. Existing placements: %s\n", + buf); + } + + repr_placements(buf, sizeof(buf), placements, i); + drm_dbg(&i915->drm, "New placements(so far validated): %s\n", buf); + } + +out_free: + kfree(placements); + return ret; +} + +static int ext_set_placements(struct i915_user_extension __user *base, + void *data) +{ + struct drm_i915_gem_create_ext_memory_regions ext; + + if (!IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) + return -ENODEV; + + if (copy_from_user(&ext, base, sizeof(ext))) + return -EFAULT; + + return set_placements(&ext, data); +} + +static const i915_user_extension_fn create_extensions[] = { + [I915_GEM_CREATE_EXT_MEMORY_REGIONS] = ext_set_placements, +}; + +/** + * Creates a new mm object and returns a handle to it. + * @dev: drm device pointer + * @data: ioctl data blob + * @file: drm file pointer + */ +int +i915_gem_create_ext_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_i915_private *i915 = to_i915(dev); + struct drm_i915_gem_create_ext *args = data; + struct create_ext ext_data = { .i915 = i915 }; + struct intel_memory_region **placements_ext; + struct drm_i915_gem_object *obj; + int ret; + + if (args->flags) + return -EINVAL; + + i915_gem_flush_free_objects(i915); + + obj = i915_gem_object_alloc(); + if (!obj) + return -ENOMEM; + + ext_data.vanilla_object = obj; + ret = i915_user_extensions(u64_to_user_ptr(args->extensions), + create_extensions, + ARRAY_SIZE(create_extensions), + &ext_data); + placements_ext = obj->mm.placements; + if (ret) + goto object_free; + + if (!placements_ext) { + struct intel_memory_region *mr = + intel_memory_region_by_type(i915, INTEL_MEMORY_SYSTEM); + + object_set_placements(obj, &mr, 1); + } + + ret = i915_gem_setup(obj, args->size); + if (ret) + goto object_free; + + return i915_gem_publish(obj, file, &args->size, &args->handle); + +object_free: + if (obj->mm.n_placements > 1) + kfree(placements_ext); + i915_gem_object_free(obj); + return ret; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h b/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h index 7fd22f3efbef..28d6526e32ab 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h @@ -14,6 +14,8 @@ int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int i915_gem_create_ext_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); int i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c index ce1c83c13d05..f44bdd08f7cb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c @@ -17,9 +17,27 @@ const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops = { .release = i915_gem_object_release_memory_region, }; +void __iomem * +i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, + unsigned long n, + unsigned long size) +{ + resource_size_t offset; + + GEM_BUG_ON(!i915_gem_object_is_contiguous(obj)); + + offset = i915_gem_object_get_dma_address(obj, n); + offset -= obj->mm.region->region.start; + + return io_mapping_map_wc(&obj->mm.region->iomap, offset, size); +} + bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) { - return obj->ops == &i915_gem_lmem_obj_ops; + struct intel_memory_region *mr = obj->mm.region; + + return mr && (mr->type == INTEL_MEMORY_LOCAL || + mr->type == INTEL_MEMORY_STOLEN_LOCAL); } struct drm_i915_gem_object * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h index 036d53c01de9..fac6bc5a5ebb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h @@ -14,6 +14,11 @@ struct intel_memory_region; extern const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops; +void __iomem * +i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, + unsigned long n, + unsigned long size); + bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); struct drm_i915_gem_object * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index ea74cbca95be..28144410df86 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -249,6 +249,9 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, if (obj->ops->release) obj->ops->release(obj); + if (obj->mm.n_placements > 1) + kfree(obj->mm.placements); + /* But keep the pointer alive for RCU-protected lookups */ call_rcu(&obj->rcu, __i915_gem_free_object_rcu); cond_resched(); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 8e485cb3343c..0727d0c76aa0 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -172,11 +172,13 @@ struct drm_i915_gem_object { #define I915_BO_ALLOC_CONTIGUOUS BIT(0) #define I915_BO_ALLOC_VOLATILE BIT(1) #define I915_BO_ALLOC_STRUCT_PAGE BIT(2) +#define I915_BO_ALLOC_CPU_CLEAR BIT(3) #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \ I915_BO_ALLOC_VOLATILE | \ - I915_BO_ALLOC_STRUCT_PAGE) -#define I915_BO_READONLY BIT(3) -#define I915_TILING_QUIRK_BIT 4 /* unknown swizzling; do not release! */ + I915_BO_ALLOC_STRUCT_PAGE | \ + I915_BO_ALLOC_CPU_CLEAR) +#define I915_BO_READONLY BIT(4) +#define I915_TILING_QUIRK_BIT 5 /* unknown swizzling; do not release! */ /* * Is the object to be mapped as read-only to the GPU @@ -220,6 +222,12 @@ struct drm_i915_gem_object { atomic_t shrink_pin; /** + * Priority list of potential placements for this object. + */ + struct intel_memory_region **placements; + int n_placements; + + /** * Memory region for this object. */ struct intel_memory_region *region; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index aed8a37ccdc9..7361971c177d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -63,6 +63,8 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); i915_gem_object_set_tiling_quirk(obj); + GEM_BUG_ON(!list_empty(&obj->mm.link)); + atomic_inc(&obj->mm.shrink_pin); shrinkable = false; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c index 81dc2bf59bc3..51a05e62875d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c @@ -8,7 +8,6 @@ #include <linux/shmem_fs.h> #include <linux/swap.h> -#include <drm/drm.h> /* for drm_legacy.h! */ #include <drm/drm_cache.h> #include "gt/intel_gt.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index 6a84fb6dde24..ce8fcfc54079 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -95,6 +95,28 @@ i915_gem_object_get_pages_buddy(struct drm_i915_gem_object *obj) sg_mark_end(sg); i915_sg_trim(st); + /* Intended for kernel internal use only */ + if (obj->flags & I915_BO_ALLOC_CPU_CLEAR) { + struct scatterlist *sg; + unsigned long i; + + for_each_sg(st->sgl, sg, st->nents, i) { + unsigned int length; + void __iomem *vaddr; + dma_addr_t daddr; + + daddr = sg_dma_address(sg); + daddr -= mem->region.start; + length = sg_dma_len(sg); + + vaddr = io_mapping_map_wc(&mem->iomap, daddr, length); + memset64((void __force *)vaddr, 0, length / sizeof(u64)); + io_mapping_unmap(vaddr); + } + + wmb(); + } + __i915_gem_object_set_pages(obj, st, sg_page_sizes); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index 4f9c8d3021ab..f4fb68e8955a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -38,15 +38,17 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) } static bool unsafe_drop_pages(struct drm_i915_gem_object *obj, - unsigned long shrink) + unsigned long shrink, bool trylock_vm) { unsigned long flags; flags = 0; if (shrink & I915_SHRINK_ACTIVE) - flags = I915_GEM_OBJECT_UNBIND_ACTIVE; + flags |= I915_GEM_OBJECT_UNBIND_ACTIVE; if (!(shrink & I915_SHRINK_BOUND)) - flags = I915_GEM_OBJECT_UNBIND_TEST; + flags |= I915_GEM_OBJECT_UNBIND_TEST; + if (trylock_vm) + flags |= I915_GEM_OBJECT_UNBIND_VM_TRYLOCK; if (i915_gem_object_unbind(obj, flags) == 0) return true; @@ -117,6 +119,9 @@ i915_gem_shrink(struct i915_gem_ww_ctx *ww, unsigned long scanned = 0; int err; + /* CHV + VTD workaround use stop_machine(); need to trylock vm->mutex */ + bool trylock_vm = !ww && intel_vm_no_concurrent_access_wa(i915); + trace_i915_gem_shrink(i915, target, shrink); /* @@ -204,7 +209,7 @@ i915_gem_shrink(struct i915_gem_ww_ctx *ww, spin_unlock_irqrestore(&i915->mm.obj_lock, flags); err = 0; - if (unsafe_drop_pages(obj, shrink)) { + if (unsafe_drop_pages(obj, shrink, trylock_vm)) { /* May arrive from get_pages on another bo */ if (!ww) { if (!i915_gem_object_trylock(obj)) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index b0597de206de..b5553fc3ac4d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -10,6 +10,7 @@ #include <drm/drm_mm.h> #include <drm/i915_drm.h> +#include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" #include "i915_drv.h" #include "i915_gem_stolen.h" @@ -122,6 +123,14 @@ static int i915_adjust_stolen(struct drm_i915_private *i915, } /* + * With stolen lmem, we don't need to check if the address range + * overlaps with the non-stolen system memory range, since lmem is local + * to the gpu. + */ + if (HAS_LMEM(i915)) + return 0; + + /* * Verify that nothing else uses this physical address. Stolen * memory should be reserved by the BIOS and hidden from the * kernel. So if the region is already marked as busy, something @@ -374,8 +383,9 @@ static void icl_get_stolen_reserved(struct drm_i915_private *i915, } } -static int i915_gem_init_stolen(struct drm_i915_private *i915) +static int i915_gem_init_stolen(struct intel_memory_region *mem) { + struct drm_i915_private *i915 = mem->i915; struct intel_uncore *uncore = &i915->uncore; resource_size_t reserved_base, stolen_top; resource_size_t reserved_total, reserved_size; @@ -396,10 +406,10 @@ static int i915_gem_init_stolen(struct drm_i915_private *i915) return 0; } - if (resource_size(&intel_graphics_stolen_res) == 0) + if (resource_size(&mem->region) == 0) return 0; - i915->dsm = intel_graphics_stolen_res; + i915->dsm = mem->region; if (i915_adjust_stolen(i915, &i915->dsm)) return 0; @@ -627,10 +637,17 @@ static int __i915_gem_object_create_stolen(struct intel_memory_region *mem, { static struct lock_class_key lock_class; unsigned int cache_level; + unsigned int flags; int err; + /* + * Stolen objects are always physically contiguous since we just + * allocate one big block underneath using the drm_mm range allocator. + */ + flags = I915_BO_ALLOC_CONTIGUOUS; + drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size); - i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, 0); + i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, flags); obj->stolen = stolen; obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; @@ -640,9 +657,11 @@ static int __i915_gem_object_create_stolen(struct intel_memory_region *mem, if (WARN_ON(!i915_gem_object_trylock(obj))) return -EBUSY; + i915_gem_object_init_memory_region(obj, mem); + err = i915_gem_object_pin_pages(obj); - if (!err) - i915_gem_object_init_memory_region(obj, mem); + if (err) + i915_gem_object_release_memory_region(obj); i915_gem_object_unlock(obj); return err; @@ -667,7 +686,8 @@ static int _i915_gem_object_stolen_init(struct intel_memory_region *mem, if (!stolen) return -ENOMEM; - ret = i915_gem_stolen_insert_node(i915, stolen, size, 4096); + ret = i915_gem_stolen_insert_node(i915, stolen, size, + mem->min_page_size); if (ret) goto err_free; @@ -688,39 +708,126 @@ struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_i915_private *i915, resource_size_t size) { - return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_STOLEN_SMEM], - size, I915_BO_ALLOC_CONTIGUOUS); + return i915_gem_object_create_region(i915->mm.stolen_region, size, 0); } -static int init_stolen(struct intel_memory_region *mem) +static int init_stolen_smem(struct intel_memory_region *mem) { - intel_memory_region_set_name(mem, "stolen"); - /* * Initialise stolen early so that we may reserve preallocated * objects for the BIOS to KMS transition. */ - return i915_gem_init_stolen(mem->i915); + return i915_gem_init_stolen(mem); } -static void release_stolen(struct intel_memory_region *mem) +static void release_stolen_smem(struct intel_memory_region *mem) { i915_gem_cleanup_stolen(mem->i915); } -static const struct intel_memory_region_ops i915_region_stolen_ops = { - .init = init_stolen, - .release = release_stolen, +static const struct intel_memory_region_ops i915_region_stolen_smem_ops = { + .init = init_stolen_smem, + .release = release_stolen_smem, .init_object = _i915_gem_object_stolen_init, }; -struct intel_memory_region *i915_gem_stolen_setup(struct drm_i915_private *i915) +static int init_stolen_lmem(struct intel_memory_region *mem) { - return intel_memory_region_create(i915, - intel_graphics_stolen_res.start, - resource_size(&intel_graphics_stolen_res), - PAGE_SIZE, 0, - &i915_region_stolen_ops); + int err; + + if (GEM_WARN_ON(resource_size(&mem->region) == 0)) + return -ENODEV; + + if (!io_mapping_init_wc(&mem->iomap, + mem->io_start, + resource_size(&mem->region))) + return -EIO; + + /* + * TODO: For stolen lmem we mostly just care about populating the dsm + * related bits and setting up the drm_mm allocator for the range. + * Perhaps split up i915_gem_init_stolen() for this. + */ + err = i915_gem_init_stolen(mem); + if (err) + goto err_fini; + + return 0; + +err_fini: + io_mapping_fini(&mem->iomap); + return err; +} + +static void release_stolen_lmem(struct intel_memory_region *mem) +{ + io_mapping_fini(&mem->iomap); + i915_gem_cleanup_stolen(mem->i915); +} + +static const struct intel_memory_region_ops i915_region_stolen_lmem_ops = { + .init = init_stolen_lmem, + .release = release_stolen_lmem, + .init_object = _i915_gem_object_stolen_init, +}; + +struct intel_memory_region * +i915_gem_stolen_lmem_setup(struct drm_i915_private *i915) +{ + struct intel_uncore *uncore = &i915->uncore; + struct pci_dev *pdev = to_pci_dev(i915->drm.dev); + struct intel_memory_region *mem; + resource_size_t io_start; + resource_size_t lmem_size; + u64 lmem_base; + + lmem_base = intel_uncore_read64(uncore, GEN12_DSMBASE); + if (GEM_WARN_ON(lmem_base >= pci_resource_len(pdev, 2))) + return ERR_PTR(-ENODEV); + + lmem_size = pci_resource_len(pdev, 2) - lmem_base; + io_start = pci_resource_start(pdev, 2) + lmem_base; + + mem = intel_memory_region_create(i915, lmem_base, lmem_size, + I915_GTT_PAGE_SIZE_4K, io_start, + &i915_region_stolen_lmem_ops); + if (IS_ERR(mem)) + return mem; + + /* + * TODO: consider creating common helper to just print all the + * interesting stuff from intel_memory_region, which we can use for all + * our probed regions. + */ + + drm_dbg(&i915->drm, "Stolen Local memory IO start: %pa\n", + &mem->io_start); + + intel_memory_region_set_name(mem, "stolen-local"); + + mem->private = true; + + return mem; +} + +struct intel_memory_region* +i915_gem_stolen_smem_setup(struct drm_i915_private *i915) +{ + struct intel_memory_region *mem; + + mem = intel_memory_region_create(i915, + intel_graphics_stolen_res.start, + resource_size(&intel_graphics_stolen_res), + PAGE_SIZE, 0, + &i915_region_stolen_smem_ops); + if (IS_ERR(mem)) + return mem; + + intel_memory_region_set_name(mem, "stolen-system"); + + mem->private = true; + + return mem; } struct drm_i915_gem_object * @@ -728,7 +835,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *i915, resource_size_t stolen_offset, resource_size_t size) { - struct intel_memory_region *mem = i915->mm.regions[INTEL_REGION_STOLEN_SMEM]; + struct intel_memory_region *mem = i915->mm.stolen_region; struct drm_i915_gem_object *obj; struct drm_mm_node *stolen; int ret; @@ -742,8 +849,8 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *i915, /* KISS and expect everything to be page-aligned */ if (GEM_WARN_ON(size == 0) || - GEM_WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) || - GEM_WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT))) + GEM_WARN_ON(!IS_ALIGNED(size, mem->min_page_size)) || + GEM_WARN_ON(!IS_ALIGNED(stolen_offset, mem->min_page_size))) return ERR_PTR(-EINVAL); stolen = kzalloc(sizeof(*stolen), GFP_KERNEL); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h index b03489706796..2bec6c367b9c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h @@ -21,7 +21,8 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, u64 end); void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, struct drm_mm_node *node); -struct intel_memory_region *i915_gem_stolen_setup(struct drm_i915_private *i915); +struct intel_memory_region *i915_gem_stolen_smem_setup(struct drm_i915_private *i915); +struct intel_memory_region *i915_gem_stolen_lmem_setup(struct drm_i915_private *i915); struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, resource_size_t size); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 5fef592390cb..ce70d0a3afb2 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -1740,7 +1740,6 @@ out: static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) { struct i915_address_space *vm; - struct page *page; u32 *vaddr; int err = 0; @@ -1748,24 +1747,18 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) if (!vm) return -ENODEV; - page = __px_page(vm->scratch[0]); - if (!page) { + if (!vm->scratch[0]) { pr_err("No scratch page!\n"); return -EINVAL; } - vaddr = kmap(page); - if (!vaddr) { - pr_err("No (mappable) scratch page!\n"); - return -EINVAL; - } + vaddr = __px_vaddr(vm->scratch[0]); memcpy(out, vaddr, sizeof(*out)); if (memchr_inv(vaddr, *out, PAGE_SIZE)) { pr_err("Inconsistent initial state of scratch page!\n"); err = -EINVAL; } - kunmap(page); return err; } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 5cf6df49c333..05a3b29f545e 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -842,6 +842,24 @@ static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) return true; } +static void object_set_placements(struct drm_i915_gem_object *obj, + struct intel_memory_region **placements, + unsigned int n_placements) +{ + GEM_BUG_ON(!n_placements); + + if (n_placements == 1) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct intel_memory_region *mr = placements[0]; + + obj->mm.placements = &i915->mm.regions[mr->id]; + obj->mm.n_placements = 1; + } else { + obj->mm.placements = placements; + obj->mm.n_placements = n_placements; + } +} + #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) static int __igt_mmap(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, @@ -950,6 +968,8 @@ static int igt_mmap(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC); @@ -1068,6 +1088,8 @@ static int igt_mmap_access(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB); @@ -1211,6 +1233,8 @@ static int igt_mmap_gpu(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC); @@ -1354,6 +1378,8 @@ static int igt_mmap_revoke(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index e08dff376339..1aee5e6b1b23 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -96,9 +96,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, * entries back to scratch. */ - vaddr = kmap_atomic_px(pt); + vaddr = px_vaddr(pt); memset32(vaddr + pte, scratch_pte, count); - kunmap_atomic(vaddr); pte = 0; } @@ -120,7 +119,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, GEM_BUG_ON(!pd->entry[act_pt]); - vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt)); + vaddr = px_vaddr(i915_pt_entry(pd, act_pt)); do { GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE); vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); @@ -136,12 +135,10 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, } if (++act_pte == GEN6_PTES) { - kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt)); + vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt)); act_pte = 0; } } while (1); - kunmap_atomic(vaddr); vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; } @@ -235,7 +232,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) goto err_scratch0; } - ret = pin_pt_dma(vm, vm->scratch[1]); + ret = map_pt_dma(vm, vm->scratch[1]); if (ret) goto err_scratch1; @@ -346,7 +343,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size) if (!vma) return ERR_PTR(-ENOMEM); - i915_active_init(&vma->active, NULL, NULL); + i915_active_init(&vma->active, NULL, NULL, 0); kref_init(&vma->ref); mutex_init(&vma->pages_mutex); diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 74bf6fc8461f..e3a8924d2286 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -242,11 +242,10 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, atomic_read(&pt->used)); GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); - vaddr = kmap_atomic_px(pt); + vaddr = px_vaddr(pt); memset64(vaddr + gen8_pd_index(start, 0), vm->scratch[0]->encode, count); - kunmap_atomic(vaddr); atomic_sub(count, &pt->used); start += count; @@ -375,7 +374,7 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, gen8_pte_t *vaddr; pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2)); - vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); + vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); do { GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE); vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; @@ -402,12 +401,10 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, } clflush_cache_range(vaddr, PAGE_SIZE); - kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); + vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); } } while (1); clflush_cache_range(vaddr, PAGE_SIZE); - kunmap_atomic(vaddr); return idx; } @@ -442,7 +439,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, encode |= GEN8_PDE_PS_2M; page_size = I915_GTT_PAGE_SIZE_2M; - vaddr = kmap_atomic_px(pd); + vaddr = px_vaddr(pd); } else { struct i915_page_table *pt = i915_pt_entry(pd, __gen8_pte_index(start, 1)); @@ -457,7 +454,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)) maybe_64K = __gen8_pte_index(start, 1); - vaddr = kmap_atomic_px(pt); + vaddr = px_vaddr(pt); } do { @@ -491,7 +488,6 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, } while (rem >= page_size && index < I915_PDES); clflush_cache_range(vaddr, PAGE_SIZE); - kunmap_atomic(vaddr); /* * Is it safe to mark the 2M block as 64K? -- Either we have @@ -505,9 +501,8 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, !iter->sg && IS_ALIGNED(vma->node.start + vma->node.size, I915_GTT_PAGE_SIZE_2M)))) { - vaddr = kmap_atomic_px(pd); + vaddr = px_vaddr(pd); vaddr[maybe_64K] |= GEN8_PDE_IPS_64K; - kunmap_atomic(vaddr); page_size = I915_GTT_PAGE_SIZE_64K; /* @@ -523,12 +518,11 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, u16 i; encode = vma->vm->scratch[0]->encode; - vaddr = kmap_atomic_px(i915_pt_entry(pd, maybe_64K)); + vaddr = px_vaddr(i915_pt_entry(pd, maybe_64K)); for (i = 1; i < index; i += 16) memset64(vaddr + i, encode, 15); - kunmap_atomic(vaddr); } } @@ -602,7 +596,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) if (IS_ERR(obj)) goto free_scratch; - ret = pin_pt_dma(vm, obj); + ret = map_pt_dma(vm, obj); if (ret) { i915_gem_object_put(obj); goto free_scratch; @@ -639,7 +633,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) if (IS_ERR(pde)) return PTR_ERR(pde); - err = pin_pt_dma(vm, pde->pt.base); + err = map_pt_dma(vm, pde->pt.base); if (err) { free_pd(vm, pde); return err; @@ -674,7 +668,7 @@ gen8_alloc_top_pd(struct i915_address_space *vm) goto err_pd; } - err = pin_pt_dma(vm, pd->pt.base); + err = map_pt_dma(vm, pd->pt.base); if (err) goto err_pd; @@ -717,7 +711,10 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) */ ppgtt->vm.has_read_only = !IS_GEN_RANGE(gt->i915, 11, 12); - ppgtt->vm.alloc_pt_dma = alloc_pt_dma; + if (HAS_LMEM(gt->i915)) + ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; + else + ppgtt->vm.alloc_pt_dma = alloc_pt_dma; err = gen8_init_scratch(&ppgtt->vm); if (err) diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 17cf2640b082..4033184f13b9 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -326,7 +326,6 @@ void intel_context_unpin(struct intel_context *ce) intel_context_put(ce); } -__i915_active_call static void __intel_context_retire(struct i915_active *active) { struct intel_context *ce = container_of(active, typeof(*ce), active); @@ -385,7 +384,7 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) mutex_init(&ce->pin_mutex); i915_active_init(&ce->active, - __intel_context_active, __intel_context_retire); + __intel_context_active, __intel_context_retire, 0); } void intel_context_fini(struct intel_context *ce) diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index 47ee8578e511..8d9184920c51 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -13,8 +13,9 @@ #include "i915_reg.h" #include "i915_request.h" #include "i915_selftest.h" -#include "gt/intel_timeline.h" #include "intel_engine_types.h" +#include "intel_gt_types.h" +#include "intel_timeline.h" #include "intel_workarounds.h" struct drm_printer; @@ -262,6 +263,11 @@ void intel_engine_init_active(struct intel_engine_cs *engine, #define ENGINE_MOCK 1 #define ENGINE_VIRTUAL 2 +static inline bool intel_engine_uses_guc(const struct intel_engine_cs *engine) +{ + return engine->gt->submission_method >= INTEL_SUBMISSION_GUC; +} + static inline bool intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) { diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 6dbdbde00f14..3f9a811eb02b 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -255,6 +255,11 @@ static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) intel_engine_set_hwsp_writemask(engine, ~0u); } +static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + GEM_DEBUG_WARN_ON(iir); +} + static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) { const struct engine_info *info = &intel_engines[id]; @@ -292,6 +297,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->hw_id = info->hw_id; engine->guc_id = MAKE_GUC_ID(info->class, info->instance); + engine->irq_handler = nop_irq_handler; + engine->class = info->class; engine->instance = info->instance; __sprint_engine_name(engine); @@ -898,7 +905,7 @@ static int engine_init_common(struct intel_engine_cs *engine) return 0; err_context: - intel_context_put(ce); + destroy_pinned_context(ce); return ret; } @@ -909,12 +916,16 @@ int intel_engines_init(struct intel_gt *gt) enum intel_engine_id id; int err; - if (intel_uc_uses_guc_submission(>->uc)) + if (intel_uc_uses_guc_submission(>->uc)) { + gt->submission_method = INTEL_SUBMISSION_GUC; setup = intel_guc_submission_setup; - else if (HAS_EXECLISTS(gt->i915)) + } else if (HAS_EXECLISTS(gt->i915)) { + gt->submission_method = INTEL_SUBMISSION_ELSP; setup = intel_execlists_submission_setup; - else + } else { + gt->submission_method = INTEL_SUBMISSION_RING; setup = intel_ring_submission_setup; + } for_each_engine(engine, gt, id) { err = engine_setup_common(engine); @@ -1479,7 +1490,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); } - if (intel_engine_in_guc_submission_mode(engine)) { + if (intel_engine_uses_guc(engine)) { /* nothing to print yet */ } else if (HAS_EXECLISTS(dev_priv)) { struct i915_request * const *port, *rq; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 7c9af86fdb1e..47f4397095e5 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -23,7 +23,7 @@ static void dbg_poison_ce(struct intel_context *ce) if (ce->state) { struct drm_i915_gem_object *obj = ce->state->obj; - int type = i915_coherent_map_type(ce->engine->i915); + int type = i915_coherent_map_type(ce->engine->i915, obj, true); void *map; if (!i915_gem_object_trylock(obj)) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 883bafc44902..9ef349cd5cea 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -402,6 +402,7 @@ struct intel_engine_cs { u32 irq_enable_mask; /* bitmask to enable ring interrupt */ void (*irq_enable)(struct intel_engine_cs *engine); void (*irq_disable)(struct intel_engine_cs *engine); + void (*irq_handler)(struct intel_engine_cs *engine, u16 iir); void (*sanitize)(struct intel_engine_cs *engine); int (*resume)(struct intel_engine_cs *engine); @@ -481,10 +482,9 @@ struct intel_engine_cs { #define I915_ENGINE_HAS_PREEMPTION BIT(2) #define I915_ENGINE_HAS_SEMAPHORES BIT(3) #define I915_ENGINE_HAS_TIMESLICES BIT(4) -#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(5) -#define I915_ENGINE_IS_VIRTUAL BIT(6) -#define I915_ENGINE_HAS_RELATIVE_MMIO BIT(7) -#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(8) +#define I915_ENGINE_IS_VIRTUAL BIT(5) +#define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) +#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) unsigned int flags; /* @@ -594,12 +594,6 @@ intel_engine_has_timeslices(const struct intel_engine_cs *engine) } static inline bool -intel_engine_needs_breadcrumb_tasklet(const struct intel_engine_cs *engine) -{ - return engine->flags & I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; -} - -static inline bool intel_engine_is_virtual(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_IS_VIRTUAL; diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index de124870af44..8db200422950 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -118,6 +118,7 @@ #include "intel_engine_stats.h" #include "intel_execlists_submission.h" #include "intel_gt.h" +#include "intel_gt_irq.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" #include "intel_lrc.h" @@ -1768,7 +1769,6 @@ process_csb(struct intel_engine_cs *engine, struct i915_request **inactive) */ GEM_BUG_ON(!tasklet_is_locked(&execlists->tasklet) && !reset_in_progress(execlists)); - GEM_BUG_ON(!intel_engine_in_execlists_submission_mode(engine)); /* * Note that csb_write, csb_status may be either in HWSP or mmio. @@ -2385,6 +2385,45 @@ static void execlists_submission_tasklet(struct tasklet_struct *t) rcu_read_unlock(); } +static void execlists_irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + bool tasklet = false; + + if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) { + u32 eir; + + /* Upper 16b are the enabling mask, rsvd for internal errors */ + eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0); + ENGINE_TRACE(engine, "CS error: %x\n", eir); + + /* Disable the error interrupt until after the reset */ + if (likely(eir)) { + ENGINE_WRITE(engine, RING_EMR, ~0u); + ENGINE_WRITE(engine, RING_EIR, eir); + WRITE_ONCE(engine->execlists.error_interrupt, eir); + tasklet = true; + } + } + + if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) { + WRITE_ONCE(engine->execlists.yield, + ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI)); + ENGINE_TRACE(engine, "semaphore yield: %08x\n", + engine->execlists.yield); + if (del_timer(&engine->execlists.timer)) + tasklet = true; + } + + if (iir & GT_CONTEXT_SWITCH_INTERRUPT) + tasklet = true; + + if (iir & GT_RENDER_USER_INTERRUPT) + intel_engine_signal_breadcrumbs(engine); + + if (tasklet) + tasklet_hi_schedule(&engine->execlists.tasklet); +} + static void __execlists_kick(struct intel_engine_execlists *execlists) { /* Kick the tasklet for some interrupt coalescing and reset handling */ @@ -3076,29 +3115,6 @@ static void execlists_set_default_submission(struct intel_engine_cs *engine) engine->submit_request = execlists_submit_request; engine->schedule = i915_schedule; engine->execlists.tasklet.callback = execlists_submission_tasklet; - - engine->reset.prepare = execlists_reset_prepare; - engine->reset.rewind = execlists_reset_rewind; - engine->reset.cancel = execlists_reset_cancel; - engine->reset.finish = execlists_reset_finish; - - engine->park = execlists_park; - engine->unpark = NULL; - - engine->flags |= I915_ENGINE_SUPPORTS_STATS; - if (!intel_vgpu_active(engine->i915)) { - engine->flags |= I915_ENGINE_HAS_SEMAPHORES; - if (can_preempt(engine)) { - engine->flags |= I915_ENGINE_HAS_PREEMPTION; - if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) - engine->flags |= I915_ENGINE_HAS_TIMESLICES; - } - } - - if (intel_engine_has_preemption(engine)) - engine->emit_bb_start = gen8_emit_bb_start; - else - engine->emit_bb_start = gen8_emit_bb_start_noarb; } static void execlists_shutdown(struct intel_engine_cs *engine) @@ -3129,6 +3145,14 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->cops = &execlists_context_ops; engine->request_alloc = execlists_request_alloc; + engine->reset.prepare = execlists_reset_prepare; + engine->reset.rewind = execlists_reset_rewind; + engine->reset.cancel = execlists_reset_cancel; + engine->reset.finish = execlists_reset_finish; + + engine->park = execlists_park; + engine->unpark = NULL; + engine->emit_flush = gen8_emit_flush_xcs; engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs; @@ -3149,6 +3173,22 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) * until a more refined solution exists. */ } + intel_engine_set_irq_handler(engine, execlists_irq_handler); + + engine->flags |= I915_ENGINE_SUPPORTS_STATS; + if (!intel_vgpu_active(engine->i915)) { + engine->flags |= I915_ENGINE_HAS_SEMAPHORES; + if (can_preempt(engine)) { + engine->flags |= I915_ENGINE_HAS_PREEMPTION; + if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + engine->flags |= I915_ENGINE_HAS_TIMESLICES; + } + } + + if (intel_engine_has_preemption(engine)) + engine->emit_bb_start = gen8_emit_bb_start; + else + engine->emit_bb_start = gen8_emit_bb_start_noarb; } static void logical_ring_default_irqs(struct intel_engine_cs *engine) @@ -3884,13 +3924,6 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, spin_unlock_irqrestore(&engine->active.lock, flags); } -bool -intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine) -{ - return engine->set_default_submission == - execlists_set_default_submission; -} - #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_execlists.c" #endif diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.h b/drivers/gpu/drm/i915/gt/intel_execlists_submission.h index fd61dae820e9..4ca9b475e252 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.h +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.h @@ -43,7 +43,4 @@ int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, const struct intel_engine_cs *master, const struct intel_engine_cs *sibling); -bool -intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine); - #endif /* __INTEL_EXECLISTS_SUBMISSION_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 38742bf33fa3..35069ca5d7de 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -658,7 +658,7 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) goto err_ppgtt; i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); - err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); i915_gem_object_unlock(ppgtt->vm.scratch[0]); if (err) goto err_stash; @@ -907,9 +907,11 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.insert_entries = gen8_ggtt_insert_entries; - /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ - if (intel_ggtt_update_needs_vtd_wa(i915) || - IS_CHERRYVIEW(i915) /* fails with concurrent use/update */) { + /* + * Serialize GTT updates with aperture access on BXT if VT-d is on, + * and always on CHV. + */ + if (intel_vm_no_concurrent_access_wa(i915)) { ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; ggtt->vm.bind_async_flags = diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index 8a322594210c..7bf84cd21543 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -653,8 +653,8 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) * banks of memory are paired and unswizzled on the * uneven portion, so leave that as unknown. */ - if (intel_uncore_read16(uncore, C0DRB3) == - intel_uncore_read16(uncore, C1DRB3)) { + if (intel_uncore_read16(uncore, C0DRB3_BW) == + intel_uncore_read16(uncore, C1DRB3_BW)) { swizzle_x = I915_BIT_6_SWIZZLE_9_10; swizzle_y = I915_BIT_6_SWIZZLE_9; } @@ -867,7 +867,7 @@ void intel_ggtt_init_fences(struct i915_ggtt *ggtt) for (i = 0; i < num_fences; i++) { struct i915_fence_reg *fence = &ggtt->fence_regs[i]; - i915_active_init(&fence->active, NULL, NULL); + i915_active_init(&fence->active, NULL, NULL, 0); fence->ggtt = ggtt; fence->id = i; list_add_tail(&fence->link, &ggtt->fence_list); diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h index 14e2ffb6c0e5..2694dbb9967e 100644 --- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h +++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT*/ /* - * Copyright � 2003-2018 Intel Corporation + * Copyright © 2003-2018 Intel Corporation */ #ifndef _INTEL_GPU_COMMANDS_H_ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index c59468107598..aa0a59c5b614 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -98,7 +98,6 @@ static void pool_free_work(struct work_struct *wrk) round_jiffies_up_relative(HZ)); } -__i915_active_call static void pool_retire(struct i915_active *ref) { struct intel_gt_buffer_pool_node *node = @@ -154,7 +153,7 @@ node_create(struct intel_gt_buffer_pool *pool, size_t sz, node->age = 0; node->pool = pool; node->pinned = false; - i915_active_init(&node->active, NULL, pool_retire); + i915_active_init(&node->active, NULL, pool_retire, 0); obj = i915_gem_object_create_internal(gt->i915, sz); if (IS_ERR(obj)) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index 9fc6c912a4e5..d29126c458ba 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -20,48 +20,6 @@ static void guc_irq_handler(struct intel_guc *guc, u16 iir) intel_guc_to_host_event_handler(guc); } -static void -cs_irq_handler(struct intel_engine_cs *engine, u32 iir) -{ - bool tasklet = false; - - if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) { - u32 eir; - - /* Upper 16b are the enabling mask, rsvd for internal errors */ - eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0); - ENGINE_TRACE(engine, "CS error: %x\n", eir); - - /* Disable the error interrupt until after the reset */ - if (likely(eir)) { - ENGINE_WRITE(engine, RING_EMR, ~0u); - ENGINE_WRITE(engine, RING_EIR, eir); - WRITE_ONCE(engine->execlists.error_interrupt, eir); - tasklet = true; - } - } - - if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) { - WRITE_ONCE(engine->execlists.yield, - ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI)); - ENGINE_TRACE(engine, "semaphore yield: %08x\n", - engine->execlists.yield); - if (del_timer(&engine->execlists.timer)) - tasklet = true; - } - - if (iir & GT_CONTEXT_SWITCH_INTERRUPT) - tasklet = true; - - if (iir & GT_RENDER_USER_INTERRUPT) { - intel_engine_signal_breadcrumbs(engine); - tasklet |= intel_engine_needs_breadcrumb_tasklet(engine); - } - - if (tasklet) - tasklet_hi_schedule(&engine->execlists.tasklet); -} - static u32 gen11_gt_engine_identity(struct intel_gt *gt, const unsigned int bank, const unsigned int bit) @@ -122,7 +80,7 @@ gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, engine = NULL; if (likely(engine)) - return cs_irq_handler(engine, iir); + return intel_engine_cs_irq(engine, iir); WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", class, instance); @@ -275,9 +233,12 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) { if (gt_iir & GT_RENDER_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], + gt_iir); + if (gt_iir & ILK_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], + gt_iir); } static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) @@ -301,11 +262,16 @@ static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) { if (gt_iir & GT_RENDER_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], + gt_iir); + if (gt_iir & GT_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], + gt_iir >> 12); + if (gt_iir & GT_BLT_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[COPY_ENGINE_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0], + gt_iir >> 22); if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | GT_BSD_CS_ERROR_INTERRUPT | @@ -324,10 +290,10 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { iir = raw_reg_read(regs, GEN8_GT_IIR(0)); if (likely(iir)) { - cs_irq_handler(gt->engine_class[RENDER_CLASS][0], - iir >> GEN8_RCS_IRQ_SHIFT); - cs_irq_handler(gt->engine_class[COPY_ENGINE_CLASS][0], - iir >> GEN8_BCS_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], + iir >> GEN8_RCS_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0], + iir >> GEN8_BCS_IRQ_SHIFT); raw_reg_write(regs, GEN8_GT_IIR(0), iir); } } @@ -335,10 +301,10 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { iir = raw_reg_read(regs, GEN8_GT_IIR(1)); if (likely(iir)) { - cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][0], - iir >> GEN8_VCS0_IRQ_SHIFT); - cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][1], - iir >> GEN8_VCS1_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], + iir >> GEN8_VCS0_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1], + iir >> GEN8_VCS1_IRQ_SHIFT); raw_reg_write(regs, GEN8_GT_IIR(1), iir); } } @@ -346,8 +312,8 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) if (master_ctl & GEN8_GT_VECS_IRQ) { iir = raw_reg_read(regs, GEN8_GT_IIR(3)); if (likely(iir)) { - cs_irq_handler(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0], - iir >> GEN8_VECS_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0], + iir >> GEN8_VECS_IRQ_SHIFT); raw_reg_write(regs, GEN8_GT_IIR(3), iir); } } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.h b/drivers/gpu/drm/i915/gt/intel_gt_irq.h index f667e976fb2b..41cad38668c5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.h @@ -8,6 +8,8 @@ #include <linux/types.h> +#include "intel_engine_types.h" + struct intel_gt; #define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \ @@ -39,4 +41,25 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl); void gen8_gt_irq_reset(struct intel_gt *gt); void gen8_gt_irq_postinstall(struct intel_gt *gt); +static inline void intel_engine_cs_irq(struct intel_engine_cs *engine, u16 iir) +{ + if (iir) + engine->irq_handler(engine, iir); +} + +static inline void +intel_engine_set_irq_handler(struct intel_engine_cs *engine, + void (*fn)(struct intel_engine_cs *engine, + u16 iir)) +{ + /* + * As the interrupt is live as allocate and setup the engines, + * err on the side of caution and apply barriers to updating + * the irq handler callback. This assures that when we do use + * the engine, we will receive interrupts only to ourselves, + * and not lose any. + */ + smp_store_mb(engine->irq_handler, fn); +} + #endif /* INTEL_GT_IRQ_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index 0caf6ca0a784..fecfacf551d5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -31,6 +31,12 @@ struct i915_ggtt; struct intel_engine_cs; struct intel_uncore; +enum intel_submission_method { + INTEL_SUBMISSION_RING, + INTEL_SUBMISSION_ELSP, + INTEL_SUBMISSION_GUC, +}; + struct intel_gt { struct drm_i915_private *i915; struct intel_uncore *uncore; @@ -118,6 +124,7 @@ struct intel_gt { struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct intel_engine_cs *engine_class[MAX_ENGINE_CLASS + 1] [MAX_ENGINE_INSTANCE + 1]; + enum intel_submission_method submission_method; /* * Default address space (either GGTT or ppGTT depending on arch). diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 941f8af016d6..9b98f9d9faa3 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -7,10 +7,26 @@ #include <linux/fault-inject.h> +#include "gem/i915_gem_lmem.h" #include "i915_trace.h" #include "intel_gt.h" #include "intel_gtt.h" +struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz) +{ + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_create_lmem(vm->i915, sz, 0); + /* + * Ensure all paging structures for this vm share the same dma-resv + * object underneath, with the idea that one object_lock() will lock + * them all at once. + */ + if (!IS_ERR(obj)) + obj->base.resv = &vm->resv; + return obj; +} + struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz) { struct drm_i915_gem_object *obj; @@ -19,33 +35,39 @@ struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz) i915_gem_shrink_all(vm->i915); obj = i915_gem_object_create_internal(vm->i915, sz); - /* ensure all dma objects have the same reservation class */ + /* + * Ensure all paging structures for this vm share the same dma-resv + * object underneath, with the idea that one object_lock() will lock + * them all at once. + */ if (!IS_ERR(obj)) obj->base.resv = &vm->resv; return obj; } -int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj) +int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj) { - int err; + enum i915_map_type type; + void *vaddr; - i915_gem_object_lock(obj, NULL); - err = i915_gem_object_pin_pages(obj); - i915_gem_object_unlock(obj); - if (err) - return err; + type = i915_coherent_map_type(vm->i915, obj, true); + vaddr = i915_gem_object_pin_map_unlocked(obj, type); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); i915_gem_object_make_unshrinkable(obj); return 0; } -int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj) +int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj) { - int err; + enum i915_map_type type; + void *vaddr; - err = i915_gem_object_pin_pages(obj); - if (err) - return err; + type = i915_coherent_map_type(vm->i915, obj, true); + vaddr = i915_gem_object_pin_map(obj, type); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); i915_gem_object_make_unshrinkable(obj); return 0; @@ -132,7 +154,22 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) */ mutex_init(&vm->mutex); lockdep_set_subclass(&vm->mutex, subclass); - i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); + + if (!intel_vm_no_concurrent_access_wa(vm->i915)) { + i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); + } else { + /* + * CHV + BXT VTD workaround use stop_machine(), + * which is allowed to allocate memory. This means &vm->mutex + * is the outer lock, and in theory we can allocate memory inside + * it through stop_machine(). + * + * Add the annotation for this, we use trylock in shrinker. + */ + mutex_acquire(&vm->mutex.dep_map, 0, 0, _THIS_IP_); + might_alloc(GFP_KERNEL); + mutex_release(&vm->mutex.dep_map, _THIS_IP_); + } dma_resv_init(&vm->resv); GEM_BUG_ON(!vm->total); @@ -155,6 +192,14 @@ void clear_pages(struct i915_vma *vma) memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); } +void *__px_vaddr(struct drm_i915_gem_object *p) +{ + enum i915_map_type type; + + GEM_BUG_ON(!i915_gem_object_has_pages(p)); + return page_unpack_bits(p->mm.mapping, &type); +} + dma_addr_t __px_dma(struct drm_i915_gem_object *p) { GEM_BUG_ON(!i915_gem_object_has_pages(p)); @@ -170,32 +215,22 @@ struct page *__px_page(struct drm_i915_gem_object *p) void fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count) { - struct page *page = __px_page(p); - void *vaddr; + void *vaddr = __px_vaddr(p); - vaddr = kmap(page); memset64(vaddr, val, count); clflush_cache_range(vaddr, PAGE_SIZE); - kunmap(page); } static void poison_scratch_page(struct drm_i915_gem_object *scratch) { - struct sgt_iter sgt; - struct page *page; + void *vaddr = __px_vaddr(scratch); u8 val; val = 0; if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) val = POISON_FREE; - for_each_sgt_page(page, sgt, scratch->mm.pages) { - void *vaddr; - - vaddr = kmap(page); - memset(vaddr, val, PAGE_SIZE); - kunmap(page); - } + memset(vaddr, val, scratch->base.size); } int setup_scratch_page(struct i915_address_space *vm) @@ -225,7 +260,7 @@ int setup_scratch_page(struct i915_address_space *vm) if (IS_ERR(obj)) goto skip; - if (pin_pt_dma(vm, obj)) + if (map_pt_dma(vm, obj)) goto skip_obj; /* We need a single contiguous page for our scratch */ diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index 79f565aeb8c0..ca00b45827b7 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -180,6 +180,9 @@ struct page *__px_page(struct drm_i915_gem_object *p); dma_addr_t __px_dma(struct drm_i915_gem_object *p); #define px_dma(px) (__px_dma(px_base(px))) +void *__px_vaddr(struct drm_i915_gem_object *p); +#define px_vaddr(px) (__px_vaddr(px_base(px))) + #define px_pt(px) \ __px_choose_expr(px, struct i915_page_table *, __x, \ __px_choose_expr(px, struct i915_page_directory *, &__x->pt, \ @@ -516,8 +519,6 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt); void i915_ggtt_suspend(struct i915_ggtt *gtt); void i915_ggtt_resume(struct i915_ggtt *ggtt); -#define kmap_atomic_px(px) kmap_atomic(__px_page(px_base(px))) - void fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count); @@ -531,12 +532,13 @@ int setup_scratch_page(struct i915_address_space *vm); void free_scratch(struct i915_address_space *vm); struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); +struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz); struct i915_page_table *alloc_pt(struct i915_address_space *vm); struct i915_page_directory *alloc_pd(struct i915_address_space *vm); struct i915_page_directory *__alloc_pd(int npde); -int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); -int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj); +int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); +int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj); void free_px(struct i915_address_space *vm, struct i915_page_table *pt, int lvl); @@ -583,7 +585,7 @@ void setup_private_pat(struct intel_uncore *uncore); int i915_vm_alloc_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, u64 size); -int i915_vm_pin_pt_stash(struct i915_address_space *vm, +int i915_vm_map_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash); void i915_vm_free_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index e86897cde984..aafe2a4df496 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -903,7 +903,9 @@ lrc_pre_pin(struct intel_context *ce, GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); *vaddr = i915_gem_object_pin_map(ce->state->obj, - i915_coherent_map_type(ce->engine->i915) | + i915_coherent_map_type(ce->engine->i915, + ce->state->obj, + false) | I915_MAP_OVERRIDE); return PTR_ERR_OR_ZERO(*vaddr); diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index 014ae8ac4480..4e3d80c2295c 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -87,11 +87,10 @@ write_dma_entry(struct drm_i915_gem_object * const pdma, const unsigned short idx, const u64 encoded_entry) { - u64 * const vaddr = kmap_atomic(__px_page(pdma)); + u64 * const vaddr = __px_vaddr(pdma); vaddr[idx] = encoded_entry; clflush_cache_range(&vaddr[idx], sizeof(u64)); - kunmap_atomic(vaddr); } void @@ -258,7 +257,7 @@ int i915_vm_alloc_pt_stash(struct i915_address_space *vm, return 0; } -int i915_vm_pin_pt_stash(struct i915_address_space *vm, +int i915_vm_map_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash) { struct i915_page_table *pt; @@ -266,7 +265,7 @@ int i915_vm_pin_pt_stash(struct i915_address_space *vm, for (n = 0; n < ARRAY_SIZE(stash->pt); n++) { for (pt = stash->pt[n]; pt; pt = pt->stash) { - err = pin_pt_dma_locked(vm, pt->base); + err = map_pt_dma_locked(vm, pt->base); if (err) return err; } diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index a377c4588aaa..8091846d955b 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -338,15 +338,69 @@ static int gen6_reset_engines(struct intel_gt *gt, return gen6_hw_domain_reset(gt, hw_mask); } -static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask) +static struct intel_engine_cs *find_sfc_paired_vecs_engine(struct intel_engine_cs *engine) +{ + int vecs_id; + + GEM_BUG_ON(engine->class != VIDEO_DECODE_CLASS); + + vecs_id = _VECS((engine->instance) / 2); + + return engine->gt->engine[vecs_id]; +} + +struct sfc_lock_data { + i915_reg_t lock_reg; + i915_reg_t ack_reg; + i915_reg_t usage_reg; + u32 lock_bit; + u32 ack_bit; + u32 usage_bit; + u32 reset_bit; +}; + +static void get_sfc_forced_lock_data(struct intel_engine_cs *engine, + struct sfc_lock_data *sfc_lock) +{ + switch (engine->class) { + default: + MISSING_CASE(engine->class); + fallthrough; + case VIDEO_DECODE_CLASS: + sfc_lock->lock_reg = GEN11_VCS_SFC_FORCED_LOCK(engine); + sfc_lock->lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; + + sfc_lock->ack_reg = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_lock->ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; + + sfc_lock->usage_reg = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_lock->usage_bit = GEN11_VCS_SFC_USAGE_BIT; + sfc_lock->reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); + + break; + case VIDEO_ENHANCEMENT_CLASS: + sfc_lock->lock_reg = GEN11_VECS_SFC_FORCED_LOCK(engine); + sfc_lock->lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; + + sfc_lock->ack_reg = GEN11_VECS_SFC_LOCK_ACK(engine); + sfc_lock->ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; + + sfc_lock->usage_reg = GEN11_VECS_SFC_USAGE(engine); + sfc_lock->usage_bit = GEN11_VECS_SFC_USAGE_BIT; + sfc_lock->reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); + + break; + } +} + +static int gen11_lock_sfc(struct intel_engine_cs *engine, + u32 *reset_mask, + u32 *unlock_mask) { struct intel_uncore *uncore = engine->uncore; u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access; - i915_reg_t sfc_forced_lock, sfc_forced_lock_ack; - u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit; - i915_reg_t sfc_usage; - u32 sfc_usage_bit; - u32 sfc_reset_bit; + struct sfc_lock_data sfc_lock; + bool lock_obtained, lock_to_other = false; int ret; switch (engine->class) { @@ -354,53 +408,72 @@ static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask) if ((BIT(engine->instance) & vdbox_sfc_access) == 0) return 0; - sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; - - sfc_forced_lock_ack = GEN11_VCS_SFC_LOCK_STATUS(engine); - sfc_forced_lock_ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; + fallthrough; + case VIDEO_ENHANCEMENT_CLASS: + get_sfc_forced_lock_data(engine, &sfc_lock); - sfc_usage = GEN11_VCS_SFC_LOCK_STATUS(engine); - sfc_usage_bit = GEN11_VCS_SFC_USAGE_BIT; - sfc_reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); break; + default: + return 0; + } - case VIDEO_ENHANCEMENT_CLASS: - sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; + if (!(intel_uncore_read_fw(uncore, sfc_lock.usage_reg) & sfc_lock.usage_bit)) { + struct intel_engine_cs *paired_vecs; - sfc_forced_lock_ack = GEN11_VECS_SFC_LOCK_ACK(engine); - sfc_forced_lock_ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; + if (engine->class != VIDEO_DECODE_CLASS || + !IS_GEN(engine->i915, 12)) + return 0; - sfc_usage = GEN11_VECS_SFC_USAGE(engine); - sfc_usage_bit = GEN11_VECS_SFC_USAGE_BIT; - sfc_reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); - break; + /* + * Wa_14010733141 + * + * If the VCS-MFX isn't using the SFC, we also need to check + * whether VCS-HCP is using it. If so, we need to issue a *VE* + * forced lock on the VE engine that shares the same SFC. + */ + if (!(intel_uncore_read_fw(uncore, + GEN12_HCP_SFC_LOCK_STATUS(engine)) & + GEN12_HCP_SFC_USAGE_BIT)) + return 0; - default: - return 0; + paired_vecs = find_sfc_paired_vecs_engine(engine); + get_sfc_forced_lock_data(paired_vecs, &sfc_lock); + lock_to_other = true; + *unlock_mask |= paired_vecs->mask; + } else { + *unlock_mask |= engine->mask; } /* - * If the engine is using a SFC, tell the engine that a software reset + * If the engine is using an SFC, tell the engine that a software reset * is going to happen. The engine will then try to force lock the SFC. * If SFC ends up being locked to the engine we want to reset, we have * to reset it as well (we will unlock it once the reset sequence is * completed). */ - if (!(intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit)) - return 0; - - rmw_set_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit); + rmw_set_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); ret = __intel_wait_for_register_fw(uncore, - sfc_forced_lock_ack, - sfc_forced_lock_ack_bit, - sfc_forced_lock_ack_bit, + sfc_lock.ack_reg, + sfc_lock.ack_bit, + sfc_lock.ack_bit, 1000, 0, NULL); - /* Was the SFC released while we were trying to lock it? */ - if (!(intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit)) + /* + * Was the SFC released while we were trying to lock it? + * + * We should reset both the engine and the SFC if: + * - We were locking the SFC to this engine and the lock succeeded + * OR + * - We were locking the SFC to a different engine (Wa_14010733141) + * but the SFC was released before the lock was obtained. + * + * Otherwise we need only reset the engine by itself and we can + * leave the SFC alone. + */ + lock_obtained = (intel_uncore_read_fw(uncore, sfc_lock.usage_reg) & + sfc_lock.usage_bit) != 0; + if (lock_obtained == lock_to_other) return 0; if (ret) { @@ -408,7 +481,7 @@ static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask) return ret; } - *hw_mask |= sfc_reset_bit; + *reset_mask |= sfc_lock.reset_bit; return 0; } @@ -416,28 +489,19 @@ static void gen11_unlock_sfc(struct intel_engine_cs *engine) { struct intel_uncore *uncore = engine->uncore; u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access; - i915_reg_t sfc_forced_lock; - u32 sfc_forced_lock_bit; - - switch (engine->class) { - case VIDEO_DECODE_CLASS: - if ((BIT(engine->instance) & vdbox_sfc_access) == 0) - return; + struct sfc_lock_data sfc_lock = {}; - sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; - break; - - case VIDEO_ENHANCEMENT_CLASS: - sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; - break; + if (engine->class != VIDEO_DECODE_CLASS && + engine->class != VIDEO_ENHANCEMENT_CLASS) + return; - default: + if (engine->class == VIDEO_DECODE_CLASS && + (BIT(engine->instance) & vdbox_sfc_access) == 0) return; - } - rmw_clear_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit); + get_sfc_forced_lock_data(engine, &sfc_lock); + + rmw_clear_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); } static int gen11_reset_engines(struct intel_gt *gt, @@ -456,23 +520,23 @@ static int gen11_reset_engines(struct intel_gt *gt, }; struct intel_engine_cs *engine; intel_engine_mask_t tmp; - u32 hw_mask; + u32 reset_mask, unlock_mask = 0; int ret; if (engine_mask == ALL_ENGINES) { - hw_mask = GEN11_GRDOM_FULL; + reset_mask = GEN11_GRDOM_FULL; } else { - hw_mask = 0; + reset_mask = 0; for_each_engine_masked(engine, gt, engine_mask, tmp) { GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask)); - hw_mask |= hw_engine_mask[engine->id]; - ret = gen11_lock_sfc(engine, &hw_mask); + reset_mask |= hw_engine_mask[engine->id]; + ret = gen11_lock_sfc(engine, &reset_mask, &unlock_mask); if (ret) goto sfc_unlock; } } - ret = gen6_hw_domain_reset(gt, hw_mask); + ret = gen6_hw_domain_reset(gt, reset_mask); sfc_unlock: /* @@ -480,10 +544,14 @@ sfc_unlock: * gen11_lock_sfc to make sure that we clean properly if something * wrong happened during the lock (e.g. lock acquired after timeout * expiration). + * + * Due to Wa_14010733141, we may have locked an SFC to an engine that + * wasn't being reset. So instead of calling gen11_unlock_sfc() + * on engine_mask, we instead call it on the mask of engines that our + * gen11_lock_sfc() calls told us actually had locks attempted. */ - if (engine_mask != ALL_ENGINES) - for_each_engine_masked(engine, gt, engine_mask, tmp) - gen11_unlock_sfc(engine); + for_each_engine_masked(engine, gt, unlock_mask, tmp) + gen11_unlock_sfc(engine); return ret; } @@ -1118,7 +1186,6 @@ static int intel_gt_reset_engine(struct intel_engine_cs *engine) int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) { struct intel_gt *gt = engine->gt; - bool uses_guc = intel_engine_in_guc_submission_mode(engine); int ret; ENGINE_TRACE(engine, "flags=%lx\n", gt->reset.flags); @@ -1134,10 +1201,10 @@ int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) "Resetting %s for %s\n", engine->name, msg); atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class]); - if (!uses_guc) - ret = intel_gt_reset_engine(engine); - else + if (intel_engine_uses_guc(engine)) ret = intel_guc_reset_engine(&engine->gt->uc.guc, engine); + else + ret = intel_gt_reset_engine(engine); if (ret) { /* If we fail here, we expect to fallback to a global reset */ ENGINE_TRACE(engine, "Failed to reset, err: %d\n", ret); diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c index aee0a77c77e0..7c4d5158e03b 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.c +++ b/drivers/gpu/drm/i915/gt/intel_ring.c @@ -51,11 +51,14 @@ int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww) if (unlikely(ret)) goto err_unpin; - if (i915_vma_is_map_and_fenceable(vma)) + if (i915_vma_is_map_and_fenceable(vma)) { addr = (void __force *)i915_vma_pin_iomap(vma); - else - addr = i915_gem_object_pin_map(vma->obj, - i915_coherent_map_type(vma->vm->i915)); + } else { + int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false); + + addr = i915_gem_object_pin_map(vma->obj, type); + } + if (IS_ERR(addr)) { ret = PTR_ERR(addr); goto err_ring; diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 9585546556ee..2b6dffcc2262 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -12,6 +12,7 @@ #include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_gt.h" +#include "intel_gt_irq.h" #include "intel_reset.h" #include "intel_ring.h" #include "shmem_utils.h" @@ -989,14 +990,10 @@ static void gen6_bsd_submit_request(struct i915_request *request) static void i9xx_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = i9xx_submit_request; - - engine->park = NULL; - engine->unpark = NULL; } static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) { - i9xx_set_default_submission(engine); engine->submit_request = gen6_bsd_submit_request; } @@ -1021,10 +1018,17 @@ static void ring_release(struct intel_engine_cs *engine) intel_timeline_put(engine->legacy.timeline); } +static void irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + intel_engine_signal_breadcrumbs(engine); +} + static void setup_irq(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; + intel_engine_set_irq_handler(engine, irq_handler); + if (INTEL_GEN(i915) >= 6) { engine->irq_enable = gen6_irq_enable; engine->irq_disable = gen6_irq_disable; diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 405d814e9040..97cab1b99871 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -1774,7 +1774,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) return; if (pm_iir & PM_VEBOX_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine[VECS0]); + intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c index f19cf6d2fa85..c4a126c8caef 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.c +++ b/drivers/gpu/drm/i915/gt/intel_timeline.c @@ -32,7 +32,6 @@ static struct i915_vma *hwsp_alloc(struct intel_gt *gt) return vma; } -__i915_active_call static void __timeline_retire(struct i915_active *active) { struct intel_timeline *tl = @@ -104,7 +103,8 @@ static int intel_timeline_init(struct intel_timeline *timeline, INIT_LIST_HEAD(&timeline->requests); i915_syncmap_init(&timeline->sync); - i915_active_init(&timeline->active, __timeline_active, __timeline_retire); + i915_active_init(&timeline->active, __timeline_active, + __timeline_retire, 0); return 0; } diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 2c6f7217469f..62cb9ee5bfc3 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -607,9 +607,38 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); } +/* + * These settings aren't actually workarounds, but general tuning settings that + * need to be programmed on several platforms. + */ +static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + /* + * Although some platforms refer to it as Wa_1604555607, we need to + * program it even on those that don't explicitly list that + * workaround. + * + * Note that the programming of this register is further modified + * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. + * Wa_1608008084 tells us the FF_MODE2 register will return the wrong + * value when read. The default value for this register is zero for all + * fields and there are no bit masks. So instead of doing a RMW we + * should just write TDS timer value. For the same reason read + * verification is ignored. + */ + wa_add(wal, + FF_MODE2, + FF_MODE2_TDS_TIMER_MASK, + FF_MODE2_TDS_TIMER_128, + 0); +} + static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { + gen12_ctx_gt_tuning_init(engine, wal); + /* * Wa_1409142259:tgl * Wa_1409347922:tgl @@ -628,27 +657,17 @@ static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_field_set(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK, GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); -} - -static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, - struct i915_wa_list *wal) -{ - gen12_ctx_workarounds_init(engine, wal); /* - * Wa_1604555607:tgl,rkl + * Wa_16011163337 * - * Note that the implementation of this workaround is further modified - * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. - * FF_MODE2 register will return the wrong value when read. The default - * value for this register is zero for all fields and there are no bit - * masks. So instead of doing a RMW we should just write the GS Timer - * and TDS timer values for Wa_1604555607 and Wa_16011163337. + * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due + * to Wa_1608008084. */ wa_add(wal, FF_MODE2, - FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK, - FF_MODE2_GS_TIMER_224 | FF_MODE2_TDS_TIMER_128, + FF_MODE2_GS_TIMER_MASK, + FF_MODE2_GS_TIMER_224, 0); } @@ -664,16 +683,6 @@ static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, /* Wa_22010493298 */ wa_masked_en(wal, HIZ_CHICKEN, DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE); - - /* - * Wa_16011163337 - * - * Like in tgl_ctx_workarounds_init(), read verification is ignored due - * to Wa_1608008084. - */ - wa_add(wal, - FF_MODE2, - FF_MODE2_GS_TIMER_MASK, FF_MODE2_GS_TIMER_224, 0); } static void @@ -690,9 +699,6 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, if (IS_DG1(i915)) dg1_ctx_workarounds_init(engine, wal); - else if (IS_ALDERLAKE_S(i915) || IS_ROCKETLAKE(i915) || - IS_TIGERLAKE(i915)) - tgl_ctx_workarounds_init(engine, wal); else if (IS_GEN(i915, 12)) gen12_ctx_workarounds_init(engine, wal); else if (IS_GEN(i915, 11)) @@ -1078,11 +1084,37 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); } +/* + * Though there are per-engine instances of these registers, + * they retain their value through engine resets and should + * only be provided on the GT workaround list rather than + * the engine-specific workaround list. + */ +static void +wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + struct intel_engine_cs *engine; + struct intel_gt *gt = &i915->gt; + int id; + + for_each_engine(engine, gt, id) { + if (engine->class != VIDEO_DECODE_CLASS || + (engine->instance % 2)) + continue; + + wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), + IECPUNIT_CLKGATE_DIS); + } +} + static void gen12_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { wa_init_mcr(i915, wal); + + /* Wa_14011060649:tgl,rkl,dg1,adls */ + wa_14011060649(i915, wal); } static void @@ -1755,11 +1787,10 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN7_FF_THREAD_MODE, GEN12_FF_TESSELATION_DOP_GATE_DISABLE); - /* Wa_22010271021:ehl */ - if (IS_JSL_EHL(i915)) - wa_masked_en(wal, - GEN9_CS_DEBUG_MODE1, - FF_DOP_CLOCK_GATE_DISABLE); + /* Wa_22010271021 */ + wa_masked_en(wal, + GEN9_CS_DEBUG_MODE1, + FF_DOP_CLOCK_GATE_DISABLE); } if (IS_GEN_RANGE(i915, 9, 12)) { @@ -1828,9 +1859,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) CACHE_MODE_0_GEN7, /* enable HiZ Raw Stall Optimization */ HIZ_RAW_STALL_OPT_DISABLE); - - /* WaDisable4x2SubspanOptimization:hsw */ - wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); } if (IS_VALLEYVIEW(i915)) { diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index e1ba03b93ffa..32589c6625e1 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -55,7 +55,7 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine) kfree(ring); return NULL; } - i915_active_init(&ring->vma->active, NULL, NULL); + i915_active_init(&ring->vma->active, NULL, NULL, 0); __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(ring->vma)); __set_bit(DRM_MM_NODE_ALLOCATED_BIT, &ring->vma->node.flags); ring->vma->node.size = sz; diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c index b9bdd1d23243..26685b927169 100644 --- a/drivers/gpu/drm/i915/gt/selftest_context.c +++ b/drivers/gpu/drm/i915/gt/selftest_context.c @@ -88,7 +88,8 @@ static int __live_context_size(struct intel_engine_cs *engine) goto err; vaddr = i915_gem_object_pin_map_unlocked(ce->state->obj, - i915_coherent_map_type(engine->i915)); + i915_coherent_map_type(engine->i915, + ce->state->obj, false)); if (IS_ERR(vaddr)) { err = PTR_ERR(vaddr); intel_context_unpin(ce); diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index b2c369317bf1..4896e4ccad50 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -77,7 +77,7 @@ static struct pulse *pulse_create(void) return p; kref_init(&p->kref); - i915_active_init(&p->active, pulse_active, pulse_retire); + i915_active_init(&p->active, pulse_active, pulse_retire, 0); return p; } diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c index 1081cd36a2bd..1f93591a8c69 100644 --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c @@ -4716,7 +4716,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_virtual_reset), }; - if (!HAS_EXECLISTS(i915)) + if (i915->gt.submission_method != INTEL_SUBMISSION_ELSP) return 0; if (intel_gt_is_wedged(&i915->gt)) diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 746985971c3a..5b63d4df8c93 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -69,7 +69,7 @@ static int hang_init(struct hang *h, struct intel_gt *gt) h->seqno = memset(vaddr, 0xff, PAGE_SIZE); vaddr = i915_gem_object_pin_map_unlocked(h->obj, - i915_coherent_map_type(gt->i915)); + i915_coherent_map_type(gt->i915, h->obj, false)); if (IS_ERR(vaddr)) { err = PTR_ERR(vaddr); goto err_unpin_hws; @@ -130,7 +130,7 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine) return ERR_CAST(obj); } - vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915)); + vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false)); if (IS_ERR(vaddr)) { i915_gem_object_put(obj); i915_vm_put(vm); diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 85e7df6a5123..d8f6623524e8 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -1221,7 +1221,9 @@ static int compare_isolation(struct intel_engine_cs *engine, } lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, - i915_coherent_map_type(engine->i915)); + i915_coherent_map_type(engine->i915, + ce->state->obj, + false)); if (IS_ERR(lrc)) { err = PTR_ERR(lrc); goto err_B1; diff --git a/drivers/gpu/drm/i915/gt/selftest_rc6.c b/drivers/gpu/drm/i915/gt/selftest_rc6.c index f097e420ac45..710f825f6e5a 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rc6.c +++ b/drivers/gpu/drm/i915/gt/selftest_rc6.c @@ -34,6 +34,7 @@ int live_rc6_manual(void *arg) struct intel_rc6 *rc6 = >->rc6; u64 rc0_power, rc6_power; intel_wakeref_t wakeref; + bool has_power; ktime_t dt; u64 res[2]; int err = 0; @@ -50,6 +51,7 @@ int live_rc6_manual(void *arg) if (IS_VALLEYVIEW(gt->i915) || IS_CHERRYVIEW(gt->i915)) return 0; + has_power = librapl_supported(gt->i915); wakeref = intel_runtime_pm_get(gt->uncore->rpm); /* Force RC6 off for starters */ @@ -71,11 +73,14 @@ int live_rc6_manual(void *arg) goto out_unlock; } - rc0_power = div64_u64(NSEC_PER_SEC * rc0_power, ktime_to_ns(dt)); - if (!rc0_power) { - pr_err("No power measured while in RC0\n"); - err = -EINVAL; - goto out_unlock; + if (has_power) { + rc0_power = div64_u64(NSEC_PER_SEC * rc0_power, + ktime_to_ns(dt)); + if (!rc0_power) { + pr_err("No power measured while in RC0\n"); + err = -EINVAL; + goto out_unlock; + } } /* Manually enter RC6 */ @@ -97,13 +102,16 @@ int live_rc6_manual(void *arg) err = -EINVAL; } - rc6_power = div64_u64(NSEC_PER_SEC * rc6_power, ktime_to_ns(dt)); - pr_info("GPU consumed %llduW in RC0 and %llduW in RC6\n", - rc0_power, rc6_power); - if (2 * rc6_power > rc0_power) { - pr_err("GPU leaked energy while in RC6!\n"); - err = -EINVAL; - goto out_unlock; + if (has_power) { + rc6_power = div64_u64(NSEC_PER_SEC * rc6_power, + ktime_to_ns(dt)); + pr_info("GPU consumed %llduW in RC0 and %llduW in RC6\n", + rc0_power, rc6_power); + if (2 * rc6_power > rc0_power) { + pr_err("GPU leaked energy while in RC6!\n"); + err = -EINVAL; + goto out_unlock; + } } /* Restore what should have been the original state! */ diff --git a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c index 99609271c3a7..c12e74171b63 100644 --- a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c @@ -291,7 +291,7 @@ int intel_ring_submission_live_selftests(struct drm_i915_private *i915) SUBTEST(live_ctx_switch_wa), }; - if (HAS_EXECLISTS(i915)) + if (i915->gt.submission_method > INTEL_SUBMISSION_RING) return 0; return intel_gt_live_subtests(tests, &i915->gt); diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 967641fee42a..3ca1bd5793c3 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -606,7 +606,7 @@ int live_rps_frequency_cs(void *arg) int err = 0; /* - * The premise is that the GPU does change freqency at our behest. + * The premise is that the GPU does change frequency at our behest. * Let's check there is a correspondence between the requested * frequency, the actual frequency, and the observed clock rate. */ @@ -747,7 +747,7 @@ int live_rps_frequency_srm(void *arg) int err = 0; /* - * The premise is that the GPU does change freqency at our behest. + * The premise is that the GPU does change frequency at our behest. * Let's check there is a correspondence between the requested * frequency, the actual frequency, and the observed clock rate. */ @@ -1139,7 +1139,7 @@ int live_rps_power(void *arg) if (!intel_rps_is_enabled(rps) || INTEL_GEN(gt->i915) < 6) return 0; - if (!librapl_energy_uJ()) + if (!librapl_supported(gt->i915)) return 0; if (igt_spinner_init(&spin, gt)) diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index f8f02aab842b..0683b27a3890 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -8,6 +8,7 @@ #include <linux/shmem_fs.h> #include "gem/i915_gem_object.h" +#include "gem/i915_gem_lmem.h" #include "shmem_utils.h" struct file *shmem_create_from_data(const char *name, void *data, size_t len) @@ -39,7 +40,8 @@ struct file *shmem_create_from_object(struct drm_i915_gem_object *obj) return file; } - ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); + ptr = i915_gem_object_pin_map_unlocked(obj, i915_gem_object_is_lmem(obj) ? + I915_MAP_WC : I915_MAP_WB); if (IS_ERR(ptr)) return ERR_CAST(ptr); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 78305b2ec89d..adae04c47aab 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -682,7 +682,9 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, if (IS_ERR(vma)) return PTR_ERR(vma); - vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB); + vaddr = i915_gem_object_pin_map_unlocked(vma->obj, + i915_coherent_map_type(guc_to_gt(guc)->i915, + vma->obj, true)); if (IS_ERR(vaddr)) { i915_vma_unpin_and_release(&vma, 0); return PTR_ERR(vaddr); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 92688a9b6717..335719f17490 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -11,6 +11,7 @@ #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm.h" #include "gt/intel_lrc.h" #include "gt/intel_mocs.h" @@ -264,6 +265,14 @@ static void guc_submission_tasklet(struct tasklet_struct *t) spin_unlock_irqrestore(&engine->active.lock, flags); } +static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + if (iir & GT_RENDER_USER_INTERRUPT) { + intel_engine_signal_breadcrumbs(engine); + tasklet_hi_schedule(&engine->execlists.tasklet); + } +} + static void guc_reset_prepare(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -608,35 +617,6 @@ static int guc_resume(struct intel_engine_cs *engine) static void guc_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = guc_submit_request; - engine->schedule = i915_schedule; - engine->execlists.tasklet.callback = guc_submission_tasklet; - - engine->reset.prepare = guc_reset_prepare; - engine->reset.rewind = guc_reset_rewind; - engine->reset.cancel = guc_reset_cancel; - engine->reset.finish = guc_reset_finish; - - engine->flags |= I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; - engine->flags |= I915_ENGINE_HAS_PREEMPTION; - - /* - * TODO: GuC supports timeslicing and semaphores as well, but they're - * handled by the firmware so some minor tweaks are required before - * enabling. - * - * engine->flags |= I915_ENGINE_HAS_TIMESLICES; - * engine->flags |= I915_ENGINE_HAS_SEMAPHORES; - */ - - engine->emit_bb_start = gen8_emit_bb_start; - - /* - * For the breadcrumb irq to work we need the interrupts to stay - * enabled. However, on all platforms on which we'll have support for - * GuC submission we don't allow disabling the interrupts at runtime, so - * we're always safe with the current flow. - */ - GEM_BUG_ON(engine->irq_enable || engine->irq_disable); } static void guc_release(struct intel_engine_cs *engine) @@ -658,6 +638,13 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->cops = &guc_context_ops; engine->request_alloc = guc_request_alloc; + engine->schedule = i915_schedule; + + engine->reset.prepare = guc_reset_prepare; + engine->reset.rewind = guc_reset_rewind; + engine->reset.cancel = guc_reset_cancel; + engine->reset.finish = guc_reset_finish; + engine->emit_flush = gen8_emit_flush_xcs; engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs; @@ -666,6 +653,19 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->emit_flush = gen12_emit_flush_xcs; } engine->set_default_submission = guc_set_default_submission; + + engine->flags |= I915_ENGINE_HAS_PREEMPTION; + + /* + * TODO: GuC supports timeslicing and semaphores as well, but they're + * handled by the firmware so some minor tweaks are required before + * enabling. + * + * engine->flags |= I915_ENGINE_HAS_TIMESLICES; + * engine->flags |= I915_ENGINE_HAS_SEMAPHORES; + */ + + engine->emit_bb_start = gen8_emit_bb_start; } static void rcs_submission_override(struct intel_engine_cs *engine) @@ -689,6 +689,7 @@ static void rcs_submission_override(struct intel_engine_cs *engine) static inline void guc_default_irqs(struct intel_engine_cs *engine) { engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT; + intel_engine_set_irq_handler(engine, cs_irq_handler); } int intel_guc_submission_setup(struct intel_engine_cs *engine) @@ -753,8 +754,3 @@ void intel_guc_submission_init_early(struct intel_guc *guc) { guc->submission_selected = __guc_submission_selected(guc); } - -bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs *engine) -{ - return engine->set_default_submission == guc_set_default_submission; -} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h index 5f7b9e6347d0..3f7005018939 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h @@ -20,7 +20,6 @@ void intel_guc_submission_fini(struct intel_guc *guc); int intel_guc_preempt_work_create(struct intel_guc *guc); void intel_guc_preempt_work_destroy(struct intel_guc *guc); int intel_guc_submission_setup(struct intel_engine_cs *engine); -bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs *engine); static inline bool intel_guc_submission_is_supported(struct intel_guc *guc) { diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c index 2126dd81ac38..56d2144dc6a0 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c @@ -82,7 +82,9 @@ static int intel_huc_rsa_data_create(struct intel_huc *huc) if (IS_ERR(vma)) return PTR_ERR(vma); - vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB); + vaddr = i915_gem_object_pin_map_unlocked(vma->obj, + i915_coherent_map_type(gt->i915, + vma->obj, true)); if (IS_ERR(vaddr)) { i915_vma_unpin_and_release(&vma, 0); return PTR_ERR(vaddr); diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index e7c2babcee8b..cbac409f6c8a 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -46,118 +46,6 @@ static const char * const supported_hypervisors[] = { [INTEL_GVT_HYPERVISOR_KVM] = "KVM", }; -static struct intel_vgpu_type * -intel_gvt_find_vgpu_type(struct intel_gvt *gvt, unsigned int type_group_id) -{ - if (WARN_ON(type_group_id >= gvt->num_types)) - return NULL; - return &gvt->types[type_group_id]; -} - -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) -{ - struct intel_vgpu_type *type; - unsigned int num = 0; - void *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = intel_gvt_find_vgpu_type(gvt, mtype_get_type_group_id(mtype)); - if (!type) - num = 0; - else - num = type->avail_instance; - - return sprintf(buf, "%u\n", num); -} - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} - -static ssize_t description_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - struct intel_vgpu_type *type; - void *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = intel_gvt_find_vgpu_type(gvt, mtype_get_type_group_id(mtype)); - if (!type) - return 0; - - return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" - "fence: %d\nresolution: %s\n" - "weight: %d\n", - BYTES_TO_MB(type->low_gm_size), - BYTES_TO_MB(type->high_gm_size), - type->fence, vgpu_edid_str(type->resolution), - type->weight); -} - -static MDEV_TYPE_ATTR_RO(available_instances); -static MDEV_TYPE_ATTR_RO(device_api); -static MDEV_TYPE_ATTR_RO(description); - -static struct attribute *gvt_type_attrs[] = { - &mdev_type_attr_available_instances.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_description.attr, - NULL, -}; - -static struct attribute_group *gvt_vgpu_type_groups[] = { - [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, -}; - -static bool intel_get_gvt_attrs(struct attribute_group ***intel_vgpu_type_groups) -{ - *intel_vgpu_type_groups = gvt_vgpu_type_groups; - return true; -} - -static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i, j; - struct intel_vgpu_type *type; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - type = &gvt->types[i]; - - group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); - if (WARN_ON(!group)) - goto unwind; - - group->name = type->name; - group->attrs = gvt_type_attrs; - gvt_vgpu_type_groups[i] = group; - } - - return 0; - -unwind: - for (j = 0; j < i; j++) { - group = gvt_vgpu_type_groups[j]; - kfree(group); - } - - return -ENOMEM; -} - -static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - group = gvt_vgpu_type_groups[i]; - gvt_vgpu_type_groups[i] = NULL; - kfree(group); - } -} - static const struct intel_gvt_ops intel_gvt_ops = { .emulate_cfg_read = intel_vgpu_emulate_cfg_read, .emulate_cfg_write = intel_vgpu_emulate_cfg_write, @@ -169,8 +57,6 @@ static const struct intel_gvt_ops intel_gvt_ops = { .vgpu_reset = intel_gvt_reset_vgpu, .vgpu_activate = intel_gvt_activate_vgpu, .vgpu_deactivate = intel_gvt_deactivate_vgpu, - .gvt_find_vgpu_type = intel_gvt_find_vgpu_type, - .get_gvt_attrs = intel_get_gvt_attrs, .vgpu_query_plane = intel_vgpu_query_plane, .vgpu_get_dmabuf = intel_vgpu_get_dmabuf, .write_protect_handler = intel_vgpu_page_track_handler, @@ -274,7 +160,6 @@ void intel_gvt_clean_device(struct drm_i915_private *i915) return; intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); - intel_gvt_cleanup_vgpu_type_groups(gvt); intel_gvt_clean_vgpu_types(gvt); intel_gvt_debugfs_clean(gvt); @@ -363,12 +248,6 @@ int intel_gvt_init_device(struct drm_i915_private *i915) if (ret) goto out_clean_thread; - ret = intel_gvt_init_vgpu_type_groups(gvt); - if (ret) { - gvt_err("failed to init vgpu type groups: %d\n", ret); - goto out_clean_types; - } - vgpu = intel_gvt_create_idle_vgpu(gvt); if (IS_ERR(vgpu)) { ret = PTR_ERR(vgpu); @@ -454,7 +333,8 @@ EXPORT_SYMBOL_GPL(intel_gvt_register_hypervisor); void intel_gvt_unregister_hypervisor(void) { - intel_gvt_hypervisor_host_exit(intel_gvt_host.dev); + void *gvt = (void *)kdev_to_i915(intel_gvt_host.dev)->gvt; + intel_gvt_hypervisor_host_exit(intel_gvt_host.dev, gvt); module_put(THIS_MODULE); } EXPORT_SYMBOL_GPL(intel_gvt_unregister_hypervisor); diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 88ab360fcb31..0c0615602343 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -574,9 +574,6 @@ struct intel_gvt_ops { void (*vgpu_reset)(struct intel_vgpu *); void (*vgpu_activate)(struct intel_vgpu *); void (*vgpu_deactivate)(struct intel_vgpu *); - struct intel_vgpu_type *(*gvt_find_vgpu_type)( - struct intel_gvt *gvt, unsigned int type_group_id); - bool (*get_gvt_attrs)(struct attribute_group ***intel_vgpu_type_groups); int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *); int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int); int (*write_protect_handler)(struct intel_vgpu *, u64, void *, diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h index b79da5124f83..f33e3cbd0439 100644 --- a/drivers/gpu/drm/i915/gvt/hypercall.h +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -49,7 +49,7 @@ enum hypervisor_type { struct intel_gvt_mpt { enum hypervisor_type type; int (*host_init)(struct device *dev, void *gvt, const void *ops); - void (*host_exit)(struct device *dev); + void (*host_exit)(struct device *dev, void *gvt); int (*attach_vgpu)(void *vgpu, unsigned long *handle); void (*detach_vgpu)(void *vgpu); int (*inject_msi)(unsigned long handle, u32 addr, u16 data); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 65ff43cfc0f7..48b4d4cf805d 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -144,6 +144,104 @@ static inline bool handle_valid(unsigned long handle) return !!(handle & ~0xff); } +static ssize_t available_instances_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, + char *buf) +{ + struct intel_vgpu_type *type; + unsigned int num = 0; + struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; + + type = &gvt->types[mtype_get_type_group_id(mtype)]; + if (!type) + num = 0; + else + num = type->avail_instance; + + return sprintf(buf, "%u\n", num); +} + +static ssize_t device_api_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); +} + +static ssize_t description_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + struct intel_vgpu_type *type; + struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; + + type = &gvt->types[mtype_get_type_group_id(mtype)]; + if (!type) + return 0; + + return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" + "fence: %d\nresolution: %s\n" + "weight: %d\n", + BYTES_TO_MB(type->low_gm_size), + BYTES_TO_MB(type->high_gm_size), + type->fence, vgpu_edid_str(type->resolution), + type->weight); +} + +static MDEV_TYPE_ATTR_RO(available_instances); +static MDEV_TYPE_ATTR_RO(device_api); +static MDEV_TYPE_ATTR_RO(description); + +static struct attribute *gvt_type_attrs[] = { + &mdev_type_attr_available_instances.attr, + &mdev_type_attr_device_api.attr, + &mdev_type_attr_description.attr, + NULL, +}; + +static struct attribute_group *gvt_vgpu_type_groups[] = { + [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, +}; + +static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i, j; + struct intel_vgpu_type *type; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + type = &gvt->types[i]; + + group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); + if (!group) + goto unwind; + + group->name = type->name; + group->attrs = gvt_type_attrs; + gvt_vgpu_type_groups[i] = group; + } + + return 0; + +unwind: + for (j = 0; j < i; j++) { + group = gvt_vgpu_type_groups[j]; + kfree(group); + } + + return -ENOMEM; +} + +static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + group = gvt_vgpu_type_groups[i]; + gvt_vgpu_type_groups[i] = NULL; + kfree(group); + } +} + static int kvmgt_guest_init(struct mdev_device *mdev); static void intel_vgpu_release_work(struct work_struct *work); static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); @@ -694,14 +792,13 @@ static int intel_vgpu_create(struct mdev_device *mdev) struct intel_vgpu *vgpu = NULL; struct intel_vgpu_type *type; struct device *pdev; - void *gvt; + struct intel_gvt *gvt; int ret; pdev = mdev_parent_dev(mdev); gvt = kdev_to_i915(pdev)->gvt; - type = intel_gvt_ops->gvt_find_vgpu_type(gvt, - mdev_get_type_group_id(mdev)); + type = &gvt->types[mdev_get_type_group_id(mdev)]; if (!type) { ret = -EINVAL; goto out; @@ -1667,19 +1764,26 @@ static struct mdev_parent_ops intel_vgpu_ops = { static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) { - struct attribute_group **kvm_vgpu_type_groups; + int ret; + + ret = intel_gvt_init_vgpu_type_groups((struct intel_gvt *)gvt); + if (ret) + return ret; intel_gvt_ops = ops; - if (!intel_gvt_ops->get_gvt_attrs(&kvm_vgpu_type_groups)) - return -EFAULT; - intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; + intel_vgpu_ops.supported_type_groups = gvt_vgpu_type_groups; - return mdev_register_device(dev, &intel_vgpu_ops); + ret = mdev_register_device(dev, &intel_vgpu_ops); + if (ret) + intel_gvt_cleanup_vgpu_type_groups((struct intel_gvt *)gvt); + + return ret; } -static void kvmgt_host_exit(struct device *dev) +static void kvmgt_host_exit(struct device *dev, void *gvt) { mdev_unregister_device(dev); + intel_gvt_cleanup_vgpu_type_groups((struct intel_gvt *)gvt); } static int kvmgt_page_track_add(unsigned long handle, u64 gfn) diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index 550a456e936f..e6c5a792a49a 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -63,13 +63,13 @@ static inline int intel_gvt_hypervisor_host_init(struct device *dev, /** * intel_gvt_hypervisor_host_exit - exit GVT-g host side */ -static inline void intel_gvt_hypervisor_host_exit(struct device *dev) +static inline void intel_gvt_hypervisor_host_exit(struct device *dev, void *gvt) { /* optional to provide */ if (!intel_gvt_host.mpt->host_exit) return; - intel_gvt_host.mpt->host_exit(dev); + intel_gvt_host.mpt->host_exit(dev, gvt); } /** diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index aa573b078ae7..b1aa1c482c32 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -343,18 +343,15 @@ out: void __i915_active_init(struct i915_active *ref, int (*active)(struct i915_active *ref), void (*retire)(struct i915_active *ref), + unsigned long flags, struct lock_class_key *mkey, struct lock_class_key *wkey) { - unsigned long bits; - debug_active_init(ref); - ref->flags = 0; + ref->flags = flags; ref->active = active; - ref->retire = ptr_unpack_bits(retire, &bits, 2); - if (bits & I915_ACTIVE_MAY_SLEEP) - ref->flags |= I915_ACTIVE_RETIRE_SLEEPS; + ref->retire = retire; spin_lock_init(&ref->tree_lock); ref->tree = RB_ROOT; @@ -1156,8 +1153,7 @@ static int auto_active(struct i915_active *ref) return 0; } -__i915_active_call static void -auto_retire(struct i915_active *ref) +static void auto_retire(struct i915_active *ref) { i915_active_put(ref); } @@ -1171,7 +1167,7 @@ struct i915_active *i915_active_create(void) return NULL; kref_init(&aa->ref); - i915_active_init(&aa->base, auto_active, auto_retire); + i915_active_init(&aa->base, auto_active, auto_retire, 0); return &aa->base; } diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h index fb165d3f01cf..d0feda68b874 100644 --- a/drivers/gpu/drm/i915/i915_active.h +++ b/drivers/gpu/drm/i915/i915_active.h @@ -152,15 +152,16 @@ i915_active_fence_isset(const struct i915_active_fence *active) void __i915_active_init(struct i915_active *ref, int (*active)(struct i915_active *ref), void (*retire)(struct i915_active *ref), + unsigned long flags, struct lock_class_key *mkey, struct lock_class_key *wkey); /* Specialise each class of i915_active to avoid impossible lockdep cycles. */ -#define i915_active_init(ref, active, retire) do { \ - static struct lock_class_key __mkey; \ - static struct lock_class_key __wkey; \ - \ - __i915_active_init(ref, active, retire, &__mkey, &__wkey); \ +#define i915_active_init(ref, active, retire, flags) do { \ + static struct lock_class_key __mkey; \ + static struct lock_class_key __wkey; \ + \ + __i915_active_init(ref, active, retire, flags, &__mkey, &__wkey); \ } while (0) struct dma_fence * diff --git a/drivers/gpu/drm/i915/i915_active_types.h b/drivers/gpu/drm/i915/i915_active_types.h index 6360c3e4b765..c149f348a972 100644 --- a/drivers/gpu/drm/i915/i915_active_types.h +++ b/drivers/gpu/drm/i915/i915_active_types.h @@ -24,11 +24,6 @@ struct i915_active_fence { struct active_node; -#define I915_ACTIVE_MAY_SLEEP BIT(0) - -#define __i915_active_call __aligned(4) -#define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2) - struct i915_active { atomic_t count; struct mutex mutex; diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index e6f1e93abbbb..5b4b2bd46e7c 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1369,6 +1369,20 @@ static int check_bbstart(u32 *cmd, u32 offset, u32 length, return 0; } +/** + * intel_engine_cmd_parser_alloc_jump_whitelist() - preallocate jump whitelist for intel_engine_cmd_parser() + * @batch_length: length of the commands in batch_obj + * @trampoline: Whether jump trampolines are used. + * + * Preallocates a jump whitelist for parsing the cmd buffer in intel_engine_cmd_parser(). + * This has to be preallocated, because the command parser runs in signaling context, + * and may not allocate any memory. + * + * Return: NULL or pointer to a jump whitelist, or ERR_PTR() on failure. Use + * IS_ERR() to check for errors. Must bre freed() with kfree(). + * + * NULL is a valid value, meaning no allocation was required. + */ unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length, bool trampoline) { @@ -1401,7 +1415,9 @@ unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length, * @batch_offset: byte offset in the batch at which execution starts * @batch_length: length of the commands in batch_obj * @shadow: validated copy of the batch buffer in question - * @trampoline: whether to emit a conditional trampoline at the end of the batch + * @jump_whitelist: buffer preallocated with intel_engine_cmd_parser_alloc_jump_whitelist() + * @shadow_map: mapping to @shadow vma + * @batch_map: mapping to @batch vma * * Parses the specified batch buffer looking for privilege violations as * described in the overview. diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index b654b7498bcd..8dd374691102 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -622,9 +622,9 @@ static int i915_swizzle_info(struct seq_file *m, void *data) seq_printf(m, "DDC2 = 0x%08x\n", intel_uncore_read(uncore, DCC2)); seq_printf(m, "C0DRB3 = 0x%04x\n", - intel_uncore_read16(uncore, C0DRB3)); + intel_uncore_read16(uncore, C0DRB3_BW)); seq_printf(m, "C1DRB3 = 0x%04x\n", - intel_uncore_read16(uncore, C1DRB3)); + intel_uncore_read16(uncore, C1DRB3_BW)); } else if (INTEL_GEN(dev_priv) >= 6) { seq_printf(m, "MAD_DIMM_C0 = 0x%08x\n", intel_uncore_read(uncore, MAD_DIMM_C0)); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index db513f93f0f5..5118dc8386b2 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -1727,6 +1727,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CREATE_EXT, i915_gem_create_ext_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 374372773b48..9aee6a045590 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -51,7 +51,6 @@ #include <linux/xarray.h> #include <drm/intel-gtt.h> -#include <drm/drm_legacy.h> /* for struct drm_dma_handle */ #include <drm/drm_gem.h> #include <drm/drm_auth.h> #include <drm/drm_cache.h> @@ -78,6 +77,7 @@ #include "gem/i915_gem_context_types.h" #include "gem/i915_gem_shrinker.h" #include "gem/i915_gem_stolen.h" +#include "gem/i915_gem_lmem.h" #include "gt/intel_engine.h" #include "gt/intel_gt_types.h" @@ -514,6 +514,13 @@ struct intel_l3_parity { }; struct i915_gem_mm { + /* + * Shortcut for the stolen region. This points to either + * INTEL_REGION_STOLEN_SMEM for integrated platforms, or + * INTEL_REGION_STOLEN_LMEM for discrete, or NULL if the device doesn't + * support stolen. + */ + struct intel_memory_region *stolen_region; /** Memory allocator for GTT stolen memory */ struct drm_mm stolen; /** Protects the usage of the GTT stolen memory allocator. This is @@ -1721,9 +1728,15 @@ static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) } static inline bool -intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) +intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *i915) { - return IS_BROXTON(dev_priv) && intel_vtd_active(); + return IS_BROXTON(i915) && intel_vtd_active(); +} + +static inline bool +intel_vm_no_concurrent_access_wa(struct drm_i915_private *i915) +{ + return IS_CHERRYVIEW(i915) || intel_ggtt_update_needs_vtd_wa(i915); } /* i915_drv.c */ @@ -1803,6 +1816,7 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, #define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) #define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1) #define I915_GEM_OBJECT_UNBIND_TEST BIT(2) +#define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3) void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); @@ -1935,9 +1949,15 @@ static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) } static inline enum i915_map_type -i915_coherent_map_type(struct drm_i915_private *i915) +i915_coherent_map_type(struct drm_i915_private *i915, + struct drm_i915_gem_object *obj, bool always_coherent) { - return HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC; + if (i915_gem_object_is_lmem(obj)) + return I915_MAP_WC; + if (HAS_LLC(i915) || always_coherent) + return I915_MAP_WB; + else + return I915_MAP_WC; } #endif diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index b23f58e94cfb..cffd7f4f87dc 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -157,8 +157,18 @@ try_again: if (vma) { ret = -EBUSY; if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE || - !i915_vma_is_active(vma)) - ret = i915_vma_unbind(vma); + !i915_vma_is_active(vma)) { + if (flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK) { + if (mutex_trylock(&vma->vm->mutex)) { + ret = __i915_vma_unbind(vma); + mutex_unlock(&vma->vm->mutex); + } else { + ret = -EBUSY; + } + } else { + ret = i915_vma_unbind(vma); + } + } __i915_vma_put(vma); } @@ -999,12 +1009,11 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, obj->mm.madv = args->madv; if (i915_gem_object_has_pages(obj)) { - struct list_head *list; + unsigned long flags; - if (i915_gem_object_is_shrinkable(obj)) { - unsigned long flags; - - spin_lock_irqsave(&i915->mm.obj_lock, flags); + spin_lock_irqsave(&i915->mm.obj_lock, flags); + if (!list_empty(&obj->mm.link)) { + struct list_head *list; if (obj->mm.madv != I915_MADV_WILLNEED) list = &i915->mm.purge_list; @@ -1012,8 +1021,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, list = &i915->mm.shrink_list; list_move_tail(&obj->mm.link, list); - spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } /* if the object is no longer attached, discard its backing storage */ diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h index eb435f9e0220..b98d8cdbe4f2 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.h +++ b/drivers/gpu/drm/i915/i915_gpu_error.h @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * - * Copyright � 2008-2018 Intel Corporation + * Copyright © 2008-2018 Intel Corporation */ #ifndef _I915_GPU_ERROR_H_ diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index d4611c643446..9ff511862848 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -4024,7 +4024,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[RCS0], iir); if (iir & I915_MASTER_ERROR_INTERRUPT) i8xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4132,7 +4132,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[RCS0], iir); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4277,10 +4277,12 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[RCS0], + iir); if (iir & I915_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[VCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[VCS0], + iir >> 25); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c index 25576fa73ff0..9a777b0ff59b 100644 --- a/drivers/gpu/drm/i915/i915_mm.c +++ b/drivers/gpu/drm/i915/i915_mm.c @@ -28,10 +28,46 @@ #include "i915_drv.h" -#define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP) +struct remap_pfn { + struct mm_struct *mm; + unsigned long pfn; + pgprot_t prot; + + struct sgt_iter sgt; + resource_size_t iobase; +}; #define use_dma(io) ((io) != -1) +static inline unsigned long sgt_pfn(const struct remap_pfn *r) +{ + if (use_dma(r->iobase)) + return (r->sgt.dma + r->sgt.curr + r->iobase) >> PAGE_SHIFT; + else + return r->sgt.pfn + (r->sgt.curr >> PAGE_SHIFT); +} + +static int remap_sg(pte_t *pte, unsigned long addr, void *data) +{ + struct remap_pfn *r = data; + + if (GEM_WARN_ON(!r->sgt.sgp)) + return -EINVAL; + + /* Special PTE are not associated with any struct page */ + set_pte_at(r->mm, addr, pte, + pte_mkspecial(pfn_pte(sgt_pfn(r), r->prot))); + r->pfn++; /* track insertions in case we need to unwind later */ + + r->sgt.curr += PAGE_SIZE; + if (r->sgt.curr >= r->sgt.max) + r->sgt = __sgt_iter(__sg_next(r->sgt.sgp), use_dma(r->iobase)); + + return 0; +} + +#define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP) + /** * remap_io_sg - remap an IO mapping to userspace * @vma: user vma to map to @@ -46,8 +82,13 @@ int remap_io_sg(struct vm_area_struct *vma, unsigned long addr, unsigned long size, struct scatterlist *sgl, resource_size_t iobase) { - unsigned long pfn, len, remapped = 0; - int err = 0; + struct remap_pfn r = { + .mm = vma->vm_mm, + .prot = vma->vm_page_prot, + .sgt = __sgt_iter(sgl, use_dma(iobase)), + .iobase = iobase, + }; + int err; /* We rely on prevalidation of the io-mapping to skip track_pfn(). */ GEM_BUG_ON((vma->vm_flags & EXPECTED_FLAGS) != EXPECTED_FLAGS); @@ -55,25 +96,11 @@ int remap_io_sg(struct vm_area_struct *vma, if (!use_dma(iobase)) flush_cache_range(vma, addr, size); - do { - if (use_dma(iobase)) { - if (!sg_dma_len(sgl)) - break; - pfn = (sg_dma_address(sgl) + iobase) >> PAGE_SHIFT; - len = sg_dma_len(sgl); - } else { - pfn = page_to_pfn(sg_page(sgl)); - len = sgl->length; - } - - err = remap_pfn_range(vma, addr + remapped, pfn, len, - vma->vm_page_prot); - if (err) - break; - remapped += len; - } while ((sgl = __sg_next(sgl))); - - if (err) - zap_vma_ptes(vma, addr, remapped); - return err; + err = apply_to_page_range(r.mm, addr, size, remap_sg, &r); + if (unlikely(err)) { + zap_vma_ptes(vma, addr, r.pfn << PAGE_SHIFT); + return err; + } + + return 0; } diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 14cd64cc61d0..4a114a5ad000 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -71,18 +71,18 @@ struct drm_printer; param(int, fastboot, -1, 0600) \ param(int, enable_dpcd_backlight, -1, 0600) \ param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \ - param(unsigned long, fake_lmem_start, 0, 0400) \ - param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, 0600) \ + param(unsigned long, fake_lmem_start, 0, IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) ? 0400 : 0) \ + param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, CONFIG_DRM_I915_REQUEST_TIMEOUT ? 0600 : 0) \ /* leave bools at the end to not create holes */ \ param(bool, enable_hangcheck, true, 0600) \ param(bool, load_detect_test, false, 0600) \ param(bool, force_reset_modeset_test, false, 0600) \ - param(bool, error_capture, true, 0600) \ + param(bool, error_capture, true, IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ? 0600 : 0) \ param(bool, disable_display, false, 0400) \ param(bool, verbose_state_checks, true, 0) \ param(bool, nuclear_pageflip, false, 0400) \ param(bool, enable_dp_mst, true, 0600) \ - param(bool, enable_gvt, false, 0400) + param(bool, enable_gvt, false, IS_ENABLED(CONFIG_DRM_I915_GVT) ? 0400 : 0) #define MEMBER(T, member, ...) T member; struct i915_params { diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 1680062a2149..574881c0e339 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -908,7 +908,7 @@ static const struct intel_device_info rkl_info = { }; #define DGFX_FEATURES \ - .memory_regions = REGION_SMEM | REGION_LMEM, \ + .memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \ .has_master_unit_irq = 1, \ .has_llc = 0, \ .has_snoop = 1, \ diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index de8ebc34af0f..cb03e4152d2d 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -1257,11 +1257,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) case 8: case 9: case 10: - if (intel_engine_in_execlists_submission_mode(ce->engine)) { - stream->specific_ctx_id_mask = - (1U << GEN8_CTX_ID_WIDTH) - 1; - stream->specific_ctx_id = stream->specific_ctx_id_mask; - } else { + if (intel_engine_uses_guc(ce->engine)) { /* * When using GuC, the context descriptor we write in * i915 is read by GuC and rewritten before it's @@ -1280,6 +1276,10 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) */ stream->specific_ctx_id_mask = (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; + } else { + stream->specific_ctx_id_mask = + (1U << GEN8_CTX_ID_WIDTH) - 1; + stream->specific_ctx_id = stream->specific_ctx_id_mask; } break; diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 41651ac255fa..a75cd1db320b 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -476,6 +476,8 @@ engine_event_status(struct intel_engine_cs *engine, static int config_status(struct drm_i915_private *i915, u64 config) { + struct intel_gt *gt = &i915->gt; + switch (config) { case I915_PMU_ACTUAL_FREQUENCY: if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) @@ -489,7 +491,7 @@ config_status(struct drm_i915_private *i915, u64 config) case I915_PMU_INTERRUPTS: break; case I915_PMU_RC6_RESIDENCY: - if (!HAS_RC6(i915)) + if (!gt->rc6.supported) return -ENODEV; break; case I915_PMU_SOFTWARE_GT_AWAKE_TIME: diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index fed337ad7b68..e49da36c62fb 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -419,11 +419,73 @@ static int query_perf_config(struct drm_i915_private *i915, } } +static int query_memregion_info(struct drm_i915_private *i915, + struct drm_i915_query_item *query_item) +{ + struct drm_i915_query_memory_regions __user *query_ptr = + u64_to_user_ptr(query_item->data_ptr); + struct drm_i915_memory_region_info __user *info_ptr = + &query_ptr->regions[0]; + struct drm_i915_memory_region_info info = { }; + struct drm_i915_query_memory_regions query; + struct intel_memory_region *mr; + u32 total_length; + int ret, id, i; + + if (!IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) + return -ENODEV; + + if (query_item->flags != 0) + return -EINVAL; + + total_length = sizeof(query); + for_each_memory_region(mr, i915, id) { + if (mr->private) + continue; + + total_length += sizeof(info); + } + + ret = copy_query_item(&query, sizeof(query), total_length, query_item); + if (ret != 0) + return ret; + + if (query.num_regions) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(query.rsvd); i++) { + if (query.rsvd[i]) + return -EINVAL; + } + + for_each_memory_region(mr, i915, id) { + if (mr->private) + continue; + + info.region.memory_class = mr->type; + info.region.memory_instance = mr->instance; + info.probed_size = mr->total; + info.unallocated_size = mr->avail; + + if (__copy_to_user(info_ptr, &info, sizeof(info))) + return -EFAULT; + + query.num_regions++; + info_ptr++; + } + + if (__copy_to_user(query_ptr, &query, sizeof(query))) + return -EFAULT; + + return total_length; +} + static int (* const i915_query_funcs[])(struct drm_i915_private *dev_priv, struct drm_i915_query_item *query_item) = { query_topology_info, query_engine_info, query_perf_config, + query_memregion_info, }; int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 07cca3b423bc..c294e3f93bed 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -416,6 +416,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_VECS_SFC_USAGE(engine) _MMIO((engine)->mmio_base + 0x2014) #define GEN11_VECS_SFC_USAGE_BIT (1 << 0) +#define GEN12_HCP_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x2910) +#define GEN12_HCP_SFC_FORCED_LOCK_BIT REG_BIT(0) +#define GEN12_HCP_SFC_LOCK_STATUS(engine) _MMIO((engine)->mmio_base + 0x2914) +#define GEN12_HCP_SFC_LOCK_ACK_BIT REG_BIT(1) +#define GEN12_HCP_SFC_USAGE_BIT REG_BIT(0) + #define GEN12_SFC_DONE(n) _MMIO(0x1cc00 + (n) * 0x100) #define GEN12_SFC_DONE_MAX 4 @@ -487,6 +493,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GAB_CTL _MMIO(0x24000) #define GAB_CTL_CONT_AFTER_PAGEFAULT (1 << 8) +#define GU_CNTL _MMIO(0x101010) +#define LMEM_INIT REG_BIT(7) + #define GEN6_STOLEN_RESERVED _MMIO(0x1082C0) #define GEN6_STOLEN_RESERVED_ADDR_MASK (0xFFF << 20) #define GEN7_STOLEN_RESERVED_ADDR_MASK (0x3FFF << 18) @@ -2715,6 +2724,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RING_INDIRECT_CTX_OFFSET(base) _MMIO((base) + 0x1c8) /* gen8+ */ #define RING_CTX_TIMESTAMP(base) _MMIO((base) + 0x3a8) /* gen8+ */ +#define VDBOX_CGCTL3F10(base) _MMIO((base) + 0x3f10) +#define IECPUNIT_CLKGATE_DIS REG_BIT(22) + #define ERROR_GEN6 _MMIO(0x40a0) #define GEN7_ERR_INT _MMIO(0x44040) #define ERR_INT_POISON (1 << 31) @@ -3781,8 +3793,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define CSHRDDR3CTL_DDR3 (1 << 2) /* 965 MCH register controlling DRAM channel configuration */ -#define C0DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x206) -#define C1DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x606) +#define C0DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x206) +#define C1DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x606) /* snb MCH registers for reading the DRAM channel configuration */ #define MAD_DIMM_C0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004) @@ -12208,6 +12220,7 @@ enum skl_power_gate { #define GEN12_GLOBAL_MOCS(i) _MMIO(0x4000 + (i) * 4) /* Global MOCS regs */ #define GEN12_GSMBASE _MMIO(0x108100) +#define GEN12_DSMBASE _MMIO(0x1080C0) /* gamt regs */ #define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4) diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index bec9c3652188..970d8f4986bb 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -929,7 +929,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) u32 seqno; int ret; - might_sleep_if(gfpflags_allow_blocking(gfp)); + might_alloc(gfp); /* Check that the caller provided an already pinned context */ __intel_context_pin(ce); diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 07490db51cdc..a6cd0fa62847 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -27,6 +27,7 @@ #include "display/intel_frontbuffer.h" +#include "gem/i915_gem_lmem.h" #include "gt/intel_engine.h" #include "gt/intel_engine_heartbeat.h" #include "gt/intel_gt.h" @@ -93,7 +94,6 @@ static int __i915_vma_active(struct i915_active *ref) return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT; } -__i915_active_call static void __i915_vma_retire(struct i915_active *ref) { i915_vma_put(active_to_vma(ref)); @@ -124,7 +124,7 @@ vma_create(struct drm_i915_gem_object *obj, vma->size = obj->base.size; vma->display_alignment = I915_GTT_MIN_ALIGNMENT; - i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire); + i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0); /* Declare ourselves safe for use inside shrinkers */ if (IS_ENABLED(CONFIG_LOCKDEP)) { @@ -448,9 +448,11 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) void __iomem *ptr; int err; - if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { - err = -ENODEV; - goto err; + if (!i915_gem_object_is_lmem(vma->obj)) { + if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { + err = -ENODEV; + goto err; + } } GEM_BUG_ON(!i915_vma_is_ggtt(vma)); @@ -458,9 +460,19 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) ptr = READ_ONCE(vma->iomap); if (ptr == NULL) { - ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, - vma->node.start, - vma->node.size); + /* + * TODO: consider just using i915_gem_object_pin_map() for lmem + * instead, which already supports mapping non-contiguous chunks + * of pages, that way we can also drop the + * I915_BO_ALLOC_CONTIGUOUS when allocating the object. + */ + if (i915_gem_object_is_lmem(vma->obj)) + ptr = i915_gem_object_lmem_io_map(vma->obj, 0, + vma->obj->base.size); + else + ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, + vma->node.start, + vma->node.size); if (ptr == NULL) { err = -ENOMEM; goto err; @@ -905,8 +917,7 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, if (err) goto err_fence; - err = i915_vm_pin_pt_stash(vma->vm, - &work->stash); + err = i915_vm_map_pt_stash(vma->vm, &work->stash); if (err) goto err_fence; } diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index bf837b6bb185..d98e8b81d322 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -22,9 +22,29 @@ static const struct { .class = INTEL_MEMORY_STOLEN_SYSTEM, .instance = 0, }, + [INTEL_REGION_STOLEN_LMEM] = { + .class = INTEL_MEMORY_STOLEN_LOCAL, + .instance = 0, + }, }; struct intel_memory_region * +intel_memory_region_lookup(struct drm_i915_private *i915, + u16 class, u16 instance) +{ + struct intel_memory_region *mr; + int id; + + /* XXX: consider maybe converting to an rb tree at some point */ + for_each_memory_region(mr, i915, id) { + if (mr->type == class && mr->instance == instance) + return mr; + } + + return NULL; +} + +struct intel_memory_region * intel_memory_region_by_type(struct drm_i915_private *i915, enum intel_memory_type mem_type) { @@ -278,8 +298,15 @@ int intel_memory_regions_hw_probe(struct drm_i915_private *i915) case INTEL_MEMORY_SYSTEM: mem = i915_gem_shmem_setup(i915); break; + case INTEL_MEMORY_STOLEN_LOCAL: + mem = i915_gem_stolen_lmem_setup(i915); + if (!IS_ERR(mem)) + i915->mm.stolen_region = mem; + break; case INTEL_MEMORY_STOLEN_SYSTEM: - mem = i915_gem_stolen_setup(i915); + mem = i915_gem_stolen_smem_setup(i915); + if (!IS_ERR(mem)) + i915->mm.stolen_region = mem; break; default: continue; diff --git a/drivers/gpu/drm/i915/intel_memory_region.h b/drivers/gpu/drm/i915/intel_memory_region.h index edd49067c8ca..d24ce5a0b30b 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.h +++ b/drivers/gpu/drm/i915/intel_memory_region.h @@ -11,6 +11,7 @@ #include <linux/mutex.h> #include <linux/io-mapping.h> #include <drm/drm_mm.h> +#include <drm/i915_drm.h> #include "i915_buddy.h" @@ -19,25 +20,25 @@ struct drm_i915_gem_object; struct intel_memory_region; struct sg_table; -/** - * Base memory type - */ enum intel_memory_type { - INTEL_MEMORY_SYSTEM = 0, - INTEL_MEMORY_LOCAL, + INTEL_MEMORY_SYSTEM = I915_MEMORY_CLASS_SYSTEM, + INTEL_MEMORY_LOCAL = I915_MEMORY_CLASS_DEVICE, INTEL_MEMORY_STOLEN_SYSTEM, + INTEL_MEMORY_STOLEN_LOCAL, }; enum intel_region_id { INTEL_REGION_SMEM = 0, INTEL_REGION_LMEM, INTEL_REGION_STOLEN_SMEM, + INTEL_REGION_STOLEN_LMEM, INTEL_REGION_UNKNOWN, /* Should be last */ }; #define REGION_SMEM BIT(INTEL_REGION_SMEM) #define REGION_LMEM BIT(INTEL_REGION_LMEM) #define REGION_STOLEN_SMEM BIT(INTEL_REGION_STOLEN_SMEM) +#define REGION_STOLEN_LMEM BIT(INTEL_REGION_STOLEN_LMEM) #define I915_ALLOC_MIN_PAGE_SIZE BIT(0) #define I915_ALLOC_CONTIGUOUS BIT(1) @@ -82,7 +83,8 @@ struct intel_memory_region { u16 type; u16 instance; enum intel_region_id id; - char name[8]; + char name[16]; + bool private; /* not for userspace */ struct list_head reserved; @@ -95,6 +97,10 @@ struct intel_memory_region { } objects; }; +struct intel_memory_region * +intel_memory_region_lookup(struct drm_i915_private *i915, + u16 class, u16 instance); + int intel_memory_region_init_buddy(struct intel_memory_region *mem); void intel_memory_region_release_buddy(struct intel_memory_region *mem); diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index ed5abe7be498..b4aaf8b7109f 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -1917,6 +1917,18 @@ int intel_uncore_init_mmio(struct intel_uncore *uncore) if (ret) return ret; + /* + * The boot firmware initializes local memory and assesses its health. + * If memory training fails, the punit will have been instructed to + * keep the GT powered down; we won't be able to communicate with it + * and we should not continue with driver initialization. + */ + if (IS_DGFX(i915) && + !(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) { + drm_err(&i915->drm, "LMEM not initialized by firmware\n"); + return -ENODEV; + } + if (INTEL_GEN(i915) > 5 && !intel_vgpu_active(i915)) uncore->flags |= UNCORE_HAS_FORCEWAKE; diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c index 4002c984c2e0..61bf4560d8af 100644 --- a/drivers/gpu/drm/i915/selftests/i915_active.c +++ b/drivers/gpu/drm/i915/selftests/i915_active.c @@ -68,7 +68,7 @@ static struct live_active *__live_alloc(struct drm_i915_private *i915) return NULL; kref_init(&active->ref); - i915_active_init(&active->base, __live_active, __live_retire); + i915_active_init(&active->base, __live_active, __live_retire, 0); return active; } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index dc394fb7ccfa..152d9ab135b1 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -87,14 +87,14 @@ static void simulate_hibernate(struct drm_i915_private *i915) intel_runtime_pm_put(&i915->runtime_pm, wakeref); } -static int pm_prepare(struct drm_i915_private *i915) +static int igt_pm_prepare(struct drm_i915_private *i915) { i915_gem_suspend(i915); return 0; } -static void pm_suspend(struct drm_i915_private *i915) +static void igt_pm_suspend(struct drm_i915_private *i915) { intel_wakeref_t wakeref; @@ -104,7 +104,7 @@ static void pm_suspend(struct drm_i915_private *i915) } } -static void pm_hibernate(struct drm_i915_private *i915) +static void igt_pm_hibernate(struct drm_i915_private *i915) { intel_wakeref_t wakeref; @@ -116,7 +116,7 @@ static void pm_hibernate(struct drm_i915_private *i915) } } -static void pm_resume(struct drm_i915_private *i915) +static void igt_pm_resume(struct drm_i915_private *i915) { intel_wakeref_t wakeref; @@ -148,16 +148,16 @@ static int igt_gem_suspend(void *arg) if (err) goto out; - err = pm_prepare(i915); + err = igt_pm_prepare(i915); if (err) goto out; - pm_suspend(i915); + igt_pm_suspend(i915); /* Here be dragons! Note that with S3RST any S3 may become S4! */ simulate_hibernate(i915); - pm_resume(i915); + igt_pm_resume(i915); err = switch_to_context(ctx); out: @@ -183,16 +183,16 @@ static int igt_gem_hibernate(void *arg) if (err) goto out; - err = pm_prepare(i915); + err = igt_pm_prepare(i915); if (err) goto out; - pm_hibernate(i915); + igt_pm_hibernate(i915); /* Here be dragons! */ simulate_hibernate(i915); - pm_resume(i915); + igt_pm_resume(i915); err = switch_to_context(ctx); out: diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 45c6c0107c7c..0a1472bb12bc 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -186,7 +186,7 @@ retry: if (err) goto err_ppgtt_cleanup; - err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); if (err) { i915_vm_free_pt_stash(&ppgtt->vm, &stash); goto err_ppgtt_cleanup; @@ -208,7 +208,7 @@ retry: if (err) goto err_ppgtt_cleanup; - err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); if (err) { i915_vm_free_pt_stash(&ppgtt->vm, &stash); goto err_ppgtt_cleanup; @@ -325,11 +325,10 @@ retry: BIT_ULL(size))) goto alloc_vm_end; - err = i915_vm_pin_pt_stash(vm, &stash); + err = i915_vm_map_pt_stash(vm, &stash); if (!err) vm->allocate_va_range(vm, &stash, addr, BIT_ULL(size)); - i915_vm_free_pt_stash(vm, &stash); alloc_vm_end: if (err == -EDEADLK) { @@ -1968,10 +1967,9 @@ retry: if (err) goto end_ww; - err = i915_vm_pin_pt_stash(vm, &stash); + err = i915_vm_map_pt_stash(vm, &stash); if (!err) vm->allocate_va_range(vm, &stash, offset, chunk_size); - i915_vm_free_pt_stash(vm, &stash); end_ww: if (err == -EDEADLK) { diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c index e9d86dab8677..bfb0290967a1 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf.c +++ b/drivers/gpu/drm/i915/selftests/i915_perf.c @@ -307,7 +307,7 @@ static int live_noa_gpr(void *arg) } /* Poison the ce->vm so we detect writes not to the GGTT gt->scratch */ - scratch = kmap(__px_page(ce->vm->scratch[0])); + scratch = __px_vaddr(ce->vm->scratch[0]); memset(scratch, POISON_FREE, PAGE_SIZE); rq = intel_context_create_request(ce); @@ -405,7 +405,6 @@ static int live_noa_gpr(void *arg) out_rq: i915_request_put(rq); out_ce: - kunmap(__px_page(ce->vm->scratch[0])); intel_context_put(ce); out: stream_destroy(stream); diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index 5fe7b80ca0bd..dd0607254a95 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -967,6 +967,9 @@ static int igt_vma_remapped_gtt(void *arg) intel_wakeref_t wakeref; int err = 0; + if (!i915_ggtt_has_aperture(&i915->ggtt)) + return 0; + obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c index cfbbe415b57c..5fe397b7d1d9 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.c +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -94,9 +94,9 @@ int igt_spinner_pin(struct igt_spinner *spin, } if (!spin->batch) { - unsigned int mode = - i915_coherent_map_type(spin->gt->i915); + unsigned int mode; + mode = i915_coherent_map_type(spin->gt->i915, spin->obj, false); vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma); if (IS_ERR(vaddr)) return PTR_ERR(vaddr); diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index a5fc0bf3feb9..f85fd8cbfbf5 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -513,7 +513,7 @@ static int igt_cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) if (err) return err; - ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); + ptr = i915_gem_object_pin_map(obj, I915_MAP_WC); if (IS_ERR(ptr)) return PTR_ERR(ptr); @@ -593,7 +593,9 @@ static int igt_gpu_write(struct i915_gem_context *ctx, if (err) break; + i915_gem_object_lock(obj, NULL); err = igt_cpu_check(obj, dword, rng); + i915_gem_object_unlock(obj); if (err) break; } while (!__igt_timeout(end_time, NULL)); @@ -629,6 +631,88 @@ out_put: return err; } +static int igt_lmem_create_cleared_cpu(void *arg) +{ + struct drm_i915_private *i915 = arg; + I915_RND_STATE(prng); + IGT_TIMEOUT(end_time); + u32 size, i; + int err; + + i915_gem_drain_freed_objects(i915); + + size = max_t(u32, PAGE_SIZE, i915_prandom_u32_max_state(SZ_32M, &prng)); + size = round_up(size, PAGE_SIZE); + i = 0; + + do { + struct drm_i915_gem_object *obj; + unsigned int flags; + u32 dword, val; + void *vaddr; + + /* + * Alternate between cleared and uncleared allocations, while + * also dirtying the pages each time to check that the pages are + * always cleared if requested, since we should get some overlap + * of the underlying pages, if not all, since we are the only + * user. + */ + + flags = I915_BO_ALLOC_CPU_CLEAR; + if (i & 1) + flags = 0; + + obj = i915_gem_object_create_lmem(i915, size, flags); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + i915_gem_object_lock(obj, NULL); + err = i915_gem_object_pin_pages(obj); + if (err) + goto out_put; + + dword = i915_prandom_u32_max_state(PAGE_SIZE / sizeof(u32), + &prng); + + if (flags & I915_BO_ALLOC_CPU_CLEAR) { + err = igt_cpu_check(obj, dword, 0); + if (err) { + pr_err("%s failed with size=%u, flags=%u\n", + __func__, size, flags); + goto out_unpin; + } + } + + vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + goto out_unpin; + } + + val = prandom_u32_state(&prng); + + memset32(vaddr, val, obj->base.size / sizeof(u32)); + + i915_gem_object_flush_map(obj); + i915_gem_object_unpin_map(obj); +out_unpin: + i915_gem_object_unpin_pages(obj); + __i915_gem_object_put_pages(obj); +out_put: + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); + + if (err) + break; + ++i; + } while (!__igt_timeout(end_time, NULL)); + + pr_info("%s completed (%u) iterations\n", __func__, i); + + return err; +} + static int igt_lmem_write_gpu(void *arg) { struct drm_i915_private *i915 = arg; @@ -1043,6 +1127,7 @@ int intel_memory_region_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_lmem_create), + SUBTEST(igt_lmem_create_cleared_cpu), SUBTEST(igt_lmem_write_cpu), SUBTEST(igt_lmem_write_gpu), }; diff --git a/drivers/gpu/drm/i915/selftests/librapl.c b/drivers/gpu/drm/i915/selftests/librapl.c index 58710ac3f979..eb03b5b28bad 100644 --- a/drivers/gpu/drm/i915/selftests/librapl.c +++ b/drivers/gpu/drm/i915/selftests/librapl.c @@ -5,8 +5,18 @@ #include <asm/msr.h> +#include "i915_drv.h" #include "librapl.h" +bool librapl_supported(const struct drm_i915_private *i915) +{ + /* Discrete cards require hwmon integration */ + if (IS_DGFX(i915)) + return false; + + return librapl_energy_uJ(); +} + u64 librapl_energy_uJ(void) { unsigned long long power; diff --git a/drivers/gpu/drm/i915/selftests/librapl.h b/drivers/gpu/drm/i915/selftests/librapl.h index 887f3e91dd05..e3b24fad0a7a 100644 --- a/drivers/gpu/drm/i915/selftests/librapl.h +++ b/drivers/gpu/drm/i915/selftests/librapl.h @@ -8,6 +8,10 @@ #include <linux/types.h> +struct drm_i915_private; + +bool librapl_supported(const struct drm_i915_private *i915); + u64 librapl_energy_uJ(void); #endif /* SELFTEST_LIBRAPL_H */ diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c index 09225b770bb8..5244f4763477 100644 --- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c +++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c @@ -9,6 +9,7 @@ #include <linux/component.h> #include <linux/clk.h> #include <linux/dma-mapping.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/of_device.h> @@ -23,6 +24,7 @@ #include <drm/drm_color_mgmt.h> #include <drm/drm_crtc.h> #include <drm/drm_crtc_helper.h> +#include <drm/drm_damage_helper.h> #include <drm/drm_drv.h> #include <drm/drm_encoder.h> #include <drm/drm_gem_cma_helper.h> @@ -57,6 +59,7 @@ struct ingenic_dma_hwdescs { struct jz_soc_info { bool needs_dev_clk; bool has_osd; + bool map_noncoherent; unsigned int max_width, max_height; const u32 *formats_f0, *formats_f1; unsigned int num_formats_f0, num_formats_f1; @@ -342,7 +345,7 @@ static void ingenic_drm_crtc_atomic_flush(struct drm_crtc *crtc, if (priv->update_clk_rate) { mutex_lock(&priv->clk_mutex); clk_set_rate(priv->pix_clk, - crtc_state->adjusted_mode.clock * 1000); + crtc_state->adjusted_mode.crtc_clock * 1000); priv->update_clk_rate = false; mutex_unlock(&priv->clk_mutex); } @@ -410,6 +413,9 @@ static int ingenic_drm_plane_atomic_check(struct drm_plane *plane, old_plane_state->fb->format->format != new_plane_state->fb->format->format)) crtc_state->mode_changed = true; + if (priv->soc_info->map_noncoherent) + drm_atomic_helper_check_plane_damage(state, new_plane_state); + return 0; } @@ -526,6 +532,13 @@ void ingenic_drm_plane_config(struct device *dev, } } +bool ingenic_drm_map_noncoherent(const struct device *dev) +{ + const struct ingenic_drm *priv = dev_get_drvdata(dev); + + return priv->soc_info->map_noncoherent; +} + static void ingenic_drm_update_palette(struct ingenic_drm *priv, const struct drm_color_lut *lut) { @@ -544,8 +557,8 @@ static void ingenic_drm_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_state *state) { struct ingenic_drm *priv = drm_device_get_priv(plane->dev); - struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state, - plane); + struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state, plane); + struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state, plane); struct drm_crtc_state *crtc_state; struct ingenic_dma_hwdesc *hwdesc; unsigned int width, height, cpp, offset; @@ -553,6 +566,9 @@ static void ingenic_drm_plane_atomic_update(struct drm_plane *plane, u32 fourcc; if (newstate && newstate->fb) { + if (priv->soc_info->map_noncoherent) + drm_fb_cma_sync_non_coherent(&priv->drm, oldstate, newstate); + crtc_state = newstate->crtc->state; addr = drm_fb_cma_get_gem_addr(newstate->fb, newstate, 0); @@ -742,6 +758,33 @@ static void ingenic_drm_disable_vblank(struct drm_crtc *crtc) regmap_update_bits(priv->map, JZ_REG_LCD_CTRL, JZ_LCD_CTRL_EOF_IRQ, 0); } +static struct drm_framebuffer * +ingenic_drm_gem_fb_create(struct drm_device *drm, struct drm_file *file, + const struct drm_mode_fb_cmd2 *mode_cmd) +{ + struct ingenic_drm *priv = drm_device_get_priv(drm); + + if (priv->soc_info->map_noncoherent) + return drm_gem_fb_create_with_dirty(drm, file, mode_cmd); + + return drm_gem_fb_create(drm, file, mode_cmd); +} + +static struct drm_gem_object * +ingenic_drm_gem_create_object(struct drm_device *drm, size_t size) +{ + struct ingenic_drm *priv = drm_device_get_priv(drm); + struct drm_gem_cma_object *obj; + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) + return ERR_PTR(-ENOMEM); + + obj->map_noncoherent = priv->soc_info->map_noncoherent; + + return &obj->base; +} + DEFINE_DRM_GEM_CMA_FOPS(ingenic_drm_fops); static const struct drm_driver ingenic_drm_driver_data = { @@ -754,6 +797,7 @@ static const struct drm_driver ingenic_drm_driver_data = { .patchlevel = 0, .fops = &ingenic_drm_fops, + .gem_create_object = ingenic_drm_gem_create_object, DRM_GEM_CMA_DRIVER_OPS, .irq_handler = ingenic_drm_irq_handler, @@ -804,7 +848,7 @@ static const struct drm_encoder_helper_funcs ingenic_drm_encoder_helper_funcs = }; static const struct drm_mode_config_funcs ingenic_drm_mode_config_funcs = { - .fb_create = drm_gem_fb_create, + .fb_create = ingenic_drm_gem_fb_create, .output_poll_changed = drm_fb_helper_output_poll_changed, .atomic_check = drm_atomic_helper_check, .atomic_commit = drm_atomic_helper_commit, @@ -961,6 +1005,9 @@ static int ingenic_drm_bind(struct device *dev, bool has_components) return ret; } + if (soc_info->map_noncoherent) + drm_plane_enable_fb_damage_clips(&priv->f1); + drm_crtc_helper_add(&priv->crtc, &ingenic_drm_crtc_helper_funcs); ret = drm_crtc_init_with_planes(drm, &priv->crtc, primary, @@ -989,6 +1036,9 @@ static int ingenic_drm_bind(struct device *dev, bool has_components) return ret; } + if (soc_info->map_noncoherent) + drm_plane_enable_fb_damage_clips(&priv->f0); + if (IS_ENABLED(CONFIG_DRM_INGENIC_IPU) && has_components) { ret = component_bind_all(dev, drm); if (ret) { @@ -1245,6 +1295,7 @@ static const u32 jz4770_formats_f0[] = { static const struct jz_soc_info jz4740_soc_info = { .needs_dev_clk = true, .has_osd = false, + .map_noncoherent = false, .max_width = 800, .max_height = 600, .formats_f1 = jz4740_formats, @@ -1255,6 +1306,7 @@ static const struct jz_soc_info jz4740_soc_info = { static const struct jz_soc_info jz4725b_soc_info = { .needs_dev_clk = false, .has_osd = true, + .map_noncoherent = false, .max_width = 800, .max_height = 600, .formats_f1 = jz4725b_formats_f1, @@ -1266,6 +1318,7 @@ static const struct jz_soc_info jz4725b_soc_info = { static const struct jz_soc_info jz4770_soc_info = { .needs_dev_clk = false, .has_osd = true, + .map_noncoherent = true, .max_width = 1280, .max_height = 720, .formats_f1 = jz4770_formats_f1, diff --git a/drivers/gpu/drm/ingenic/ingenic-drm.h b/drivers/gpu/drm/ingenic/ingenic-drm.h index 1b4347f7f084..22654ac1dde1 100644 --- a/drivers/gpu/drm/ingenic/ingenic-drm.h +++ b/drivers/gpu/drm/ingenic/ingenic-drm.h @@ -184,6 +184,7 @@ struct platform_driver; void ingenic_drm_plane_config(struct device *dev, struct drm_plane *plane, u32 fourcc); void ingenic_drm_plane_disable(struct device *dev, struct drm_plane *plane); +bool ingenic_drm_map_noncoherent(const struct device *dev); extern struct platform_driver *ingenic_ipu_driver_ptr; diff --git a/drivers/gpu/drm/ingenic/ingenic-ipu.c b/drivers/gpu/drm/ingenic/ingenic-ipu.c index 3b1091e7c0cd..61b6d9fdbba1 100644 --- a/drivers/gpu/drm/ingenic/ingenic-ipu.c +++ b/drivers/gpu/drm/ingenic/ingenic-ipu.c @@ -20,10 +20,13 @@ #include <drm/drm_atomic.h> #include <drm/drm_atomic_helper.h> +#include <drm/drm_damage_helper.h> #include <drm/drm_drv.h> #include <drm/drm_fb_cma_helper.h> #include <drm/drm_fourcc.h> #include <drm/drm_gem_atomic_helper.h> +#include <drm/drm_gem_cma_helper.h> +#include <drm/drm_gem_framebuffer_helper.h> #include <drm/drm_plane.h> #include <drm/drm_plane_helper.h> #include <drm/drm_property.h> @@ -285,8 +288,8 @@ static void ingenic_ipu_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_state *state) { struct ingenic_ipu *ipu = plane_to_ingenic_ipu(plane); - struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state, - plane); + struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state, plane); + struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state, plane); const struct drm_format_info *finfo; u32 ctrl, stride = 0, coef_index = 0, format = 0; bool needs_modeset, upscaling_w, upscaling_h; @@ -317,6 +320,9 @@ static void ingenic_ipu_plane_atomic_update(struct drm_plane *plane, JZ_IPU_CTRL_CHIP_EN | JZ_IPU_CTRL_LCDC_SEL); } + if (ingenic_drm_map_noncoherent(ipu->master)) + drm_fb_cma_sync_non_coherent(ipu->drm, oldstate, newstate); + /* New addresses will be committed in vblank handler... */ ipu->addr_y = drm_fb_cma_get_gem_addr(newstate->fb, newstate, 0); if (finfo->num_planes > 1) @@ -541,7 +547,7 @@ static int ingenic_ipu_plane_atomic_check(struct drm_plane *plane, if (!new_plane_state->crtc || !crtc_state->mode.hdisplay || !crtc_state->mode.vdisplay) - return 0; + goto out_check_damage; /* Plane must be fully visible */ if (new_plane_state->crtc_x < 0 || new_plane_state->crtc_y < 0 || @@ -558,7 +564,7 @@ static int ingenic_ipu_plane_atomic_check(struct drm_plane *plane, return -EINVAL; if (!osd_changed(new_plane_state, old_plane_state)) - return 0; + goto out_check_damage; crtc_state->mode_changed = true; @@ -592,6 +598,10 @@ static int ingenic_ipu_plane_atomic_check(struct drm_plane *plane, ipu->denom_w = denom_w; ipu->denom_h = denom_h; +out_check_damage: + if (ingenic_drm_map_noncoherent(ipu->master)) + drm_atomic_helper_check_plane_damage(state, new_plane_state); + return 0; } @@ -773,6 +783,9 @@ static int ingenic_ipu_bind(struct device *dev, struct device *master, void *d) return err; } + if (ingenic_drm_map_noncoherent(master)) + drm_plane_enable_fb_damage_clips(plane); + /* * Sharpness settings range is [0,32] * 0 : nearest-neighbor diff --git a/drivers/gpu/drm/mcde/mcde_dsi.c b/drivers/gpu/drm/mcde/mcde_dsi.c index b3fd3501c412..5aade1d13961 100644 --- a/drivers/gpu/drm/mcde/mcde_dsi.c +++ b/drivers/gpu/drm/mcde/mcde_dsi.c @@ -1196,10 +1196,8 @@ static int mcde_dsi_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); d->regs = devm_ioremap_resource(dev, res); - if (IS_ERR(d->regs)) { - dev_err(dev, "no DSI regs\n"); + if (IS_ERR(d->regs)) return PTR_ERR(d->regs); - } dsi_id = readl(d->regs + DSI_ID_REG); dev_info(dev, "HW revision 0x%08x\n", dsi_id); diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index c46d0374b6e6..f949767698fc 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -1617,8 +1617,9 @@ nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max, mstm->mgr.cbs = &nv50_mstm; ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev, aux, aux_max, - (u8)max_payloads, outp->dcb->dpconf.link_nr, - (u8)outp->dcb->dpconf.link_bw, conn_base_id); + max_payloads, outp->dcb->dpconf.link_nr, + drm_dp_bw_code_to_link_rate(outp->dcb->dpconf.link_bw), + conn_base_id); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 7a2624c0ba4c..c390f24f25f3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1050,15 +1050,6 @@ out_ntfy: return ret; } -static int -nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) -{ - struct nouveau_bo *nvbo = nouveau_bo(bo); - - return drm_vma_node_verify_access(&nvbo->bo.base.vma_node, - filp->private_data); -} - static void nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, struct ttm_resource *reg) @@ -1331,7 +1322,6 @@ struct ttm_device_funcs nouveau_bo_driver = { .evict_flags = nouveau_bo_evict_flags, .delete_mem_notify = nouveau_bo_delete_mem_notify, .move = nouveau_bo_move, - .verify_access = nouveau_bo_verify_access, .io_mem_reserve = &nouveau_ttm_io_mem_reserve, .io_mem_free = &nouveau_ttm_io_mem_free, }; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 3204fc0a90d2..a616cf4573b8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -1179,7 +1179,7 @@ nouveau_driver_fops = { .open = drm_open, .release = drm_release, .unlocked_ioctl = nouveau_drm_ioctl, - .mmap = nouveau_ttm_mmap, + .mmap = drm_gem_mmap, .poll = drm_poll, .read = drm_read, #if defined(CONFIG_COMPAT) @@ -1212,6 +1212,7 @@ driver_stub = { .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, .dumb_create = nouveau_display_dumb_create, .dumb_map_offset = drm_gem_ttm_dumb_map_offset, diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index a70e82413fa7..722e1decc202 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -39,6 +39,40 @@ #include <nvif/class.h> #include <nvif/push206e.h> +static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + pgprot_t prot; + vm_fault_t ret; + + ret = ttm_bo_vm_reserve(bo, vmf); + if (ret) + return ret; + + ret = nouveau_ttm_fault_reserve_notify(bo); + if (ret) + goto error_unlock; + + nouveau_bo_del_io_reserve_lru(bo); + prot = vm_get_page_prot(vma->vm_flags); + ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); + nouveau_bo_add_io_reserve_lru(bo); + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return ret; + +error_unlock: + dma_resv_unlock(bo->base.resv); + return ret; +} + +static const struct vm_operations_struct nouveau_ttm_vm_ops = { + .fault = nouveau_ttm_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access +}; + void nouveau_gem_object_del(struct drm_gem_object *gem) { @@ -180,6 +214,8 @@ const struct drm_gem_object_funcs nouveau_gem_object_funcs = { .get_sg_table = nouveau_gem_prime_get_sg_table, .vmap = drm_gem_ttm_vmap, .vunmap = drm_gem_ttm_vunmap, + .mmap = drm_gem_ttm_mmap, + .vm_ops = &nouveau_ttm_vm_ops, }; int diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index e8b506a6685b..65430912ff72 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -31,8 +31,6 @@ #include "nouveau_mem.h" #include "nouveau_ttm.h" -#include <drm/drm_legacy.h> - #include <core/tegra.h> static void @@ -127,55 +125,6 @@ const struct ttm_resource_manager_func nv04_gart_manager = { .free = nouveau_manager_del, }; -static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - struct ttm_buffer_object *bo = vma->vm_private_data; - pgprot_t prot; - vm_fault_t ret; - - ret = ttm_bo_vm_reserve(bo, vmf); - if (ret) - return ret; - - ret = nouveau_ttm_fault_reserve_notify(bo); - if (ret) - goto error_unlock; - - nouveau_bo_del_io_reserve_lru(bo); - prot = vm_get_page_prot(vma->vm_flags); - ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); - nouveau_bo_add_io_reserve_lru(bo); - if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - return ret; - -error_unlock: - dma_resv_unlock(bo->base.resv); - return ret; -} - -static const struct vm_operations_struct nouveau_ttm_vm_ops = { - .fault = nouveau_ttm_fault, - .open = ttm_bo_vm_open, - .close = ttm_bo_vm_close, - .access = ttm_bo_vm_access -}; - -int -nouveau_ttm_mmap(struct file *filp, struct vm_area_struct *vma) -{ - struct drm_file *file_priv = filp->private_data; - struct nouveau_drm *drm = nouveau_drm(file_priv->minor->dev); - int ret; - - ret = ttm_bo_mmap(filp, vma, &drm->ttm.bdev); - if (ret) - return ret; - - vma->vm_ops = &nouveau_ttm_vm_ops; - return 0; -} - static int nouveau_ttm_init_host(struct nouveau_drm *drm, u8 kind) { diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.h b/drivers/gpu/drm/nouveau/nouveau_ttm.h index dbf6dc238efd..2f0efda7ccdb 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.h +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.h @@ -17,7 +17,6 @@ struct ttm_tt *nouveau_sgdma_create_ttm(struct ttm_buffer_object *bo, int nouveau_ttm_init(struct nouveau_drm *drm); void nouveau_ttm_fini(struct nouveau_drm *drm); -int nouveau_ttm_mmap(struct file *, struct vm_area_struct *); int nouveau_ttm_global_init(struct nouveau_drm *); void nouveau_ttm_global_release(struct nouveau_drm *); diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c index 9c3563c61e8c..07a48f621289 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c @@ -27,7 +27,7 @@ static int s6e63m0_dsi_dcs_read(struct device *dev, const u8 cmd, u8 *data) return ret; } - dev_info(dev, "DSI read CMD %02x = %02x\n", cmd, *data); + dev_dbg(dev, "DSI read CMD %02x = %02x\n", cmd, *data); return 0; } @@ -42,7 +42,7 @@ static int s6e63m0_dsi_dcs_write(struct device *dev, const u8 *data, size_t len) int chunk; int ret; - dev_info(dev, "DSI writing dcs seq: %*ph\n", (int)len, data); + dev_dbg(dev, "DSI writing dcs seq: %*ph\n", (int)len, data); /* Pick out and skip past the DCS command */ cmd = *seqp; @@ -80,7 +80,7 @@ static int s6e63m0_dsi_dcs_write(struct device *dev, const u8 *data, size_t len) cmdwritten += chunk; seqp += chunk; } - dev_info(dev, "sent command %02x %02x bytes\n", cmd, cmdwritten); + dev_dbg(dev, "sent command %02x %02x bytes\n", cmd, cmdwritten); usleep_range(8000, 9000); diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 9be050ab372f..21939d4352cf 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -798,6 +798,7 @@ static int panel_simple_probe(struct device *dev, const struct panel_desc *desc) return 0; disable_pm_runtime: + pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); free_ddc: if (panel->ddc) @@ -814,6 +815,7 @@ static int panel_simple_remove(struct device *dev) drm_panel_disable(&panel->base); drm_panel_unprepare(&panel->base); + pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); if (panel->ddc) put_device(&panel->ddc->dev); diff --git a/drivers/gpu/drm/r128/ati_pcigart.c b/drivers/gpu/drm/r128/ati_pcigart.c index 5d73043446e3..0ecccf25a3c7 100644 --- a/drivers/gpu/drm/r128/ati_pcigart.c +++ b/drivers/gpu/drm/r128/ati_pcigart.c @@ -71,6 +71,8 @@ static void drm_ati_free_pcigart_table(struct drm_device *dev, drm_dma_handle_t *dmah = gart_info->table_handle; dma_free_coherent(dev->dev, dmah->size, dmah->vaddr, dmah->busaddr); + kfree(dmah); + gart_info->table_handle = NULL; } diff --git a/drivers/gpu/drm/r128/r128_drv.h b/drivers/gpu/drm/r128/r128_drv.h index 8b256123cf2b..2e1bc01aa5c9 100644 --- a/drivers/gpu/drm/r128/r128_drv.h +++ b/drivers/gpu/drm/r128/r128_drv.h @@ -29,7 +29,7 @@ * Rickard E. (Rik) Faith <faith@valinux.com> * Kevin E. Martin <martin@valinux.com> * Gareth Hughes <gareth@valinux.com> - * Michel D�zer <daenzerm@student.ethz.ch> + * Michel Dänzer <daenzerm@student.ethz.ch> */ #ifndef __R128_DRV_H__ diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c index 13072c2a6502..ec867fa880a4 100644 --- a/drivers/gpu/drm/radeon/radeon_dp_mst.c +++ b/drivers/gpu/drm/radeon/radeon_dp_mst.c @@ -642,7 +642,7 @@ radeon_dp_mst_init(struct radeon_connector *radeon_connector) radeon_connector->mst_mgr.cbs = &mst_cbs; return drm_dp_mst_topology_mgr_init(&radeon_connector->mst_mgr, dev, &radeon_connector->ddc_bus->aux, 16, 6, - 4, (u8)max_link_rate, + 4, drm_dp_bw_code_to_link_rate(max_link_rate), radeon_connector->base.base.id); } diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c index 31d3dd0e5258..8cd135fa6dcd 100644 --- a/drivers/gpu/drm/radeon/radeon_drv.c +++ b/drivers/gpu/drm/radeon/radeon_drv.c @@ -545,7 +545,7 @@ static const struct file_operations radeon_driver_kms_fops = { .open = drm_open, .release = drm_release, .unlocked_ioctl = radeon_drm_ioctl, - .mmap = radeon_mmap, + .mmap = drm_gem_mmap, .poll = drm_poll, .read = drm_read, #ifdef CONFIG_COMPAT @@ -620,6 +620,7 @@ static const struct drm_driver kms_driver = { .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_import_sg_table = radeon_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, .name = DRIVER_NAME, .desc = DRIVER_DESC, diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index 05ea2f39f626..ff8849827d61 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -44,6 +44,42 @@ void radeon_gem_prime_unpin(struct drm_gem_object *obj); const struct drm_gem_object_funcs radeon_gem_object_funcs; +static vm_fault_t radeon_gem_fault(struct vm_fault *vmf) +{ + struct ttm_buffer_object *bo = vmf->vma->vm_private_data; + struct radeon_device *rdev = radeon_get_rdev(bo->bdev); + vm_fault_t ret; + + down_read(&rdev->pm.mclk_lock); + + ret = ttm_bo_vm_reserve(bo, vmf); + if (ret) + goto unlock_mclk; + + ret = radeon_bo_fault_reserve_notify(bo); + if (ret) + goto unlock_resv; + + ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, + TTM_BO_VM_NUM_PREFAULT, 1); + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + goto unlock_mclk; + +unlock_resv: + dma_resv_unlock(bo->base.resv); + +unlock_mclk: + up_read(&rdev->pm.mclk_lock); + return ret; +} + +static const struct vm_operations_struct radeon_gem_vm_ops = { + .fault = radeon_gem_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access +}; + static void radeon_gem_object_free(struct drm_gem_object *gobj) { struct radeon_bo *robj = gem_to_radeon_bo(gobj); @@ -226,6 +262,17 @@ static int radeon_gem_handle_lockup(struct radeon_device *rdev, int r) return r; } +static int radeon_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + struct radeon_bo *bo = gem_to_radeon_bo(obj); + struct radeon_device *rdev = radeon_get_rdev(bo->tbo.bdev); + + if (radeon_ttm_tt_has_userptr(rdev, bo->tbo.ttm)) + return -EPERM; + + return drm_gem_ttm_mmap(obj, vma); +} + const struct drm_gem_object_funcs radeon_gem_object_funcs = { .free = radeon_gem_object_free, .open = radeon_gem_object_open, @@ -236,6 +283,8 @@ const struct drm_gem_object_funcs radeon_gem_object_funcs = { .get_sg_table = radeon_gem_prime_get_sg_table, .vmap = drm_gem_ttm_vmap, .vunmap = drm_gem_ttm_vunmap, + .mmap = radeon_gem_object_mmap, + .vm_ops = &radeon_gem_vm_ops, }; /* diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 3361d11769a2..a71d94f7067b 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -135,17 +135,6 @@ static void radeon_evict_flags(struct ttm_buffer_object *bo, *placement = rbo->placement; } -static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) -{ - struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); - struct radeon_device *rdev = radeon_get_rdev(bo->bdev); - - if (radeon_ttm_tt_has_userptr(rdev, bo->ttm)) - return -EPERM; - return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, - filp->private_data); -} - static int radeon_move_blit(struct ttm_buffer_object *bo, bool evict, struct ttm_resource *new_mem, @@ -703,7 +692,6 @@ static struct ttm_device_funcs radeon_bo_driver = { .eviction_valuable = ttm_bo_eviction_valuable, .evict_flags = &radeon_evict_flags, .move = &radeon_bo_move, - .verify_access = &radeon_verify_access, .delete_mem_notify = &radeon_bo_delete_mem_notify, .io_mem_reserve = &radeon_ttm_io_mem_reserve, }; @@ -800,59 +788,6 @@ void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) man->size = size >> PAGE_SHIFT; } -static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf) -{ - struct ttm_buffer_object *bo = vmf->vma->vm_private_data; - struct radeon_device *rdev = radeon_get_rdev(bo->bdev); - vm_fault_t ret; - - down_read(&rdev->pm.mclk_lock); - - ret = ttm_bo_vm_reserve(bo, vmf); - if (ret) - goto unlock_mclk; - - ret = radeon_bo_fault_reserve_notify(bo); - if (ret) - goto unlock_resv; - - ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, - TTM_BO_VM_NUM_PREFAULT, 1); - if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - goto unlock_mclk; - -unlock_resv: - dma_resv_unlock(bo->base.resv); - -unlock_mclk: - up_read(&rdev->pm.mclk_lock); - return ret; -} - -static const struct vm_operations_struct radeon_ttm_vm_ops = { - .fault = radeon_ttm_fault, - .open = ttm_bo_vm_open, - .close = ttm_bo_vm_close, - .access = ttm_bo_vm_access -}; - -int radeon_mmap(struct file *filp, struct vm_area_struct *vma) -{ - int r; - struct drm_file *file_priv = filp->private_data; - struct radeon_device *rdev = file_priv->minor->dev->dev_private; - - if (rdev == NULL) - return -EINVAL; - - r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); - if (unlikely(r != 0)) - return r; - - vma->vm_ops = &radeon_ttm_vm_ops; - return 0; -} - #if defined(CONFIG_DEBUG_FS) static int radeon_mm_vram_dump_table_show(struct seq_file *m, void *unused) diff --git a/drivers/gpu/drm/radeon/radeon_ttm.h b/drivers/gpu/drm/radeon/radeon_ttm.h index 4d7b90ee2774..91ea7141bc81 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.h +++ b/drivers/gpu/drm/radeon/radeon_ttm.h @@ -32,6 +32,5 @@ struct radeon_device; int radeon_ttm_init(struct radeon_device *rdev); void radeon_ttm_fini(struct radeon_device *rdev); -int radeon_mmap(struct file *filp, struct vm_area_struct *vma); #endif /* __RADEON_TTM_H__ */ diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c index a4a45daf93f2..8ab3247dbc4a 100644 --- a/drivers/gpu/drm/rockchip/cdn-dp-core.c +++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c @@ -42,6 +42,7 @@ #define CDN_FW_TIMEOUT_MS (64 * 1000) #define CDN_DPCD_TIMEOUT_MS 5000 #define CDN_DP_FIRMWARE "rockchip/dptx.bin" +MODULE_FIRMWARE(CDN_DP_FIRMWARE); struct cdn_dp_data { u8 max_phy; @@ -73,6 +74,7 @@ static int cdn_dp_grf_write(struct cdn_dp_device *dp, ret = regmap_write(dp->grf, reg, val); if (ret) { DRM_DEV_ERROR(dp->dev, "Could not write to GRF: %d\n", ret); + clk_disable_unprepare(dp->grf_clk); return ret; } diff --git a/drivers/gpu/drm/rockchip/cdn-dp-reg.c b/drivers/gpu/drm/rockchip/cdn-dp-reg.c index 9d2163ef4d6e..33fb4d05c506 100644 --- a/drivers/gpu/drm/rockchip/cdn-dp-reg.c +++ b/drivers/gpu/drm/rockchip/cdn-dp-reg.c @@ -658,7 +658,7 @@ int cdn_dp_config_video(struct cdn_dp_device *dp) */ do { tu_size_reg += 2; - symbol = tu_size_reg * mode->clock * bit_per_pix; + symbol = (u64)tu_size_reg * mode->clock * bit_per_pix; do_div(symbol, dp->max_lanes * link_rate * 8); rem = do_div(symbol, 1000); if (tu_size_reg > 64) { diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c index 24a71091759c..ec7729d18cb8 100644 --- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c @@ -243,7 +243,6 @@ struct dw_mipi_dsi_rockchip { struct dw_mipi_dsi *dmd; const struct rockchip_dw_dsi_chip_data *cdata; struct dw_mipi_dsi_plat_data pdata; - int devcnt; }; struct dphy_pll_parameter_map { @@ -317,11 +316,6 @@ static inline u32 dsi_read(struct dw_mipi_dsi_rockchip *dsi, u32 reg) return readl(dsi->base + reg); } -static inline void dsi_set(struct dw_mipi_dsi_rockchip *dsi, u32 reg, u32 mask) -{ - dsi_write(dsi, reg, dsi_read(dsi, reg) | mask); -} - static inline void dsi_update_bits(struct dw_mipi_dsi_rockchip *dsi, u32 reg, u32 mask, u32 val) { @@ -692,13 +686,8 @@ static const struct dw_mipi_dsi_phy_ops dw_mipi_dsi_rockchip_phy_ops = { .get_timing = dw_mipi_dsi_phy_get_timing, }; -static void dw_mipi_dsi_rockchip_config(struct dw_mipi_dsi_rockchip *dsi, - int mux) +static void dw_mipi_dsi_rockchip_config(struct dw_mipi_dsi_rockchip *dsi) { - if (dsi->cdata->lcdsel_grf_reg) - regmap_write(dsi->grf_regmap, dsi->cdata->lcdsel_grf_reg, - mux ? dsi->cdata->lcdsel_lit : dsi->cdata->lcdsel_big); - if (dsi->cdata->lanecfg1_grf_reg) regmap_write(dsi->grf_regmap, dsi->cdata->lanecfg1_grf_reg, dsi->cdata->lanecfg1); @@ -712,6 +701,13 @@ static void dw_mipi_dsi_rockchip_config(struct dw_mipi_dsi_rockchip *dsi, dsi->cdata->enable); } +static void dw_mipi_dsi_rockchip_set_lcdsel(struct dw_mipi_dsi_rockchip *dsi, + int mux) +{ + regmap_write(dsi->grf_regmap, dsi->cdata->lcdsel_grf_reg, + mux ? dsi->cdata->lcdsel_lit : dsi->cdata->lcdsel_big); +} + static int dw_mipi_dsi_encoder_atomic_check(struct drm_encoder *encoder, struct drm_crtc_state *crtc_state, @@ -767,9 +763,9 @@ static void dw_mipi_dsi_encoder_enable(struct drm_encoder *encoder) return; } - dw_mipi_dsi_rockchip_config(dsi, mux); + dw_mipi_dsi_rockchip_set_lcdsel(dsi, mux); if (dsi->slave) - dw_mipi_dsi_rockchip_config(dsi->slave, mux); + dw_mipi_dsi_rockchip_set_lcdsel(dsi->slave, mux); clk_disable_unprepare(dsi->grf_clk); } @@ -923,6 +919,24 @@ static int dw_mipi_dsi_rockchip_bind(struct device *dev, return ret; } + /* + * With the GRF clock running, write lane and dual-mode configurations + * that won't change immediately. If we waited until enable() to do + * this, things like panel preparation would not be able to send + * commands over DSI. + */ + ret = clk_prepare_enable(dsi->grf_clk); + if (ret) { + DRM_DEV_ERROR(dsi->dev, "Failed to enable grf_clk: %d\n", ret); + return ret; + } + + dw_mipi_dsi_rockchip_config(dsi); + if (dsi->slave) + dw_mipi_dsi_rockchip_config(dsi->slave); + + clk_disable_unprepare(dsi->grf_clk); + ret = rockchip_dsi_drm_create_encoder(dsi, drm_dev); if (ret) { DRM_DEV_ERROR(dev, "Failed to create drm encoder\n"); @@ -1121,9 +1135,6 @@ static int dw_mipi_dsi_rockchip_remove(struct platform_device *pdev) { struct dw_mipi_dsi_rockchip *dsi = platform_get_drvdata(pdev); - if (dsi->devcnt == 0) - component_del(dsi->dev, &dw_mipi_dsi_rockchip_ops); - dw_mipi_dsi_remove(dsi->dmd); return 0; diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c index 212bd87c0c4a..b730b8d5d949 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c @@ -16,6 +16,7 @@ #include <linux/console.h> #include <linux/iommu.h> +#include <drm/drm_aperture.h> #include <drm/drm_drv.h> #include <drm/drm_fb_helper.h> #include <drm/drm_gem_cma_helper.h> @@ -114,6 +115,15 @@ static int rockchip_drm_bind(struct device *dev) struct rockchip_drm_private *private; int ret; + /* Remove existing drivers that may own the framebuffer memory. */ + ret = drm_aperture_remove_framebuffers(false, "rockchip-drm-fb"); + if (ret) { + DRM_DEV_ERROR(dev, + "Failed to remove existing framebuffers - %d.\n", + ret); + return ret; + } + drm_dev = drm_dev_alloc(&rockchip_drm_driver, dev); if (IS_ERR(drm_dev)) return PTR_ERR(drm_dev); diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c index 64469439ddf2..f5b9028a16a3 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c @@ -1022,6 +1022,7 @@ static void vop_plane_atomic_update(struct drm_plane *plane, VOP_WIN_SET(vop, win, alpha_en, 1); } else { VOP_WIN_SET(vop, win, src_alpha_ctl, SRC_ALPHA_EN(0)); + VOP_WIN_SET(vop, win, alpha_en, 0); } VOP_WIN_SET(vop, win, enable, 1); diff --git a/drivers/gpu/drm/rockchip/rockchip_lvds.c b/drivers/gpu/drm/rockchip/rockchip_lvds.c index bd5ba10822c2..489d63c05c0d 100644 --- a/drivers/gpu/drm/rockchip/rockchip_lvds.c +++ b/drivers/gpu/drm/rockchip/rockchip_lvds.c @@ -499,11 +499,11 @@ static int px30_lvds_probe(struct platform_device *pdev, if (IS_ERR(lvds->dphy)) return PTR_ERR(lvds->dphy); - phy_init(lvds->dphy); + ret = phy_init(lvds->dphy); if (ret) return ret; - phy_set_mode(lvds->dphy, PHY_MODE_LVDS); + ret = phy_set_mode(lvds->dphy, PHY_MODE_LVDS); if (ret) return ret; diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c index 80053d91a301..ca7cc82125cb 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c @@ -77,15 +77,20 @@ static const uint64_t format_modifiers_win_lite[] = { DRM_FORMAT_MOD_INVALID, }; -static const struct vop_scl_regs rk3036_win_scl = { +static const struct vop_scl_regs rk3036_win0_scl = { .scale_yrgb_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0), .scale_yrgb_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 16), .scale_cbcr_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 0x0), .scale_cbcr_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 16), }; +static const struct vop_scl_regs rk3036_win1_scl = { + .scale_yrgb_x = VOP_REG(RK3036_WIN1_SCL_FACTOR_YRGB, 0xffff, 0x0), + .scale_yrgb_y = VOP_REG(RK3036_WIN1_SCL_FACTOR_YRGB, 0xffff, 16), +}; + static const struct vop_win_phy rk3036_win0_data = { - .scl = &rk3036_win_scl, + .scl = &rk3036_win0_scl, .data_formats = formats_win_full, .nformats = ARRAY_SIZE(formats_win_full), .format_modifiers = format_modifiers_win_full, @@ -99,9 +104,13 @@ static const struct vop_win_phy rk3036_win0_data = { .uv_mst = VOP_REG(RK3036_WIN0_CBR_MST, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3036_WIN0_VIR, 0xffff, 0), .uv_vir = VOP_REG(RK3036_WIN0_VIR, 0x1fff, 16), + .alpha_mode = VOP_REG(RK3036_DSP_CTRL0, 0x1, 18), + .alpha_en = VOP_REG(RK3036_ALPHA_CTRL, 0x1, 0), + .alpha_pre_mul = VOP_REG(RK3036_DSP_CTRL0, 0x1, 29), }; static const struct vop_win_phy rk3036_win1_data = { + .scl = &rk3036_win1_scl, .data_formats = formats_win_lite, .nformats = ARRAY_SIZE(formats_win_lite), .format_modifiers = format_modifiers_win_lite, @@ -113,6 +122,9 @@ static const struct vop_win_phy rk3036_win1_data = { .dsp_st = VOP_REG(RK3036_WIN1_DSP_ST, 0x1fff1fff, 0), .yrgb_mst = VOP_REG(RK3036_WIN1_MST, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3036_WIN1_VIR, 0xffff, 0), + .alpha_mode = VOP_REG(RK3036_DSP_CTRL0, 0x1, 19), + .alpha_en = VOP_REG(RK3036_ALPHA_CTRL, 0x1, 1), + .alpha_pre_mul = VOP_REG(RK3036_DSP_CTRL0, 0x1, 29), }; static const struct vop_win_data rk3036_vop_win_data[] = { @@ -179,6 +191,9 @@ static const struct vop_win_phy rk3126_win1_data = { .dsp_st = VOP_REG(RK3126_WIN1_DSP_ST, 0x1fff1fff, 0), .yrgb_mst = VOP_REG(RK3126_WIN1_MST, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3036_WIN1_VIR, 0xffff, 0), + .alpha_mode = VOP_REG(RK3036_DSP_CTRL0, 0x1, 19), + .alpha_en = VOP_REG(RK3036_ALPHA_CTRL, 0x1, 1), + .alpha_pre_mul = VOP_REG(RK3036_DSP_CTRL0, 0x1, 29), }; static const struct vop_win_data rk3126_vop_win_data[] = { @@ -312,6 +327,7 @@ static const struct vop_win_data px30_vop_big_win_data[] = { }; static const struct vop_data px30_vop_big = { + .version = VOP_VERSION(2, 6), .intr = &px30_intr, .feature = VOP_FEATURE_INTERNAL_RGB, .common = &px30_common, @@ -327,6 +343,7 @@ static const struct vop_win_data px30_vop_lit_win_data[] = { }; static const struct vop_data px30_vop_lit = { + .version = VOP_VERSION(2, 5), .intr = &px30_intr, .feature = VOP_FEATURE_INTERNAL_RGB, .common = &px30_common, @@ -349,8 +366,8 @@ static const struct vop_win_phy rk3066_win0_data = { .nformats = ARRAY_SIZE(formats_win_full), .format_modifiers = format_modifiers_win_full, .enable = VOP_REG(RK3066_SYS_CTRL1, 0x1, 0), - .format = VOP_REG(RK3066_SYS_CTRL0, 0x7, 4), - .rb_swap = VOP_REG(RK3066_SYS_CTRL0, 0x1, 19), + .format = VOP_REG(RK3066_SYS_CTRL1, 0x7, 4), + .rb_swap = VOP_REG(RK3066_SYS_CTRL1, 0x1, 19), .act_info = VOP_REG(RK3066_WIN0_ACT_INFO, 0x1fff1fff, 0), .dsp_info = VOP_REG(RK3066_WIN0_DSP_INFO, 0x0fff0fff, 0), .dsp_st = VOP_REG(RK3066_WIN0_DSP_ST, 0x1fff1fff, 0), @@ -358,16 +375,17 @@ static const struct vop_win_phy rk3066_win0_data = { .uv_mst = VOP_REG(RK3066_WIN0_CBR_MST0, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3066_WIN0_VIR, 0xffff, 0), .uv_vir = VOP_REG(RK3066_WIN0_VIR, 0x1fff, 16), + .alpha_mode = VOP_REG(RK3066_DSP_CTRL0, 0x1, 21), + .alpha_en = VOP_REG(RK3066_BLEND_CTRL, 0x1, 0), }; static const struct vop_win_phy rk3066_win1_data = { - .scl = &rk3066_win_scl, .data_formats = formats_win_full, .nformats = ARRAY_SIZE(formats_win_full), .format_modifiers = format_modifiers_win_full, .enable = VOP_REG(RK3066_SYS_CTRL1, 0x1, 1), - .format = VOP_REG(RK3066_SYS_CTRL0, 0x7, 7), - .rb_swap = VOP_REG(RK3066_SYS_CTRL0, 0x1, 23), + .format = VOP_REG(RK3066_SYS_CTRL1, 0x7, 7), + .rb_swap = VOP_REG(RK3066_SYS_CTRL1, 0x1, 23), .act_info = VOP_REG(RK3066_WIN1_ACT_INFO, 0x1fff1fff, 0), .dsp_info = VOP_REG(RK3066_WIN1_DSP_INFO, 0x0fff0fff, 0), .dsp_st = VOP_REG(RK3066_WIN1_DSP_ST, 0x1fff1fff, 0), @@ -375,6 +393,8 @@ static const struct vop_win_phy rk3066_win1_data = { .uv_mst = VOP_REG(RK3066_WIN1_CBR_MST, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3066_WIN1_VIR, 0xffff, 0), .uv_vir = VOP_REG(RK3066_WIN1_VIR, 0x1fff, 16), + .alpha_mode = VOP_REG(RK3066_DSP_CTRL0, 0x1, 22), + .alpha_en = VOP_REG(RK3066_BLEND_CTRL, 0x1, 1), }; static const struct vop_win_phy rk3066_win2_data = { @@ -382,12 +402,14 @@ static const struct vop_win_phy rk3066_win2_data = { .nformats = ARRAY_SIZE(formats_win_lite), .format_modifiers = format_modifiers_win_lite, .enable = VOP_REG(RK3066_SYS_CTRL1, 0x1, 2), - .format = VOP_REG(RK3066_SYS_CTRL0, 0x7, 10), - .rb_swap = VOP_REG(RK3066_SYS_CTRL0, 0x1, 27), + .format = VOP_REG(RK3066_SYS_CTRL1, 0x7, 10), + .rb_swap = VOP_REG(RK3066_SYS_CTRL1, 0x1, 27), .dsp_info = VOP_REG(RK3066_WIN2_DSP_INFO, 0x0fff0fff, 0), .dsp_st = VOP_REG(RK3066_WIN2_DSP_ST, 0x1fff1fff, 0), .yrgb_mst = VOP_REG(RK3066_WIN2_MST, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3066_WIN2_VIR, 0xffff, 0), + .alpha_mode = VOP_REG(RK3066_DSP_CTRL0, 0x1, 23), + .alpha_en = VOP_REG(RK3066_BLEND_CTRL, 0x1, 2), }; static const struct vop_modeset rk3066_modeset = { @@ -408,6 +430,9 @@ static const struct vop_common rk3066_common = { .dither_down_en = VOP_REG(RK3066_DSP_CTRL0, 0x1, 11), .dither_down_mode = VOP_REG(RK3066_DSP_CTRL0, 0x1, 10), .dsp_blank = VOP_REG(RK3066_DSP_CTRL1, 0x1, 24), + .dither_up = VOP_REG(RK3066_DSP_CTRL0, 0x1, 9), + .dsp_lut_en = VOP_REG(RK3066_SYS_CTRL1, 0x1, 31), + .data_blank = VOP_REG(RK3066_DSP_CTRL1, 0x1, 25), }; static const struct vop_win_data rk3066_vop_win_data[] = { @@ -470,6 +495,9 @@ static const struct vop_win_phy rk3188_win0_data = { .yrgb_mst = VOP_REG(RK3188_WIN0_YRGB_MST0, 0xffffffff, 0), .uv_mst = VOP_REG(RK3188_WIN0_CBR_MST0, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3188_WIN_VIR, 0x1fff, 0), + .alpha_mode = VOP_REG(RK3188_DSP_CTRL0, 0x1, 18), + .alpha_en = VOP_REG(RK3188_ALPHA_CTRL, 0x1, 0), + .alpha_pre_mul = VOP_REG(RK3188_DSP_CTRL0, 0x1, 29), }; static const struct vop_win_phy rk3188_win1_data = { @@ -484,6 +512,9 @@ static const struct vop_win_phy rk3188_win1_data = { .dsp_st = VOP_REG(RK3188_WIN1_DSP_ST, 0x0fff0fff, 0), .yrgb_mst = VOP_REG(RK3188_WIN1_MST, 0xffffffff, 0), .yrgb_vir = VOP_REG(RK3188_WIN_VIR, 0x1fff, 16), + .alpha_mode = VOP_REG(RK3188_DSP_CTRL0, 0x1, 19), + .alpha_en = VOP_REG(RK3188_ALPHA_CTRL, 0x1, 1), + .alpha_pre_mul = VOP_REG(RK3188_DSP_CTRL0, 0x1, 29), }; static const struct vop_modeset rk3188_modeset = { @@ -505,7 +536,10 @@ static const struct vop_common rk3188_common = { .dither_down_sel = VOP_REG(RK3188_DSP_CTRL0, 0x1, 27), .dither_down_en = VOP_REG(RK3188_DSP_CTRL0, 0x1, 11), .dither_down_mode = VOP_REG(RK3188_DSP_CTRL0, 0x1, 10), - .dsp_blank = VOP_REG(RK3188_DSP_CTRL1, 0x3, 24), + .dsp_blank = VOP_REG(RK3188_DSP_CTRL1, 0x1, 24), + .dither_up = VOP_REG(RK3188_DSP_CTRL0, 0x1, 9), + .dsp_lut_en = VOP_REG(RK3188_SYS_CTRL, 0x1, 28), + .data_blank = VOP_REG(RK3188_DSP_CTRL1, 0x1, 25), }; static const struct vop_win_data rk3188_vop_win_data[] = { diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h index 6e9fa5815d4d..0b3cd65ba5c1 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h +++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h @@ -955,6 +955,7 @@ #define RK3188_DSP_CTRL0 0x04 #define RK3188_DSP_CTRL1 0x08 #define RK3188_INT_STATUS 0x10 +#define RK3188_ALPHA_CTRL 0x14 #define RK3188_WIN0_YRGB_MST0 0x20 #define RK3188_WIN0_CBR_MST0 0x24 #define RK3188_WIN0_YRGB_MST1 0x28 diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 0249c7450188..79554aa4dbb1 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -116,7 +116,8 @@ static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity) rmb(); /* for list_empty to work without lock */ if (list_empty(&entity->list) || - spsc_queue_count(&entity->job_queue) == 0) + spsc_queue_count(&entity->job_queue) == 0 || + entity->stopped) return true; return false; @@ -221,11 +222,16 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) { struct drm_sched_job *job; + struct dma_fence *f; int r; while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { struct drm_sched_fence *s_fence = job->s_fence; + /* Wait for all dependencies to avoid data corruptions */ + while ((f = job->sched->ops->dependency(job, entity))) + dma_fence_wait(f, false); + drm_sched_fence_scheduled(s_fence); dma_fence_set_error(&s_fence->finished, -ESRCH); diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index f4f474944169..a2a953693b45 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work) { struct drm_gpu_scheduler *sched; struct drm_sched_job *job; + enum drm_gpu_sched_stat status = DRM_GPU_SCHED_STAT_NOMINAL; sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work); @@ -331,7 +332,7 @@ static void drm_sched_job_timedout(struct work_struct *work) list_del_init(&job->list); spin_unlock(&sched->job_list_lock); - job->sched->ops->timedout_job(job); + status = job->sched->ops->timedout_job(job); /* * Guilty job did complete and hence needs to be manually removed @@ -345,9 +346,11 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock(&sched->job_list_lock); } - spin_lock(&sched->job_list_lock); - drm_sched_start_timeout(sched); - spin_unlock(&sched->job_list_lock); + if (status != DRM_GPU_SCHED_STAT_ENODEV) { + spin_lock(&sched->job_list_lock); + drm_sched_start_timeout(sched); + spin_unlock(&sched->job_list_lock); + } } /** @@ -895,9 +898,33 @@ EXPORT_SYMBOL(drm_sched_init); */ void drm_sched_fini(struct drm_gpu_scheduler *sched) { + struct drm_sched_entity *s_entity; + int i; + if (sched->thread) kthread_stop(sched->thread); + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + struct drm_sched_rq *rq = &sched->sched_rq[i]; + + if (!rq) + continue; + + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) + /* + * Prevents reinsertion and marks job_queue as idle, + * it will removed from rq in drm_sched_entity_fini + * eventually + */ + s_entity->stopped = true; + spin_unlock(&rq->lock); + + } + + /* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */ + wake_up_all(&sched->job_scheduled); + /* Confirm no work left behind accessing device structures */ cancel_delayed_work_sync(&sched->work_tdr); diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c index 4f605c5fe856..08ae66b1e6f5 100644 --- a/drivers/gpu/drm/tiny/simpledrm.c +++ b/drivers/gpu/drm/tiny/simpledrm.c @@ -468,7 +468,6 @@ static int simpledrm_device_init_fb(struct simpledrm_device *sdev) { int width, height, stride; const struct drm_format_info *format; - struct drm_format_name_buf buf; struct drm_device *dev = &sdev->dev; struct platform_device *pdev = sdev->pdev; const struct simplefb_platform_data *pd = dev_get_platdata(&pdev->dev); @@ -512,9 +511,8 @@ static int simpledrm_device_init_fb(struct simpledrm_device *sdev) drm_dbg_kms(dev, "display mode={" DRM_MODE_FMT "}\n", DRM_MODE_ARG(&sdev->mode)); drm_dbg_kms(dev, - "framebuffer format=\"%s\", size=%dx%d, stride=%d byte\n", - drm_get_format_name(format->format, &buf), width, - height, stride); + "framebuffer format=%p4cc, size=%dx%d, stride=%d byte\n", + &format->format, width, height, stride); return 0; } diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index ca1b098b6a56..51a94fd63bd7 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -682,7 +682,9 @@ int ttm_mem_evict_first(struct ttm_device *bdev, } /* - * Add the last move fence to the BO and reserve a new shared slot. + * Add the last move fence to the BO and reserve a new shared slot. We only use + * a shared slot to avoid unecessary sync and rely on the subsequent bo move to + * either stall or use an exclusive fence respectively set bo->moving. */ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, struct ttm_resource_manager *man, diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index b31b18058965..9bd15cb39145 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -34,6 +34,8 @@ #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <drm/drm_vma_manager.h> +#include <drm/drm_drv.h> +#include <drm/drm_managed.h> #include <linux/mm.h> #include <linux/pfn_t.h> #include <linux/rbtree.h> @@ -380,19 +382,69 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, } EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); +static void ttm_bo_release_dummy_page(struct drm_device *dev, void *res) +{ + struct page *dummy_page = (struct page *)res; + + __free_page(dummy_page); +} + +vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + struct drm_device *ddev = bo->base.dev; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address; + unsigned long pfn; + struct page *page; + + /* Allocate new dummy page to map all the VA range in this VMA to it*/ + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + /* Set the page to be freed using drmm release action */ + if (drmm_add_action_or_reset(ddev, ttm_bo_release_dummy_page, page)) + return VM_FAULT_OOM; + + pfn = page_to_pfn(page); + + /* Prefault the entire VMA range right away to avoid further faults */ + for (address = vma->vm_start; address < vma->vm_end; address += PAGE_SIZE) { + + if (vma->vm_flags & VM_MIXEDMAP) + ret = vmf_insert_mixed_prot(vma, address, + __pfn_to_pfn_t(pfn, PFN_DEV), + prot); + else + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + } + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_dummy_page); + vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; pgprot_t prot; struct ttm_buffer_object *bo = vma->vm_private_data; + struct drm_device *ddev = bo->base.dev; vm_fault_t ret; + int idx; ret = ttm_bo_vm_reserve(bo, vmf); if (ret) return ret; prot = vma->vm_page_prot; - ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); + if (drm_dev_enter(ddev, &idx)) { + ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); + drm_dev_exit(idx); + } else { + ret = ttm_bo_vm_dummy_page(vmf, prot); + } if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) return ret; @@ -508,33 +560,14 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { .access = ttm_bo_vm_access, }; -static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_device *bdev, - unsigned long offset, - unsigned long pages) -{ - struct drm_vma_offset_node *node; - struct ttm_buffer_object *bo = NULL; - - drm_vma_offset_lock_lookup(bdev->vma_manager); - - node = drm_vma_offset_lookup_locked(bdev->vma_manager, offset, pages); - if (likely(node)) { - bo = container_of(node, struct ttm_buffer_object, - base.vma_node); - bo = ttm_bo_get_unless_zero(bo); - } - - drm_vma_offset_unlock_lookup(bdev->vma_manager); - - if (!bo) - pr_err("Could not find buffer object to map\n"); - - return bo; -} - static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_struct *vma) { - vma->vm_ops = &ttm_bo_vm_ops; + /* + * Drivers may want to override the vm_ops field. Otherwise we + * use TTM's default callbacks. + */ + if (!vma->vm_ops) + vma->vm_ops = &ttm_bo_vm_ops; /* * Note: We're transferring the bo reference to @@ -554,35 +587,6 @@ static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_s vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; } -int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, - struct ttm_device *bdev) -{ - struct ttm_buffer_object *bo; - int ret; - - if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START)) - return -EINVAL; - - bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); - if (unlikely(!bo)) - return -EINVAL; - - if (unlikely(!bo->bdev->funcs->verify_access)) { - ret = -EPERM; - goto out_unref; - } - ret = bo->bdev->funcs->verify_access(bo, filp); - if (unlikely(ret != 0)) - goto out_unref; - - ttm_bo_mmap_vma_setup(bo, vma); - return 0; -out_unref: - ttm_bo_put(bo); - return ret; -} -EXPORT_SYMBOL(ttm_bo_mmap); - int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { ttm_bo_get(bo); diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c index 76657dcdf9b0..18f5009ce90e 100644 --- a/drivers/gpu/drm/vc4/vc4_crtc.c +++ b/drivers/gpu/drm/vc4/vc4_crtc.c @@ -279,14 +279,22 @@ static u32 vc4_crtc_get_fifo_full_level_bits(struct vc4_crtc *vc4_crtc, * allows drivers to push pixels to more than one encoder from the * same CRTC. */ -static struct drm_encoder *vc4_get_crtc_encoder(struct drm_crtc *crtc) +static struct drm_encoder *vc4_get_crtc_encoder(struct drm_crtc *crtc, + struct drm_atomic_state *state, + struct drm_connector_state *(*get_state)(struct drm_atomic_state *state, + struct drm_connector *connector)) { struct drm_connector *connector; struct drm_connector_list_iter conn_iter; drm_connector_list_iter_begin(crtc->dev, &conn_iter); drm_for_each_connector_iter(connector, &conn_iter) { - if (connector->state->crtc == crtc) { + struct drm_connector_state *conn_state = get_state(state, connector); + + if (!conn_state) + continue; + + if (conn_state->crtc == crtc) { drm_connector_list_iter_end(&conn_iter); return connector->encoder; } @@ -305,16 +313,17 @@ static void vc4_crtc_pixelvalve_reset(struct drm_crtc *crtc) CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) | PV_CONTROL_FIFO_CLR); } -static void vc4_crtc_config_pv(struct drm_crtc *crtc) +static void vc4_crtc_config_pv(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct drm_device *dev = crtc->dev; struct vc4_dev *vc4 = to_vc4_dev(dev); - struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc); + struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc, state, + drm_atomic_get_new_connector_state); struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder); struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc); const struct vc4_pv_data *pv_data = vc4_crtc_to_vc4_pv_data(vc4_crtc); - struct drm_crtc_state *state = crtc->state; - struct drm_display_mode *mode = &state->adjusted_mode; + struct drm_crtc_state *crtc_state = crtc->state; + struct drm_display_mode *mode = &crtc_state->adjusted_mode; bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE; u32 pixel_rep = (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1; bool is_dsi = (vc4_encoder->type == VC4_ENCODER_TYPE_DSI0 || @@ -421,10 +430,10 @@ static void require_hvs_enabled(struct drm_device *dev) } static int vc4_crtc_disable(struct drm_crtc *crtc, + struct drm_encoder *encoder, struct drm_atomic_state *state, unsigned int channel) { - struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc); struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder); struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc); struct drm_device *dev = crtc->dev; @@ -465,10 +474,29 @@ static int vc4_crtc_disable(struct drm_crtc *crtc, return 0; } +static struct drm_encoder *vc4_crtc_get_encoder_by_type(struct drm_crtc *crtc, + enum vc4_encoder_type type) +{ + struct drm_encoder *encoder; + + drm_for_each_encoder(encoder, crtc->dev) { + struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder); + + if (vc4_encoder->type == type) + return encoder; + } + + return NULL; +} + int vc4_crtc_disable_at_boot(struct drm_crtc *crtc) { struct drm_device *drm = crtc->dev; struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc); + enum vc4_encoder_type encoder_type; + const struct vc4_pv_data *pv_data; + struct drm_encoder *encoder; + unsigned encoder_sel; int channel; if (!(of_device_is_compatible(vc4_crtc->pdev->dev.of_node, @@ -487,7 +515,17 @@ int vc4_crtc_disable_at_boot(struct drm_crtc *crtc) if (channel < 0) return 0; - return vc4_crtc_disable(crtc, NULL, channel); + encoder_sel = VC4_GET_FIELD(CRTC_READ(PV_CONTROL), PV_CONTROL_CLK_SELECT); + if (WARN_ON(encoder_sel != 0)) + return 0; + + pv_data = vc4_crtc_to_vc4_pv_data(vc4_crtc); + encoder_type = pv_data->encoder_types[encoder_sel]; + encoder = vc4_crtc_get_encoder_by_type(crtc, encoder_type); + if (WARN_ON(!encoder)) + return 0; + + return vc4_crtc_disable(crtc, encoder, NULL, channel); } static void vc4_crtc_atomic_disable(struct drm_crtc *crtc, @@ -496,6 +534,8 @@ static void vc4_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state, crtc); struct vc4_crtc_state *old_vc4_state = to_vc4_crtc_state(old_state); + struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc, state, + drm_atomic_get_old_connector_state); struct drm_device *dev = crtc->dev; require_hvs_enabled(dev); @@ -503,7 +543,7 @@ static void vc4_crtc_atomic_disable(struct drm_crtc *crtc, /* Disable vblank irq handling before crtc is disabled. */ drm_crtc_vblank_off(crtc); - vc4_crtc_disable(crtc, state, old_vc4_state->assigned_channel); + vc4_crtc_disable(crtc, encoder, state, old_vc4_state->assigned_channel); /* * Make sure we issue a vblank event after disabling the CRTC if @@ -524,7 +564,8 @@ static void vc4_crtc_atomic_enable(struct drm_crtc *crtc, { struct drm_device *dev = crtc->dev; struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc); - struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc); + struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc, state, + drm_atomic_get_new_connector_state); struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder); require_hvs_enabled(dev); @@ -539,7 +580,7 @@ static void vc4_crtc_atomic_enable(struct drm_crtc *crtc, if (vc4_encoder->pre_crtc_configure) vc4_encoder->pre_crtc_configure(encoder, state); - vc4_crtc_config_pv(crtc); + vc4_crtc_config_pv(crtc, state); CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) | PV_CONTROL_EN); @@ -994,7 +1035,7 @@ static const struct vc4_pv_data bcm2711_pv3_data = { .fifo_depth = 64, .pixels_per_clock = 1, .encoder_types = { - [0] = VC4_ENCODER_TYPE_VEC, + [PV_CONTROL_CLK_SELECT_VEC] = VC4_ENCODER_TYPE_VEC, }, }; @@ -1035,6 +1076,9 @@ static void vc4_set_crtc_possible_masks(struct drm_device *drm, struct vc4_encoder *vc4_encoder; int i; + if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL) + continue; + vc4_encoder = to_vc4_encoder(encoder); for (i = 0; i < ARRAY_SIZE(pv_data->encoder_types); i++) { if (vc4_encoder->type == encoder_types[i]) { diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c index 9eff45b48869..8a60fb8ad370 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.c +++ b/drivers/gpu/drm/vc4/vc4_drv.c @@ -304,12 +304,21 @@ static const struct component_master_ops vc4_drm_ops = { .unbind = vc4_drm_unbind, }; +/* + * This list determines the binding order of our components, and we have + * a few constraints: + * - The TXP driver needs to be bound before the PixelValves (CRTC) + * but after the HVS to set the possible_crtc field properly + * - The HDMI driver needs to be bound after the HVS so that we can + * lookup the HVS maximum core clock rate and figure out if we + * support 4kp60 or not. + */ static struct platform_driver *const component_drivers[] = { + &vc4_hvs_driver, &vc4_hdmi_driver, &vc4_vec_driver, &vc4_dpi_driver, &vc4_dsi_driver, - &vc4_hvs_driver, &vc4_txp_driver, &vc4_crtc_driver, &vc4_v3d_driver, diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index c27b287d2053..f9de8632a28b 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -35,6 +35,7 @@ #include <drm/drm_edid.h> #include <drm/drm_probe_helper.h> #include <drm/drm_simple_kms_helper.h> +#include <drm/drm_scdc_helper.h> #include <linux/clk.h> #include <linux/component.h> #include <linux/i2c.h> @@ -76,6 +77,8 @@ #define VC5_HDMI_VERTB_VSPO_SHIFT 16 #define VC5_HDMI_VERTB_VSPO_MASK VC4_MASK(29, 16) +#define VC5_HDMI_SCRAMBLER_CTL_ENABLE BIT(0) + #define VC5_HDMI_DEEP_COLOR_CONFIG_1_INIT_PACK_PHASE_SHIFT 8 #define VC5_HDMI_DEEP_COLOR_CONFIG_1_INIT_PACK_PHASE_MASK VC4_MASK(10, 8) @@ -91,10 +94,14 @@ # define VC4_HD_M_ENABLE BIT(0) #define CEC_CLOCK_FREQ 40000 -#define VC4_HSM_MID_CLOCK 149985000 #define HDMI_14_MAX_TMDS_CLK (340 * 1000 * 1000) +static bool vc4_hdmi_mode_needs_scrambling(const struct drm_display_mode *mode) +{ + return (mode->clock * 1000) > HDMI_14_MAX_TMDS_CLK; +} + static int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *)m->private; @@ -211,6 +218,18 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector) ret = drm_add_edid_modes(connector, edid); kfree(edid); + if (vc4_hdmi->disable_4kp60) { + struct drm_device *drm = connector->dev; + struct drm_display_mode *mode; + + list_for_each_entry(mode, &connector->probed_modes, head) { + if (vc4_hdmi_mode_needs_scrambling(mode)) { + drm_warn_once(drm, "The core clock cannot reach frequencies high enough to support 4k @ 60Hz."); + drm_warn_once(drm, "Please change your config.txt file to add hdmi_enable_4kp60."); + } + } + } + return ret; } @@ -502,6 +521,88 @@ static void vc4_hdmi_set_infoframes(struct drm_encoder *encoder) vc4_hdmi_set_hdr_infoframe(encoder); } +static bool vc4_hdmi_supports_scrambling(struct drm_encoder *encoder, + struct drm_display_mode *mode) +{ + struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder); + struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder); + struct drm_display_info *display = &vc4_hdmi->connector.display_info; + + if (!vc4_encoder->hdmi_monitor) + return false; + + if (!display->hdmi.scdc.supported || + !display->hdmi.scdc.scrambling.supported) + return false; + + return true; +} + +#define SCRAMBLING_POLLING_DELAY_MS 1000 + +static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder) +{ + struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode; + struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder); + + if (!vc4_hdmi_supports_scrambling(encoder, mode)) + return; + + if (!vc4_hdmi_mode_needs_scrambling(mode)) + return; + + drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, true); + drm_scdc_set_scrambling(vc4_hdmi->ddc, true); + + HDMI_WRITE(HDMI_SCRAMBLER_CTL, HDMI_READ(HDMI_SCRAMBLER_CTL) | + VC5_HDMI_SCRAMBLER_CTL_ENABLE); + + queue_delayed_work(system_wq, &vc4_hdmi->scrambling_work, + msecs_to_jiffies(SCRAMBLING_POLLING_DELAY_MS)); +} + +static void vc4_hdmi_disable_scrambling(struct drm_encoder *encoder) +{ + struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder); + struct drm_crtc *crtc = encoder->crtc; + + /* + * At boot, encoder->crtc will be NULL. Since we don't know the + * state of the scrambler and in order to avoid any + * inconsistency, let's disable it all the time. + */ + if (crtc && !vc4_hdmi_supports_scrambling(encoder, &crtc->mode)) + return; + + if (crtc && !vc4_hdmi_mode_needs_scrambling(&crtc->mode)) + return; + + if (delayed_work_pending(&vc4_hdmi->scrambling_work)) + cancel_delayed_work_sync(&vc4_hdmi->scrambling_work); + + HDMI_WRITE(HDMI_SCRAMBLER_CTL, HDMI_READ(HDMI_SCRAMBLER_CTL) & + ~VC5_HDMI_SCRAMBLER_CTL_ENABLE); + + drm_scdc_set_scrambling(vc4_hdmi->ddc, false); + drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, false); +} + +static void vc4_hdmi_scrambling_wq(struct work_struct *work) +{ + struct vc4_hdmi *vc4_hdmi = container_of(to_delayed_work(work), + struct vc4_hdmi, + scrambling_work); + + if (drm_scdc_get_scrambling_status(vc4_hdmi->ddc)) + return; + + drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, true); + drm_scdc_set_scrambling(vc4_hdmi->ddc, true); + + queue_delayed_work(system_wq, &vc4_hdmi->scrambling_work, + msecs_to_jiffies(SCRAMBLING_POLLING_DELAY_MS)); +} + static void vc4_hdmi_encoder_post_crtc_disable(struct drm_encoder *encoder, struct drm_atomic_state *state) { @@ -514,6 +615,8 @@ static void vc4_hdmi_encoder_post_crtc_disable(struct drm_encoder *encoder, HDMI_WRITE(HDMI_VID_CTL, HDMI_READ(HDMI_VID_CTL) | VC4_HD_VID_CTL_BLANKPIX); + + vc4_hdmi_disable_scrambling(encoder); } static void vc4_hdmi_encoder_post_crtc_powerdown(struct drm_encoder *encoder, @@ -795,10 +898,10 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder, conn_state_to_vc4_hdmi_conn_state(conn_state); struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode; struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder); - unsigned long pixel_rate, hsm_rate; + unsigned long bvb_rate, pixel_rate, hsm_rate; int ret; - ret = pm_runtime_get_sync(&vc4_hdmi->pdev->dev); + ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev); if (ret < 0) { DRM_ERROR("Failed to retain power domain: %d\n", ret); return; @@ -849,12 +952,14 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder, vc4_hdmi_cec_update_clk_div(vc4_hdmi); - /* - * FIXME: When the pixel freq is 594MHz (4k60), this needs to be setup - * at 300MHz. - */ - ret = clk_set_min_rate(vc4_hdmi->pixel_bvb_clock, - (hsm_rate > VC4_HSM_MID_CLOCK ? 150000000 : 75000000)); + if (pixel_rate > 297000000) + bvb_rate = 300000000; + else if (pixel_rate > 148500000) + bvb_rate = 150000000; + else + bvb_rate = 75000000; + + ret = clk_set_min_rate(vc4_hdmi->pixel_bvb_clock, bvb_rate); if (ret) { DRM_ERROR("Failed to set pixel bvb clock rate: %d\n", ret); clk_disable_unprepare(vc4_hdmi->hsm_clock); @@ -962,6 +1067,7 @@ static void vc4_hdmi_encoder_post_crtc_enable(struct drm_encoder *encoder, } vc4_hdmi_recenter_fifo(vc4_hdmi); + vc4_hdmi_enable_scrambling(encoder); } static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder) @@ -1014,6 +1120,9 @@ static int vc4_hdmi_encoder_atomic_check(struct drm_encoder *encoder, if (pixel_rate > vc4_hdmi->variant->max_pixel_clock) return -EINVAL; + if (vc4_hdmi->disable_4kp60 && (pixel_rate > HDMI_14_MAX_TMDS_CLK)) + return -EINVAL; + vc4_state->pixel_rate = pixel_rate; return 0; @@ -1033,6 +1142,9 @@ vc4_hdmi_encoder_mode_valid(struct drm_encoder *encoder, if ((mode->clock * 1000) > vc4_hdmi->variant->max_pixel_clock) return MODE_CLOCK_HIGH; + if (vc4_hdmi->disable_4kp60 && vc4_hdmi_mode_needs_scrambling(mode)) + return MODE_CLOCK_HIGH; + return MODE_OK; } @@ -1999,6 +2111,7 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data) vc4_hdmi = devm_kzalloc(dev, sizeof(*vc4_hdmi), GFP_KERNEL); if (!vc4_hdmi) return -ENOMEM; + INIT_DELAYED_WORK(&vc4_hdmi->scrambling_work, vc4_hdmi_scrambling_wq); dev_set_drvdata(dev, vc4_hdmi); encoder = &vc4_hdmi->encoder.base.base; @@ -2048,9 +2161,25 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data) vc4_hdmi->disable_wifi_frequencies = of_property_read_bool(dev->of_node, "wifi-2.4ghz-coexistence"); + if (variant->max_pixel_clock == 600000000) { + struct vc4_dev *vc4 = to_vc4_dev(drm); + long max_rate = clk_round_rate(vc4->hvs->core_clk, 550000000); + + if (max_rate < 550000000) + vc4_hdmi->disable_4kp60 = true; + } + if (vc4_hdmi->variant->reset) vc4_hdmi->variant->reset(vc4_hdmi); + if ((of_device_is_compatible(dev->of_node, "brcm,bcm2711-hdmi0") || + of_device_is_compatible(dev->of_node, "brcm,bcm2711-hdmi1")) && + HDMI_READ(HDMI_VID_CTL) & VC4_HD_VID_CTL_ENABLE) { + clk_prepare_enable(vc4_hdmi->pixel_clock); + clk_prepare_enable(vc4_hdmi->hsm_clock); + clk_prepare_enable(vc4_hdmi->pixel_bvb_clock); + } + pm_runtime_enable(dev); drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS); diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.h b/drivers/gpu/drm/vc4/vc4_hdmi.h index 060bcaefbeb5..39bd2c413ec0 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.h +++ b/drivers/gpu/drm/vc4/vc4_hdmi.h @@ -129,6 +129,8 @@ struct vc4_hdmi { struct vc4_hdmi_encoder encoder; struct drm_connector connector; + struct delayed_work scrambling_work; + struct i2c_adapter *ddc; void __iomem *hdmicore_regs; void __iomem *hd_regs; @@ -157,6 +159,14 @@ struct vc4_hdmi { */ bool disable_wifi_frequencies; + /* + * Even if HDMI0 on the RPi4 can output modes requiring a pixel + * rate higher than 297MHz, it needs some adjustments in the + * config.txt file to be able to do so and thus won't always be + * available. + */ + bool disable_4kp60; + struct cec_adapter *cec_adap; struct cec_msg cec_rx_msg; bool cec_tx_ok; diff --git a/drivers/gpu/drm/vc4/vc4_hdmi_regs.h b/drivers/gpu/drm/vc4/vc4_hdmi_regs.h index e1b58eac766f..19d2fdc446bc 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi_regs.h +++ b/drivers/gpu/drm/vc4/vc4_hdmi_regs.h @@ -100,6 +100,7 @@ enum vc4_hdmi_field { HDMI_RM_FORMAT, HDMI_RM_OFFSET, HDMI_SCHEDULER_CONTROL, + HDMI_SCRAMBLER_CTL, HDMI_SW_RESET_CONTROL, HDMI_TX_PHY_CHANNEL_SWAP, HDMI_TX_PHY_CLK_DIV, @@ -238,6 +239,7 @@ static const struct vc4_hdmi_register __maybe_unused vc5_hdmi_hdmi0_fields[] = { VC4_HDMI_REG(HDMI_GCP_CONFIG, 0x178), VC4_HDMI_REG(HDMI_GCP_WORD_1, 0x17c), VC4_HDMI_REG(HDMI_HOTPLUG, 0x1a8), + VC4_HDMI_REG(HDMI_SCRAMBLER_CTL, 0x1c4), VC5_DVP_REG(HDMI_CLOCK_STOP, 0x0bc), VC5_DVP_REG(HDMI_VEC_INTERFACE_XBAR, 0x0f0), @@ -317,6 +319,7 @@ static const struct vc4_hdmi_register __maybe_unused vc5_hdmi_hdmi1_fields[] = { VC4_HDMI_REG(HDMI_GCP_CONFIG, 0x178), VC4_HDMI_REG(HDMI_GCP_WORD_1, 0x17c), VC4_HDMI_REG(HDMI_HOTPLUG, 0x1a8), + VC4_HDMI_REG(HDMI_SCRAMBLER_CTL, 0x1c4), VC5_DVP_REG(HDMI_CLOCK_STOP, 0x0bc), VC5_DVP_REG(HDMI_VEC_INTERFACE_XBAR, 0x0f0), diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c index c0122d83b651..2fc7f4b5fa09 100644 --- a/drivers/gpu/drm/vc4/vc4_txp.c +++ b/drivers/gpu/drm/vc4/vc4_txp.c @@ -507,7 +507,7 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data) return ret; encoder = &txp->connector.encoder; - encoder->possible_crtcs |= drm_crtc_mask(crtc); + encoder->possible_crtcs = drm_crtc_mask(crtc); ret = devm_request_irq(dev, irq, vc4_txp_interrupt, 0, dev_name(dev), txp); diff --git a/drivers/gpu/drm/vc4/vc4_vec.c b/drivers/gpu/drm/vc4/vc4_vec.c index 090529d0d5dc..11fc3d6f66b1 100644 --- a/drivers/gpu/drm/vc4/vc4_vec.c +++ b/drivers/gpu/drm/vc4/vc4_vec.c @@ -154,9 +154,14 @@ #define VEC_DAC_MISC_DAC_RST_N BIT(0) +struct vc4_vec_variant { + u32 dac_config; +}; + /* General VEC hardware state. */ struct vc4_vec { struct platform_device *pdev; + const struct vc4_vec_variant *variant; struct drm_encoder *encoder; struct drm_connector *connector; @@ -445,10 +450,7 @@ static void vc4_vec_encoder_enable(struct drm_encoder *encoder) VEC_WRITE(VEC_CONFIG2, VEC_CONFIG2_UV_DIG_DIS | VEC_CONFIG2_RGB_DIG_DIS); VEC_WRITE(VEC_CONFIG3, VEC_CONFIG3_HORIZ_LEN_STD); - VEC_WRITE(VEC_DAC_CONFIG, - VEC_DAC_CONFIG_DAC_CTRL(0xc) | - VEC_DAC_CONFIG_DRIVER_CTRL(0xc) | - VEC_DAC_CONFIG_LDO_BIAS_CTRL(0x46)); + VEC_WRITE(VEC_DAC_CONFIG, vec->variant->dac_config); /* Mask all interrupts. */ VEC_WRITE(VEC_MASK0, 0); @@ -501,8 +503,21 @@ static const struct drm_encoder_helper_funcs vc4_vec_encoder_helper_funcs = { .atomic_mode_set = vc4_vec_encoder_atomic_mode_set, }; +static const struct vc4_vec_variant bcm2835_vec_variant = { + .dac_config = VEC_DAC_CONFIG_DAC_CTRL(0xc) | + VEC_DAC_CONFIG_DRIVER_CTRL(0xc) | + VEC_DAC_CONFIG_LDO_BIAS_CTRL(0x46) +}; + +static const struct vc4_vec_variant bcm2711_vec_variant = { + .dac_config = VEC_DAC_CONFIG_DAC_CTRL(0x0) | + VEC_DAC_CONFIG_DRIVER_CTRL(0x80) | + VEC_DAC_CONFIG_LDO_BIAS_CTRL(0x61) +}; + static const struct of_device_id vc4_vec_dt_match[] = { - { .compatible = "brcm,bcm2835-vec", .data = NULL }, + { .compatible = "brcm,bcm2835-vec", .data = &bcm2835_vec_variant }, + { .compatible = "brcm,bcm2711-vec", .data = &bcm2711_vec_variant }, { /* sentinel */ }, }; @@ -540,6 +555,8 @@ static int vc4_vec_bind(struct device *dev, struct device *master, void *data) vec->encoder = &vc4_vec_encoder->base.base; vec->pdev = pdev; + vec->variant = (const struct vc4_vec_variant *) + of_device_get_match_data(dev); vec->regs = vc4_ioremap_regs(pdev, 0); if (IS_ERR(vec->regs)) return PTR_ERR(vec->regs); diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c index 33bf5f53ae31..ca77edbc5ea0 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.c +++ b/drivers/gpu/drm/virtio/virtgpu_drv.c @@ -125,11 +125,13 @@ static int virtio_gpu_probe(struct virtio_device *vdev) ret = drm_dev_register(dev, 0); if (ret) - goto err_free; + goto err_deinit; drm_fbdev_generic_setup(vdev->priv, 32); return 0; +err_deinit: + virtio_gpu_deinit(dev); err_free: drm_dev_put(dev); return ret; diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c index b375394193be..f3379059f324 100644 --- a/drivers/gpu/drm/virtio/virtgpu_kms.c +++ b/drivers/gpu/drm/virtio/virtgpu_kms.c @@ -234,6 +234,7 @@ err_scanouts: err_vbufs: vgdev->vdev->config->del_vqs(vgdev->vdev); err_vqs: + dev->dev_private = NULL; kfree(vgdev); return ret; } @@ -264,6 +265,9 @@ void virtio_gpu_release(struct drm_device *dev) { struct virtio_gpu_device *vgdev = dev->dev_private; + if (!vgdev) + return; + virtio_gpu_modeset_fini(vgdev); virtio_gpu_free_vbufs(vgdev); virtio_gpu_cleanup_cap_cache(vgdev); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c index 7bfe83c936ff..35b03fe21161 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c @@ -661,14 +661,6 @@ static void vmw_evict_flags(struct ttm_buffer_object *bo, *placement = vmw_sys_placement; } -static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp) -{ - struct ttm_object_file *tfile = - vmw_fpriv((struct drm_file *)filp->private_data)->tfile; - - return vmw_user_bo_verify_access(bo, tfile); -} - static int vmw_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) { struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev); @@ -771,7 +763,6 @@ struct ttm_device_funcs vmw_bo_driver = { .eviction_valuable = ttm_bo_eviction_valuable, .evict_flags = vmw_evict_flags, .move = vmw_move, - .verify_access = vmw_verify_access, .swap_notify = vmw_swap_notify, .io_mem_reserve = &vmw_ttm_io_mem_reserve, }; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c index cb9975889e2f..e6b1f98ec99f 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c @@ -27,6 +27,32 @@ #include "vmwgfx_drv.h" +static struct ttm_buffer_object *vmw_bo_vm_lookup(struct ttm_device *bdev, + unsigned long offset, + unsigned long pages) +{ + struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev); + struct drm_device *drm = &dev_priv->drm; + struct drm_vma_offset_node *node; + struct ttm_buffer_object *bo = NULL; + + drm_vma_offset_lock_lookup(bdev->vma_manager); + + node = drm_vma_offset_lookup_locked(bdev->vma_manager, offset, pages); + if (likely(node)) { + bo = container_of(node, struct ttm_buffer_object, + base.vma_node); + bo = ttm_bo_get_unless_zero(bo); + } + + drm_vma_offset_unlock_lookup(bdev->vma_manager); + + if (!bo) + drm_err(drm, "Could not find buffer object to map\n"); + + return bo; +} + int vmw_mmap(struct file *filp, struct vm_area_struct *vma) { static const struct vm_operations_struct vmw_vm_ops = { @@ -41,10 +67,25 @@ int vmw_mmap(struct file *filp, struct vm_area_struct *vma) }; struct drm_file *file_priv = filp->private_data; struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev); - int ret = ttm_bo_mmap(filp, vma, &dev_priv->bdev); + struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; + struct ttm_device *bdev = &dev_priv->bdev; + struct ttm_buffer_object *bo; + int ret; + + if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START)) + return -EINVAL; + + bo = vmw_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); + if (unlikely(!bo)) + return -EINVAL; - if (ret) - return ret; + ret = vmw_user_bo_verify_access(bo, tfile); + if (unlikely(ret != 0)) + goto out_unref; + + ret = ttm_bo_mmap_obj(vma, bo); + if (unlikely(ret != 0)) + goto out_unref; vma->vm_ops = &vmw_vm_ops; @@ -52,7 +93,13 @@ int vmw_mmap(struct file *filp, struct vm_area_struct *vma) if (!is_cow_mapping(vma->vm_flags)) vma->vm_flags = (vma->vm_flags & ~VM_MIXEDMAP) | VM_PFNMAP; + ttm_bo_put(bo); /* release extra ref taken by ttm_bo_mmap_obj() */ + return 0; + +out_unref: + ttm_bo_put(bo); + return ret; } /* struct vmw_validation_mem callback */ |