diff options
Diffstat (limited to 'drivers/misc/habanalabs/common/device.c')
| -rw-r--r-- | drivers/misc/habanalabs/common/device.c | 2535 |
1 files changed, 0 insertions, 2535 deletions
diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c deleted file mode 100644 index 87ab329e65d4..000000000000 --- a/drivers/misc/habanalabs/common/device.c +++ /dev/null @@ -1,2535 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/* - * Copyright 2016-2022 HabanaLabs, Ltd. - * All Rights Reserved. - */ - -#define pr_fmt(fmt) "habanalabs: " fmt - -#include <uapi/misc/habanalabs.h> -#include "habanalabs.h" - -#include <linux/pci.h> -#include <linux/hwmon.h> -#include <linux/vmalloc.h> - -#include <trace/events/habanalabs.h> - -#define HL_RESET_DELAY_USEC 10000 /* 10ms */ - -#define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC 5 - -enum dma_alloc_type { - DMA_ALLOC_COHERENT, - DMA_ALLOC_CPU_ACCESSIBLE, - DMA_ALLOC_POOL, -}; - -#define MEM_SCRUB_DEFAULT_VAL 0x1122334455667788 - -/* - * hl_set_dram_bar- sets the bar to allow later access to address - * - * @hdev: pointer to habanalabs device structure. - * @addr: the address the caller wants to access. - * @region: the PCI region. - * @new_bar_region_base: the new BAR region base address. - * - * @return: the old BAR base address on success, U64_MAX for failure. - * The caller should set it back to the old address after use. - * - * In case the bar space does not cover the whole address space, - * the bar base address should be set to allow access to a given address. - * This function can be called also if the bar doesn't need to be set, - * in that case it just won't change the base. - */ -static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region, - u64 *new_bar_region_base) -{ - struct asic_fixed_properties *prop = &hdev->asic_prop; - u64 bar_base_addr, old_base; - - if (is_power_of_2(prop->dram_pci_bar_size)) - bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull); - else - bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) * - prop->dram_pci_bar_size; - - old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr); - - /* in case of success we need to update the new BAR base */ - if ((old_base != U64_MAX) && new_bar_region_base) - *new_bar_region_base = bar_base_addr; - - return old_base; -} - -int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, - enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar) -{ - struct pci_mem_region *region = &hdev->pci_mem_region[region_type]; - u64 old_base = 0, rc, bar_region_base = region->region_base; - void __iomem *acc_addr; - - if (set_dram_bar) { - old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base); - if (old_base == U64_MAX) - return -EIO; - } - - acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + - (addr - bar_region_base); - - switch (acc_type) { - case DEBUGFS_READ8: - *val = readb(acc_addr); - break; - case DEBUGFS_WRITE8: - writeb(*val, acc_addr); - break; - case DEBUGFS_READ32: - *val = readl(acc_addr); - break; - case DEBUGFS_WRITE32: - writel(*val, acc_addr); - break; - case DEBUGFS_READ64: - *val = readq(acc_addr); - break; - case DEBUGFS_WRITE64: - writeq(*val, acc_addr); - break; - } - - if (set_dram_bar) { - rc = hl_set_dram_bar(hdev, old_base, region, NULL); - if (rc == U64_MAX) - return -EIO; - } - - return 0; -} - -static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, - gfp_t flag, enum dma_alloc_type alloc_type, - const char *caller) -{ - void *ptr = NULL; - - switch (alloc_type) { - case DMA_ALLOC_COHERENT: - ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag); - break; - case DMA_ALLOC_CPU_ACCESSIBLE: - ptr = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); - break; - case DMA_ALLOC_POOL: - ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle); - break; - } - - if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr)) - trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size, - caller); - - return ptr; -} - -static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, enum dma_alloc_type alloc_type, - const char *caller) -{ - /* this is needed to avoid warning on using freed pointer */ - u64 store_cpu_addr = (u64) (uintptr_t) cpu_addr; - - switch (alloc_type) { - case DMA_ALLOC_COHERENT: - hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle); - break; - case DMA_ALLOC_CPU_ACCESSIBLE: - hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, cpu_addr); - break; - case DMA_ALLOC_POOL: - hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle); - break; - } - - trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller); -} - -void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, - gfp_t flag, const char *caller) -{ - return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller); -} - -void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, const char *caller) -{ - hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller); -} - -void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size, - dma_addr_t *dma_handle, const char *caller) -{ - return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); -} - -void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr, - const char *caller) -{ - hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); -} - -void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, - dma_addr_t *dma_handle, const char *caller) -{ - return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller); -} - -void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, - const char *caller) -{ - hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller); -} - -int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) -{ - struct asic_fixed_properties *prop = &hdev->asic_prop; - struct scatterlist *sg; - int rc, i; - - rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0); - if (rc) - return rc; - - /* Shift to the device's base physical address of host memory if necessary */ - if (prop->device_dma_offset_for_host_access) - for_each_sgtable_dma_sg(sgt, sg, i) - sg->dma_address += prop->device_dma_offset_for_host_access; - - return 0; -} - -void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) -{ - struct asic_fixed_properties *prop = &hdev->asic_prop; - struct scatterlist *sg; - int i; - - /* Cancel the device's base physical address of host memory if necessary */ - if (prop->device_dma_offset_for_host_access) - for_each_sgtable_dma_sg(sgt, sg, i) - sg->dma_address -= prop->device_dma_offset_for_host_access; - - dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0); -} - -/* - * hl_access_cfg_region - access the config region - * - * @hdev: pointer to habanalabs device structure - * @addr: the address to access - * @val: the value to write from or read to - * @acc_type: the type of access (read/write 64/32) - */ -int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, - enum debugfs_access_type acc_type) -{ - struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG]; - u32 val_h, val_l; - - if (!IS_ALIGNED(addr, sizeof(u32))) { - dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32)); - return -EINVAL; - } - - switch (acc_type) { - case DEBUGFS_READ32: - *val = RREG32(addr - cfg_region->region_base); - break; - case DEBUGFS_WRITE32: - WREG32(addr - cfg_region->region_base, *val); - break; - case DEBUGFS_READ64: - val_l = RREG32(addr - cfg_region->region_base); - val_h = RREG32(addr + sizeof(u32) - cfg_region->region_base); - - *val = (((u64) val_h) << 32) | val_l; - break; - case DEBUGFS_WRITE64: - WREG32(addr - cfg_region->region_base, lower_32_bits(*val)); - WREG32(addr + sizeof(u32) - cfg_region->region_base, upper_32_bits(*val)); - break; - default: - dev_err(hdev->dev, "access type %d is not supported\n", acc_type); - return -EOPNOTSUPP; - } - - return 0; -} - -/* - * hl_access_dev_mem - access device memory - * - * @hdev: pointer to habanalabs device structure - * @region_type: the type of the region the address belongs to - * @addr: the address to access - * @val: the value to write from or read to - * @acc_type: the type of access (r/w, 32/64) - */ -int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, - u64 addr, u64 *val, enum debugfs_access_type acc_type) -{ - switch (region_type) { - case PCI_REGION_CFG: - return hl_access_cfg_region(hdev, addr, val, acc_type); - case PCI_REGION_SRAM: - case PCI_REGION_DRAM: - return hl_access_sram_dram_region(hdev, addr, val, acc_type, - region_type, (region_type == PCI_REGION_DRAM)); - default: - return -EFAULT; - } - - return 0; -} - -void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...) -{ - va_list args; - int str_size; - - va_start(args, fmt); - /* Calculate formatted string length. Assuming each string is null terminated, hence - * increment result by 1 - */ - str_size = vsnprintf(NULL, 0, fmt, args) + 1; - va_end(args); - - if ((e->actual_size + str_size) < e->allocated_buf_size) { - va_start(args, fmt); - vsnprintf(e->buf + e->actual_size, str_size, fmt, args); - va_end(args); - } - - /* Need to update the size even when not updating destination buffer to get the exact size - * of all input strings - */ - e->actual_size += str_size; -} - -enum hl_device_status hl_device_status(struct hl_device *hdev) -{ - enum hl_device_status status; - - if (hdev->reset_info.in_reset) { - if (hdev->reset_info.in_compute_reset) - status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE; - else - status = HL_DEVICE_STATUS_IN_RESET; - } else if (hdev->reset_info.needs_reset) { - status = HL_DEVICE_STATUS_NEEDS_RESET; - } else if (hdev->disabled) { - status = HL_DEVICE_STATUS_MALFUNCTION; - } else if (!hdev->init_done) { - status = HL_DEVICE_STATUS_IN_DEVICE_CREATION; - } else { - status = HL_DEVICE_STATUS_OPERATIONAL; - } - - return status; -} - -bool hl_device_operational(struct hl_device *hdev, - enum hl_device_status *status) -{ - enum hl_device_status current_status; - - current_status = hl_device_status(hdev); - if (status) - *status = current_status; - - switch (current_status) { - case HL_DEVICE_STATUS_IN_RESET: - case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE: - case HL_DEVICE_STATUS_MALFUNCTION: - case HL_DEVICE_STATUS_NEEDS_RESET: - return false; - case HL_DEVICE_STATUS_OPERATIONAL: - case HL_DEVICE_STATUS_IN_DEVICE_CREATION: - default: - return true; - } -} - -bool hl_ctrl_device_operational(struct hl_device *hdev, - enum hl_device_status *status) -{ - enum hl_device_status current_status; - - current_status = hl_device_status(hdev); - if (status) - *status = current_status; - - switch (current_status) { - case HL_DEVICE_STATUS_MALFUNCTION: - return false; - case HL_DEVICE_STATUS_IN_RESET: - case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE: - case HL_DEVICE_STATUS_NEEDS_RESET: - case HL_DEVICE_STATUS_OPERATIONAL: - case HL_DEVICE_STATUS_IN_DEVICE_CREATION: - default: - return true; - } -} - -static void print_idle_status_mask(struct hl_device *hdev, const char *message, - u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE]) -{ - u32 pad_width[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {}; - - BUILD_BUG_ON(HL_BUSY_ENGINES_MASK_EXT_SIZE != 4); - - pad_width[3] = idle_mask[3] ? 16 : 0; - pad_width[2] = idle_mask[2] || pad_width[3] ? 16 : 0; - pad_width[1] = idle_mask[1] || pad_width[2] ? 16 : 0; - pad_width[0] = idle_mask[0] || pad_width[1] ? 16 : 0; - - dev_err(hdev->dev, "%s (mask %0*llx_%0*llx_%0*llx_%0*llx)\n", - message, pad_width[3], idle_mask[3], pad_width[2], idle_mask[2], - pad_width[1], idle_mask[1], pad_width[0], idle_mask[0]); -} - -static void hpriv_release(struct kref *ref) -{ - u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; - bool reset_device, device_is_idle = true; - struct hl_fpriv *hpriv; - struct hl_device *hdev; - - hpriv = container_of(ref, struct hl_fpriv, refcount); - - hdev = hpriv->hdev; - - hdev->asic_funcs->send_device_activity(hdev, false); - - put_pid(hpriv->taskpid); - - hl_debugfs_remove_file(hpriv); - - mutex_destroy(&hpriv->ctx_lock); - mutex_destroy(&hpriv->restore_phase_mutex); - - /* Device should be reset if reset-upon-device-release is enabled, or if there is a pending - * reset that waits for device release. - */ - reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active; - - /* Unless device is reset in any case, check idle status and reset if device is not idle */ - if (!reset_device && hdev->pdev && !hdev->pldm) - device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask, - HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL); - if (!device_is_idle) { - print_idle_status_mask(hdev, "device is not idle after user context is closed", - idle_mask); - reset_device = true; - } - - /* We need to remove the user from the list to make sure the reset process won't - * try to kill the user process. Because, if we got here, it means there are no - * more driver/device resources that the user process is occupying so there is - * no need to kill it - * - * However, we can't set the compute_ctx to NULL at this stage. This is to prevent - * a race between the release and opening the device again. We don't want to let - * a user open the device while there a reset is about to happen. - */ - mutex_lock(&hdev->fpriv_list_lock); - list_del(&hpriv->dev_node); - mutex_unlock(&hdev->fpriv_list_lock); - - if (reset_device) { - hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE); - } else { - /* Scrubbing is handled within hl_device_reset(), so here need to do it directly */ - int rc = hdev->asic_funcs->scrub_device_mem(hdev); - - if (rc) - dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc); - } - - /* Now we can mark the compute_ctx as not active. Even if a reset is running in a different - * thread, we don't care because the in_reset is marked so if a user will try to open - * the device it will fail on that, even if compute_ctx is false. - */ - mutex_lock(&hdev->fpriv_list_lock); - hdev->is_compute_ctx_active = false; - mutex_unlock(&hdev->fpriv_list_lock); - - hdev->compute_ctx_in_release = 0; - - /* release the eventfd */ - if (hpriv->notifier_event.eventfd) - eventfd_ctx_put(hpriv->notifier_event.eventfd); - - mutex_destroy(&hpriv->notifier_event.lock); - - kfree(hpriv); -} - -void hl_hpriv_get(struct hl_fpriv *hpriv) -{ - kref_get(&hpriv->refcount); -} - -int hl_hpriv_put(struct hl_fpriv *hpriv) -{ - return kref_put(&hpriv->refcount, hpriv_release); -} - -/* - * hl_device_release - release function for habanalabs device - * - * @inode: pointer to inode structure - * @filp: pointer to file structure - * - * Called when process closes an habanalabs device - */ -static int hl_device_release(struct inode *inode, struct file *filp) -{ - struct hl_fpriv *hpriv = filp->private_data; - struct hl_device *hdev = hpriv->hdev; - - filp->private_data = NULL; - - if (!hdev) { - pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n"); - put_pid(hpriv->taskpid); - return 0; - } - - /* Each pending user interrupt holds the user's context, hence we - * must release them all before calling hl_ctx_mgr_fini(). - */ - hl_release_pending_user_interrupts(hpriv->hdev); - - hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr); - hl_mem_mgr_fini(&hpriv->mem_mgr); - - hdev->compute_ctx_in_release = 1; - - if (!hl_hpriv_put(hpriv)) { - dev_notice(hdev->dev, "User process closed FD but device still in use\n"); - hl_device_reset(hdev, HL_DRV_RESET_HARD); - } - - hdev->last_open_session_duration_jif = - jiffies - hdev->last_successful_open_jif; - - return 0; -} - -static int hl_device_release_ctrl(struct inode *inode, struct file *filp) -{ - struct hl_fpriv *hpriv = filp->private_data; - struct hl_device *hdev = hpriv->hdev; - - filp->private_data = NULL; - - if (!hdev) { - pr_err("Closing FD after device was removed\n"); - goto out; - } - - mutex_lock(&hdev->fpriv_ctrl_list_lock); - list_del(&hpriv->dev_node); - mutex_unlock(&hdev->fpriv_ctrl_list_lock); -out: - /* release the eventfd */ - if (hpriv->notifier_event.eventfd) - eventfd_ctx_put(hpriv->notifier_event.eventfd); - - mutex_destroy(&hpriv->notifier_event.lock); - put_pid(hpriv->taskpid); - - kfree(hpriv); - - return 0; -} - -/* - * hl_mmap - mmap function for habanalabs device - * - * @*filp: pointer to file structure - * @*vma: pointer to vm_area_struct of the process - * - * Called when process does an mmap on habanalabs device. Call the relevant mmap - * function at the end of the common code. - */ -static int hl_mmap(struct file *filp, struct vm_area_struct *vma) -{ - struct hl_fpriv *hpriv = filp->private_data; - struct hl_device *hdev = hpriv->hdev; - unsigned long vm_pgoff; - - if (!hdev) { - pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n"); - return -ENODEV; - } - - vm_pgoff = vma->vm_pgoff; - - switch (vm_pgoff & HL_MMAP_TYPE_MASK) { - case HL_MMAP_TYPE_BLOCK: - vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff); - return hl_hw_block_mmap(hpriv, vma); - - case HL_MMAP_TYPE_CB: - case HL_MMAP_TYPE_TS_BUFF: - return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL); - } - return -EINVAL; -} - -static const struct file_operations hl_ops = { - .owner = THIS_MODULE, - .open = hl_device_open, - .release = hl_device_release, - .mmap = hl_mmap, - .unlocked_ioctl = hl_ioctl, - .compat_ioctl = hl_ioctl -}; - -static const struct file_operations hl_ctrl_ops = { - .owner = THIS_MODULE, - .open = hl_device_open_ctrl, - .release = hl_device_release_ctrl, - .unlocked_ioctl = hl_ioctl_control, - .compat_ioctl = hl_ioctl_control -}; - -static void device_release_func(struct device *dev) -{ - kfree(dev); -} - -/* - * device_init_cdev - Initialize cdev and device for habanalabs device - * - * @hdev: pointer to habanalabs device structure - * @hclass: pointer to the class object of the device - * @minor: minor number of the specific device - * @fpos: file operations to install for this device - * @name: name of the device as it will appear in the filesystem - * @cdev: pointer to the char device object that will be initialized - * @dev: pointer to the device object that will be initialized - * - * Initialize a cdev and a Linux device for habanalabs's device. - */ -static int device_init_cdev(struct hl_device *hdev, struct class *hclass, - int minor, const struct file_operations *fops, - char *name, struct cdev *cdev, - struct device **dev) -{ - cdev_init(cdev, fops); - cdev->owner = THIS_MODULE; - - *dev = kzalloc(sizeof(**dev), GFP_KERNEL); - if (!*dev) - return -ENOMEM; - - device_initialize(*dev); - (*dev)->devt = MKDEV(hdev->major, minor); - (*dev)->class = hclass; - (*dev)->release = device_release_func; - dev_set_drvdata(*dev, hdev); - dev_set_name(*dev, "%s", name); - - return 0; -} - -static int device_cdev_sysfs_add(struct hl_device *hdev) -{ - int rc; - - rc = cdev_device_add(&hdev->cdev, hdev->dev); - if (rc) { - dev_err(hdev->dev, - "failed to add a char device to the system\n"); - return rc; - } - - rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl); - if (rc) { - dev_err(hdev->dev, - "failed to add a control char device to the system\n"); - goto delete_cdev_device; - } - - /* hl_sysfs_init() must be done after adding the device to the system */ - rc = hl_sysfs_init(hdev); - if (rc) { - dev_err(hdev->dev, "failed to initialize sysfs\n"); - goto delete_ctrl_cdev_device; - } - - hdev->cdev_sysfs_created = true; - - return 0; - -delete_ctrl_cdev_device: - cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); -delete_cdev_device: - cdev_device_del(&hdev->cdev, hdev->dev); - return rc; -} - -static void device_cdev_sysfs_del(struct hl_device *hdev) -{ - if (!hdev->cdev_sysfs_created) - goto put_devices; - - hl_sysfs_fini(hdev); - cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); - cdev_device_del(&hdev->cdev, hdev->dev); - -put_devices: - put_device(hdev->dev); - put_device(hdev->dev_ctrl); -} - -static void device_hard_reset_pending(struct work_struct *work) -{ - struct hl_device_reset_work *device_reset_work = - container_of(work, struct hl_device_reset_work, reset_work.work); - struct hl_device *hdev = device_reset_work->hdev; - u32 flags; - int rc; - - flags = device_reset_work->flags | HL_DRV_RESET_FROM_RESET_THR; - - rc = hl_device_reset(hdev, flags); - - if ((rc == -EBUSY) && !hdev->device_fini_pending) { - struct hl_ctx *ctx = hl_get_compute_ctx(hdev); - - if (ctx) { - /* The read refcount value should subtracted by one, because the read is - * protected with hl_get_compute_ctx(). - */ - dev_info(hdev->dev, - "Could not reset device (compute_ctx refcount %u). will try again in %u seconds", - kref_read(&ctx->refcount) - 1, HL_PENDING_RESET_PER_SEC); - hl_ctx_put(ctx); - } else { - dev_info(hdev->dev, "Could not reset device. will try again in %u seconds", - HL_PENDING_RESET_PER_SEC); - } - - queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work, - msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000)); - } -} - -static void device_release_watchdog_func(struct work_struct *work) -{ - struct hl_device_reset_work *device_release_watchdog_work = - container_of(work, struct hl_device_reset_work, reset_work.work); - struct hl_device *hdev = device_release_watchdog_work->hdev; - u32 flags; - - dev_dbg(hdev->dev, "Device wasn't released in time. Initiate device reset.\n"); - - flags = device_release_watchdog_work->flags | HL_DRV_RESET_FROM_WD_THR; - - hl_device_reset(hdev, flags); -} - -/* - * device_early_init - do some early initialization for the habanalabs device - * - * @hdev: pointer to habanalabs device structure - * - * Install the relevant function pointers and call the early_init function, - * if such a function exists - */ -static int device_early_init(struct hl_device *hdev) -{ - int i, rc; - char workq_name[32]; - - switch (hdev->asic_type) { - case ASIC_GOYA: - goya_set_asic_funcs(hdev); - strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name)); - break; - case ASIC_GAUDI: - gaudi_set_asic_funcs(hdev); - strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name)); - break; - case ASIC_GAUDI_SEC: - gaudi_set_asic_funcs(hdev); - strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name)); - break; - case ASIC_GAUDI2: - gaudi2_set_asic_funcs(hdev); - strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name)); - break; - case ASIC_GAUDI2B: - gaudi2_set_asic_funcs(hdev); - strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name)); - break; - break; - default: - dev_err(hdev->dev, "Unrecognized ASIC type %d\n", - hdev->asic_type); - return -EINVAL; - } - - rc = hdev->asic_funcs->early_init(hdev); - if (rc) - return rc; - - rc = hl_asid_init(hdev); - if (rc) - goto early_fini; - - if (hdev->asic_prop.completion_queues_count) { - hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count, - sizeof(struct workqueue_struct *), - GFP_KERNEL); - if (!hdev->cq_wq) { - rc = -ENOMEM; - goto asid_fini; - } - } - - for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { - snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i); - hdev->cq_wq[i] = create_singlethread_workqueue(workq_name); - if (hdev->cq_wq[i] == NULL) { - dev_err(hdev->dev, "Failed to allocate CQ workqueue\n"); - rc = -ENOMEM; - goto free_cq_wq; - } - } - - hdev->eq_wq = create_singlethread_workqueue("hl-events"); - if (hdev->eq_wq == NULL) { - dev_err(hdev->dev, "Failed to allocate EQ workqueue\n"); - rc = -ENOMEM; - goto free_cq_wq; - } - - hdev->cs_cmplt_wq = alloc_workqueue("hl-cs-completions", WQ_UNBOUND, 0); - if (!hdev->cs_cmplt_wq) { - dev_err(hdev->dev, - "Failed to allocate CS completions workqueue\n"); - rc = -ENOMEM; - goto free_eq_wq; - } - - hdev->ts_free_obj_wq = alloc_workqueue("hl-ts-free-obj", WQ_UNBOUND, 0); - if (!hdev->ts_free_obj_wq) { - dev_err(hdev->dev, - "Failed to allocate Timestamp registration free workqueue\n"); - rc = -ENOMEM; - goto free_cs_cmplt_wq; - } - - hdev->prefetch_wq = alloc_workqueue("hl-prefetch", WQ_UNBOUND, 0); - if (!hdev->prefetch_wq) { - dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n"); - rc = -ENOMEM; - goto free_ts_free_wq; - } - - hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), - GFP_KERNEL); - if (!hdev->hl_chip_info) { - rc = -ENOMEM; - goto free_prefetch_wq; - } - - rc = hl_mmu_if_set_funcs(hdev); - if (rc) - goto free_chip_info; - - hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr); - - hdev->reset_wq = create_singlethread_workqueue("hl_device_reset"); - if (!hdev->reset_wq) { - rc = -ENOMEM; - dev_err(hdev->dev, "Failed to create device reset WQ\n"); - goto free_cb_mgr; - } - - INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending); - hdev->device_reset_work.hdev = hdev; - hdev->device_fini_pending = 0; - - INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work, - device_release_watchdog_func); - hdev->device_release_watchdog_work.hdev = hdev; - - mutex_init(&hdev->send_cpu_message_lock); - mutex_init(&hdev->debug_lock); - INIT_LIST_HEAD(&hdev->cs_mirror_list); - spin_lock_init(&hdev->cs_mirror_lock); - spin_lock_init(&hdev->reset_info.lock); - INIT_LIST_HEAD(&hdev->fpriv_list); - INIT_LIST_HEAD(&hdev->fpriv_ctrl_list); - mutex_init(&hdev->fpriv_list_lock); - mutex_init(&hdev->fpriv_ctrl_list_lock); - mutex_init(&hdev->clk_throttling.lock); - - return 0; - -free_cb_mgr: - hl_mem_mgr_fini(&hdev->kernel_mem_mgr); -free_chip_info: - kfree(hdev->hl_chip_info); -free_prefetch_wq: - destroy_workqueue(hdev->prefetch_wq); -free_ts_free_wq: - destroy_workqueue(hdev->ts_free_obj_wq); -free_cs_cmplt_wq: - destroy_workqueue(hdev->cs_cmplt_wq); -free_eq_wq: - destroy_workqueue(hdev->eq_wq); -free_cq_wq: - for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) - if (hdev->cq_wq[i]) - destroy_workqueue(hdev->cq_wq[i]); - kfree(hdev->cq_wq); -asid_fini: - hl_asid_fini(hdev); -early_fini: - if (hdev->asic_funcs->early_fini) - hdev->asic_funcs->early_fini(hdev); - - return rc; -} - -/* - * device_early_fini - finalize all that was done in device_early_init - * - * @hdev: pointer to habanalabs device structure - * - */ -static void device_early_fini(struct hl_device *hdev) -{ - int i; - - mutex_destroy(&hdev->debug_lock); - mutex_destroy(&hdev->send_cpu_message_lock); - - mutex_destroy(&hdev->fpriv_list_lock); - mutex_destroy(&hdev->fpriv_ctrl_list_lock); - - mutex_destroy(&hdev->clk_throttling.lock); - - hl_mem_mgr_fini(&hdev->kernel_mem_mgr); - - kfree(hdev->hl_chip_info); - - destroy_workqueue(hdev->prefetch_wq); - destroy_workqueue(hdev->ts_free_obj_wq); - destroy_workqueue(hdev->cs_cmplt_wq); - destroy_workqueue(hdev->eq_wq); - destroy_workqueue(hdev->reset_wq); - - for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) - destroy_workqueue(hdev->cq_wq[i]); - kfree(hdev->cq_wq); - - hl_asid_fini(hdev); - - if (hdev->asic_funcs->early_fini) - hdev->asic_funcs->early_fini(hdev); -} - -static void hl_device_heartbeat(struct work_struct *work) -{ - struct hl_device *hdev = container_of(work, struct hl_device, - work_heartbeat.work); - - if (!hl_device_operational(hdev, NULL)) - goto reschedule; - - if (!hdev->asic_funcs->send_heartbeat(hdev)) - goto reschedule; - - if (hl_device_operational(hdev, NULL)) - dev_err(hdev->dev, "Device heartbeat failed!\n"); - - hl_device_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT); - - return; - -reschedule: - /* - * prev_reset_trigger tracks consecutive fatal h/w errors until first - * heartbeat immediately post reset. - * If control reached here, then at least one heartbeat work has been - * scheduled since last reset/init cycle. - * So if the device is not already in reset cycle, reset the flag - * prev_reset_trigger as no reset occurred with HL_DRV_RESET_FW_FATAL_ERR - * status for at least one heartbeat. From this point driver restarts - * tracking future consecutive fatal errors. - */ - if (!hdev->reset_info.in_reset) - hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT; - - schedule_delayed_work(&hdev->work_heartbeat, - usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); -} - -/* - * device_late_init - do late stuff initialization for the habanalabs device - * - * @hdev: pointer to habanalabs device structure - * - * Do stuff that either needs the device H/W queues to be active or needs - * to happen after all the rest of the initialization is finished - */ -static int device_late_init(struct hl_device *hdev) -{ - int rc; - - if (hdev->asic_funcs->late_init) { - rc = hdev->asic_funcs->late_init(hdev); - if (rc) { - dev_err(hdev->dev, - "failed late initialization for the H/W\n"); - return rc; - } - } - - hdev->high_pll = hdev->asic_prop.high_pll; - - if (hdev->heartbeat) { - INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); - schedule_delayed_work(&hdev->work_heartbeat, - usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); - } - - hdev->late_init_done = true; - - return 0; -} - -/* - * device_late_fini - finalize all that was done in device_late_init - * - * @hdev: pointer to habanalabs device structure - * - */ -static void device_late_fini(struct hl_device *hdev) -{ - if (!hdev->late_init_done) - return; - - if (hdev->heartbeat) - cancel_delayed_work_sync(&hdev->work_heartbeat); - - if (hdev->asic_funcs->late_fini) - hdev->asic_funcs->late_fini(hdev); - - hdev->late_init_done = false; -} - -int hl_device_utilization(struct hl_device *hdev, u32 *utilization) -{ - u64 max_power, curr_power, dc_power, dividend, divisor; - int rc; - - max_power = hdev->max_power; - dc_power = hdev->asic_prop.dc_power_default; - divisor = max_power - dc_power; - if (!divisor) { - dev_warn(hdev->dev, "device utilization is not supported\n"); - return -EOPNOTSUPP; - } - rc = hl_fw_cpucp_power_get(hdev, &curr_power); - - if (rc) - return rc; - - curr_power = clamp(curr_power, dc_power, max_power); - - dividend = (curr_power - dc_power) * 100; - *utilization = (u32) div_u64(dividend, divisor); - - return 0; -} - -int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable) -{ - int rc = 0; - - mutex_lock(&hdev->debug_lock); - - if (!enable) { - if (!hdev->in_debug) { - dev_err(hdev->dev, - "Failed to disable debug mode because device was not in debug mode\n"); - rc = -EFAULT; - goto out; - } - - if (!hdev->reset_info.hard_reset_pending) - hdev->asic_funcs->halt_coresight(hdev, ctx); - - hdev->in_debug = 0; - - goto out; - } - - if (hdev->in_debug) { - dev_err(hdev->dev, - "Failed to enable debug mode because device is already in debug mode\n"); - rc = -EFAULT; - goto out; - } - - hdev->in_debug = 1; - -out: - mutex_unlock(&hdev->debug_lock); - - return rc; -} - -static void take_release_locks(struct hl_device *hdev) -{ - /* Flush anyone that is inside the critical section of enqueue - * jobs to the H/W - */ - hdev->asic_funcs->hw_queues_lock(hdev); - hdev->asic_funcs->hw_queues_unlock(hdev); - - /* Flush processes that are sending message to CPU */ - mutex_lock(&hdev->send_cpu_message_lock); - mutex_unlock(&hdev->send_cpu_message_lock); - - /* Flush anyone that is inside device open */ - mutex_lock(&hdev->fpriv_list_lock); - mutex_unlock(&hdev->fpriv_list_lock); - mutex_lock(&hdev->fpriv_ctrl_list_lock); - mutex_unlock(&hdev->fpriv_ctrl_list_lock); -} - -static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset, - bool skip_wq_flush) -{ - if (hard_reset) - device_late_fini(hdev); - - /* - * Halt the engines and disable interrupts so we won't get any more - * completions from H/W and we won't have any accesses from the - * H/W to the host machine - */ - hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset); - - /* Go over all the queues, release all CS and their jobs */ - hl_cs_rollback_all(hdev, skip_wq_flush); - - /* flush the MMU prefetch workqueue */ - flush_workqueue(hdev->prefetch_wq); - - /* Release all pending user interrupts, each pending user interrupt - * holds a reference to user context - */ - hl_release_pending_user_interrupts(hdev); -} - -/* - * hl_device_suspend - initiate device suspend - * - * @hdev: pointer to habanalabs device structure - * - * Puts the hw in the suspend state (all asics). - * Returns 0 for success or an error on failure. - * Called at driver suspend. - */ -int hl_device_suspend(struct hl_device *hdev) -{ - int rc; - - pci_save_state(hdev->pdev); - - /* Block future CS/VM/JOB completion operations */ - spin_lock(&hdev->reset_info.lock); - if (hdev->reset_info.in_reset) { - spin_unlock(&hdev->reset_info.lock); - dev_err(hdev->dev, "Can't suspend while in reset\n"); - return -EIO; - } - hdev->reset_info.in_reset = 1; - spin_unlock(&hdev->reset_info.lock); - - /* This blocks all other stuff that is not blocked by in_reset */ - hdev->disabled = true; - - take_release_locks(hdev); - - rc = hdev->asic_funcs->suspend(hdev); - if (rc) - dev_err(hdev->dev, - "Failed to disable PCI access of device CPU\n"); - - /* Shut down the device */ - pci_disable_device(hdev->pdev); - pci_set_power_state(hdev->pdev, PCI_D3hot); - - return 0; -} - -/* - * hl_device_resume - initiate device resume - * - * @hdev: pointer to habanalabs device structure - * - * Bring the hw back to operating state (all asics). - * Returns 0 for success or an error on failure. - * Called at driver resume. - */ -int hl_device_resume(struct hl_device *hdev) -{ - int rc; - - pci_set_power_state(hdev->pdev, PCI_D0); - pci_restore_state(hdev->pdev); - rc = pci_enable_device_mem(hdev->pdev); - if (rc) { - dev_err(hdev->dev, - "Failed to enable PCI device in resume\n"); - return rc; - } - - pci_set_master(hdev->pdev); - - rc = hdev->asic_funcs->resume(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to resume device after suspend\n"); - goto disable_device; - } - - - /* 'in_reset' was set to true during suspend, now we must clear it in order - * for hard reset to be performed - */ - spin_lock(&hdev->reset_info.lock); - hdev->reset_info.in_reset = 0; - spin_unlock(&hdev->reset_info.lock); - - rc = hl_device_reset(hdev, HL_DRV_RESET_HARD); - if (rc) { - dev_err(hdev->dev, "Failed to reset device during resume\n"); - goto disable_device; - } - - return 0; - -disable_device: - pci_clear_master(hdev->pdev); - pci_disable_device(hdev->pdev); - - return rc; -} - -static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev) -{ - struct task_struct *task = NULL; - struct list_head *fd_list; - struct hl_fpriv *hpriv; - struct mutex *fd_lock; - u32 pending_cnt; - - fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; - fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; - - /* Giving time for user to close FD, and for processes that are inside - * hl_device_open to finish - */ - if (!list_empty(fd_list)) - ssleep(1); - - if (timeout) { - pending_cnt = timeout; - } else { - if (hdev->process_kill_trial_cnt) { - /* Processes have been already killed */ - pending_cnt = 1; - goto wait_for_processes; - } else { - /* Wait a small period after process kill */ - pending_cnt = HL_PENDING_RESET_PER_SEC; - } - } - - mutex_lock(fd_lock); - - /* This section must be protected because we are dereferencing - * pointers that are freed if the process exits - */ - list_for_each_entry(hpriv, fd_list, dev_node) { - task = get_pid_task(hpriv->taskpid, PIDTYPE_PID); - if (task) { - dev_info(hdev->dev, "Killing user process pid=%d\n", - task_pid_nr(task)); - send_sig(SIGKILL, task, 1); - usleep_range(1000, 10000); - - put_task_struct(task); - } else { - /* - * If we got here, it means that process was killed from outside the driver - * right after it started looping on fd_list and before get_pid_task, thus - * we don't need to kill it. - */ - dev_dbg(hdev->dev, - "Can't get task struct for user process, assuming process was killed from outside the driver\n"); - } - } - - mutex_unlock(fd_lock); - - /* - * We killed the open users, but that doesn't mean they are closed. - * It could be that they are running a long cleanup phase in the driver - * e.g. MMU unmappings, or running other long teardown flow even before - * our cleanup. - * Therefore we need to wait again to make sure they are closed before - * continuing with the reset. - */ - -wait_for_processes: - while ((!list_empty(fd_list)) && (pending_cnt)) { - dev_dbg(hdev->dev, - "Waiting for all unmap operations to finish before hard reset\n"); - - pending_cnt--; - - ssleep(1); - } - - /* All processes exited successfully */ - if (list_empty(fd_list)) - return 0; - - /* Give up waiting for processes to exit */ - if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS) - return -ETIME; - - hdev->process_kill_trial_cnt++; - - return -EBUSY; -} - -static void device_disable_open_processes(struct hl_device *hdev, bool control_dev) -{ - struct list_head *fd_list; - struct hl_fpriv *hpriv; - struct mutex *fd_lock; - - fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; - fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; - - mutex_lock(fd_lock); - list_for_each_entry(hpriv, fd_list, dev_node) - hpriv->hdev = NULL; - mutex_unlock(fd_lock); -} - -static void handle_reset_trigger(struct hl_device *hdev, u32 flags) -{ - u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT; - - /* No consecutive mechanism when user context exists */ - if (hdev->is_compute_ctx_active) - return; - - /* - * 'reset cause' is being updated here, because getting here - * means that it's the 1st time and the last time we're here - * ('in_reset' makes sure of it). This makes sure that - * 'reset_cause' will continue holding its 1st recorded reason! - */ - if (flags & HL_DRV_RESET_HEARTBEAT) { - hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; - cur_reset_trigger = HL_DRV_RESET_HEARTBEAT; - } else if (flags & HL_DRV_RESET_TDR) { - hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR; - cur_reset_trigger = HL_DRV_RESET_TDR; - } else if (flags & HL_DRV_RESET_FW_FATAL_ERR) { - hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; - cur_reset_trigger = HL_DRV_RESET_FW_FATAL_ERR; - } else { - hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; - } - - /* - * If reset cause is same twice, then reset_trigger_repeated - * is set and if this reset is due to a fatal FW error - * device is set to an unstable state. - */ - if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) { - hdev->reset_info.prev_reset_trigger = cur_reset_trigger; - hdev->reset_info.reset_trigger_repeated = 0; - } else { - hdev->reset_info.reset_trigger_repeated = 1; - } - - /* If reset is due to heartbeat, device CPU is no responsive in - * which case no point sending PCI disable message to it. - * - * If F/W is performing the reset, no need to send it a message to disable - * PCI access - */ - if ((flags & HL_DRV_RESET_HARD) && - !(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) { - /* Disable PCI access from device F/W so he won't send - * us additional interrupts. We disable MSI/MSI-X at - * the halt_engines function and we can't have the F/W - * sending us interrupts after that. We need to disable - * the access here because if the device is marked - * disable, the message won't be send. Also, in case - * of heartbeat, the device CPU is marked as disable - * so this message won't be sent - */ - if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) - dev_warn(hdev->dev, - "Failed to disable PCI access by F/W\n"); - } -} - -/* - * hl_device_reset - reset the device - * - * @hdev: pointer to habanalabs device structure - * @flags: reset flags. - * - * Block future CS and wait for pending CS to be enqueued - * Call ASIC H/W fini - * Flush all completions - * Re-initialize all internal data structures - * Call ASIC H/W init, late_init - * Test queues - * Enable device - * - * Returns 0 for success or an error on failure. - */ -int hl_device_reset(struct hl_device *hdev, u32 flags) -{ - bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false, - reset_upon_device_release = false, schedule_hard_reset = false, delay_reset, - from_dev_release, from_watchdog_thread; - u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; - struct hl_ctx *ctx; - int i, rc; - - if (!hdev->init_done) { - dev_err(hdev->dev, "Can't reset before initialization is done\n"); - return 0; - } - - hard_reset = !!(flags & HL_DRV_RESET_HARD); - from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR); - fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW); - from_dev_release = !!(flags & HL_DRV_RESET_DEV_RELEASE); - delay_reset = !!(flags & HL_DRV_RESET_DELAY); - from_watchdog_thread = !!(flags & HL_DRV_RESET_FROM_WD_THR); - - if (!hard_reset && !hdev->asic_prop.supports_compute_reset) { - hard_instead_soft = true; - hard_reset = true; - } - - if (hdev->reset_upon_device_release && (flags & HL_DRV_RESET_DEV_RELEASE)) { - if (hard_reset) { - dev_crit(hdev->dev, - "Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n"); - return -EINVAL; - } - - reset_upon_device_release = true; - - goto do_reset; - } - - if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) { - hard_instead_soft = true; - hard_reset = true; - } - - if (hard_instead_soft) - dev_dbg(hdev->dev, "Doing hard-reset instead of compute reset\n"); - -do_reset: - /* Re-entry of reset thread */ - if (from_hard_reset_thread && hdev->process_kill_trial_cnt) - goto kill_processes; - - /* - * Prevent concurrency in this function - only one reset should be - * done at any given time. Only need to perform this if we didn't - * get from the dedicated hard reset thread - */ - if (!from_hard_reset_thread) { - /* Block future CS/VM/JOB completion operations */ - spin_lock(&hdev->reset_info.lock); - if (hdev->reset_info.in_reset) { - /* We only allow scheduling of a hard reset during compute reset */ - if (hard_reset && hdev->reset_info.in_compute_reset) - hdev->reset_info.hard_reset_schedule_flags = flags; - spin_unlock(&hdev->reset_info.lock); - return 0; - } - - /* This still allows the completion of some KDMA ops - * Update this before in_reset because in_compute_reset implies we are in reset - */ - hdev->reset_info.in_compute_reset = !hard_reset; - - hdev->reset_info.in_reset = 1; - - spin_unlock(&hdev->reset_info.lock); - - /* Cancel the device release watchdog work if required. - * In case of reset-upon-device-release while the release watchdog work is - * scheduled, do hard-reset instead of compute-reset. - */ - if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) { - hdev->reset_info.watchdog_active = 0; - if (!from_watchdog_thread) - cancel_delayed_work_sync( - &hdev->device_release_watchdog_work.reset_work); - - if (from_dev_release) { - flags |= HL_DRV_RESET_HARD; - flags &= ~HL_DRV_RESET_DEV_RELEASE; - hard_reset = true; - } - } - - if (delay_reset) - usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1); - - handle_reset_trigger(hdev, flags); - - /* This also blocks future CS/VM/JOB completion operations */ - hdev->disabled = true; - - take_release_locks(hdev); - - if (hard_reset) - dev_info(hdev->dev, "Going to reset device\n"); - else if (reset_upon_device_release) - dev_dbg(hdev->dev, "Going to reset device after release by user\n"); - else - dev_dbg(hdev->dev, "Going to reset engines of inference device\n"); - } - -again: - if ((hard_reset) && (!from_hard_reset_thread)) { - hdev->reset_info.hard_reset_pending = true; - - hdev->process_kill_trial_cnt = 0; - - hdev->device_reset_work.flags = flags; - - /* - * Because the reset function can't run from heartbeat work, - * we need to call the reset function from a dedicated work. - */ - queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0); - - return 0; - } - - cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release); - -kill_processes: - if (hard_reset) { - /* Kill processes here after CS rollback. This is because the - * process can't really exit until all its CSs are done, which - * is what we do in cs rollback - */ - rc = device_kill_open_processes(hdev, 0, false); - - if (rc == -EBUSY) { - if (hdev->device_fini_pending) { - dev_crit(hdev->dev, - "Failed to kill all open processes, stopping hard reset\n"); - goto out_err; - } - - /* signal reset thread to reschedule */ - return rc; - } - - if (rc) { - dev_crit(hdev->dev, - "Failed to kill all open processes, stopping hard reset\n"); - goto out_err; - } - - /* Flush the Event queue workers to make sure no other thread is - * reading or writing to registers during the reset - */ - flush_workqueue(hdev->eq_wq); - } - - /* Reset the H/W. It will be in idle state after this returns */ - hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset); - - if (hard_reset) { - hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; - - /* Release kernel context */ - if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1) - hdev->kernel_ctx = NULL; - - hl_vm_fini(hdev); - hl_mmu_fini(hdev); - hl_eq_reset(hdev, &hdev->event_queue); - } - - /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */ - hl_hw_queue_reset(hdev, hard_reset); - for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) - hl_cq_reset(hdev, &hdev->completion_queue[i]); - - /* Make sure the context switch phase will run again */ - ctx = hl_get_compute_ctx(hdev); - if (ctx) { - atomic_set(&ctx->thread_ctx_switch_token, 1); - ctx->thread_ctx_switch_wait_token = 0; - hl_ctx_put(ctx); - } - - /* Finished tear-down, starting to re-initialize */ - - if (hard_reset) { - hdev->device_cpu_disabled = false; - hdev->reset_info.hard_reset_pending = false; - - if (hdev->reset_info.reset_trigger_repeated && - (hdev->reset_info.prev_reset_trigger == - HL_DRV_RESET_FW_FATAL_ERR)) { - /* if there 2 back to back resets from FW, - * ensure driver puts the driver in a unusable state - */ - dev_crit(hdev->dev, - "Consecutive FW fatal errors received, stopping hard reset\n"); - rc = -EIO; - goto out_err; - } - - if (hdev->kernel_ctx) { - dev_crit(hdev->dev, - "kernel ctx was alive during hard reset, something is terribly wrong\n"); - rc = -EBUSY; - goto out_err; - } - - rc = hl_mmu_init(hdev); - if (rc) { - dev_err(hdev->dev, - "Failed to initialize MMU S/W after hard reset\n"); - goto out_err; - } - - /* Allocate the kernel context */ - hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), - GFP_KERNEL); - if (!hdev->kernel_ctx) { - rc = -ENOMEM; - hl_mmu_fini(hdev); - goto out_err; - } - - hdev->is_compute_ctx_active = false; - - rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); - if (rc) { - dev_err(hdev->dev, - "failed to init kernel ctx in hard reset\n"); - kfree(hdev->kernel_ctx); - hdev->kernel_ctx = NULL; - hl_mmu_fini(hdev); - goto out_err; - } - } - - /* Device is now enabled as part of the initialization requires - * communication with the device firmware to get information that - * is required for the initialization itself - */ - hdev->disabled = false; - - /* F/W security enabled indication might be updated after hard-reset */ - if (hard_reset) { - rc = hl_fw_read_preboot_status(hdev); - if (rc) - goto out_err; - } - - rc = hdev->asic_funcs->hw_init(hdev); - if (rc) { - dev_err(hdev->dev, "failed to initialize the H/W after reset\n"); - goto out_err; - } - - /* If device is not idle fail the reset process */ - if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, - HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) { - print_idle_status_mask(hdev, "device is not idle after reset", idle_mask); - rc = -EIO; - goto out_err; - } - - /* Check that the communication with the device is working */ - rc = hdev->asic_funcs->test_queues(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to detect if device is alive after reset\n"); - goto out_err; - } - - if (hard_reset) { - rc = device_late_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed late init after hard reset\n"); - goto out_err; - } - - rc = hl_vm_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to init memory module after hard reset\n"); - goto out_err; - } - - if (!hdev->asic_prop.fw_security_enabled) - hl_fw_set_max_power(hdev); - } else { - rc = hdev->asic_funcs->compute_reset_late_init(hdev); - if (rc) { - if (reset_upon_device_release) - dev_err(hdev->dev, - "Failed late init in reset after device release\n"); - else - dev_err(hdev->dev, "Failed late init after compute reset\n"); - goto out_err; - } - } - - rc = hdev->asic_funcs->scrub_device_mem(hdev); - if (rc) { - dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc); - return rc; - } - - spin_lock(&hdev->reset_info.lock); - hdev->reset_info.in_compute_reset = 0; - - /* Schedule hard reset only if requested and if not already in hard reset. - * We keep 'in_reset' enabled, so no other reset can go in during the hard - * reset schedule - */ - if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags) - schedule_hard_reset = true; - else - hdev->reset_info.in_reset = 0; - - spin_unlock(&hdev->reset_info.lock); - - hdev->reset_info.needs_reset = false; - - if (hard_reset) - dev_info(hdev->dev, "Successfully finished resetting the device\n"); - else - dev_dbg(hdev->dev, "Successfully finished resetting the device\n"); - - if (hard_reset) { - hdev->reset_info.hard_reset_cnt++; - - /* After reset is done, we are ready to receive events from - * the F/W. We can't do it before because we will ignore events - * and if those events are fatal, we won't know about it and - * the device will be operational although it shouldn't be - */ - hdev->asic_funcs->enable_events_from_fw(hdev); - } else { - if (!reset_upon_device_release) - hdev->reset_info.compute_reset_cnt++; - - if (schedule_hard_reset) { - dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n"); - flags = hdev->reset_info.hard_reset_schedule_flags; - hdev->reset_info.hard_reset_schedule_flags = 0; - hdev->disabled = true; - hard_reset = true; - handle_reset_trigger(hdev, flags); - goto again; - } - } - - return 0; - -out_err: - hdev->disabled = true; - - spin_lock(&hdev->reset_info.lock); - hdev->reset_info.in_compute_reset = 0; - - if (hard_reset) { - dev_err(hdev->dev, "Failed to reset! Device is NOT usable\n"); - hdev->reset_info.hard_reset_cnt++; - } else if (reset_upon_device_release) { - spin_unlock(&hdev->reset_info.lock); - dev_err(hdev->dev, "Failed to reset device after user release\n"); - flags |= HL_DRV_RESET_HARD; - flags &= ~HL_DRV_RESET_DEV_RELEASE; - hard_reset = true; - goto again; - } else { - spin_unlock(&hdev->reset_info.lock); - dev_err(hdev->dev, "Failed to do compute reset\n"); - hdev->reset_info.compute_reset_cnt++; - flags |= HL_DRV_RESET_HARD; - hard_reset = true; - goto again; - } - - hdev->reset_info.in_reset = 0; - - spin_unlock(&hdev->reset_info.lock); - - return rc; -} - -/* - * hl_device_cond_reset() - conditionally reset the device. - * @hdev: pointer to habanalabs device structure. - * @reset_flags: reset flags. - * @event_mask: events to notify user about. - * - * Conditionally reset the device, or alternatively schedule a watchdog work to reset the device - * unless another reset precedes it. - */ -int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask) -{ - struct hl_ctx *ctx = NULL; - - /* Device release watchdog is only for hard reset */ - if (!(flags & HL_DRV_RESET_HARD) && hdev->asic_prop.allow_inference_soft_reset) - goto device_reset; - - /* F/W reset cannot be postponed */ - if (flags & HL_DRV_RESET_BYPASS_REQ_TO_FW) - goto device_reset; - - /* Device release watchdog is relevant only if user exists and gets a reset notification */ - if (!(event_mask & HL_NOTIFIER_EVENT_DEVICE_RESET)) { - dev_err(hdev->dev, "Resetting device without a reset indication to user\n"); - goto device_reset; - } - - ctx = hl_get_compute_ctx(hdev); - if (!ctx || !ctx->hpriv->notifier_event.eventfd) - goto device_reset; - - /* Schedule the device release watchdog work unless reset is already in progress or if the - * work is already scheduled. - */ - spin_lock(&hdev->reset_info.lock); - if (hdev->reset_info.in_reset) { - spin_unlock(&hdev->reset_info.lock); - goto device_reset; - } - - if (hdev->reset_info.watchdog_active) - goto out; - - hdev->device_release_watchdog_work.flags = flags; - dev_dbg(hdev->dev, "Device is going to be reset in %u sec unless being released\n", - hdev->device_release_watchdog_timeout_sec); - schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work, - msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000)); - hdev->reset_info.watchdog_active = 1; -out: - spin_unlock(&hdev->reset_info.lock); - - hl_notifier_event_send_all(hdev, event_mask); - - hl_ctx_put(ctx); - - return 0; - -device_reset: - if (event_mask) - hl_notifier_event_send_all(hdev, event_mask); - if (ctx) - hl_ctx_put(ctx); - - return hl_device_reset(hdev, flags); -} - -static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask) -{ - mutex_lock(¬ifier_event->lock); - notifier_event->events_mask |= event_mask; - - if (notifier_event->eventfd) - eventfd_signal(notifier_event->eventfd, 1); - - mutex_unlock(¬ifier_event->lock); -} - -/* - * hl_notifier_event_send_all - notify all user processes via eventfd - * - * @hdev: pointer to habanalabs device structure - * @event_mask: the occurred event/s - * Returns 0 for success or an error on failure. - */ -void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask) -{ - struct hl_fpriv *hpriv; - - if (!event_mask) { - dev_warn(hdev->dev, "Skip sending zero event"); - return; - } - - mutex_lock(&hdev->fpriv_list_lock); - - list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) - hl_notifier_event_send(&hpriv->notifier_event, event_mask); - - mutex_unlock(&hdev->fpriv_list_lock); - - /* control device */ - mutex_lock(&hdev->fpriv_ctrl_list_lock); - - list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node) - hl_notifier_event_send(&hpriv->notifier_event, event_mask); - - mutex_unlock(&hdev->fpriv_ctrl_list_lock); -} - -/* - * hl_device_init - main initialization function for habanalabs device - * - * @hdev: pointer to habanalabs device structure - * - * Allocate an id for the device, do early initialization and then call the - * ASIC specific initialization functions. Finally, create the cdev and the - * Linux device to expose it to the user - */ -int hl_device_init(struct hl_device *hdev, struct class *hclass) -{ - int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt; - char *name; - bool add_cdev_sysfs_on_err = false; - - hdev->cdev_idx = hdev->id / 2; - - name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx); - if (!name) { - rc = -ENOMEM; - goto out_disabled; - } - - /* Initialize cdev and device structures */ - rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name, - &hdev->cdev, &hdev->dev); - - kfree(name); - - if (rc) - goto out_disabled; - - name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx); - if (!name) { - rc = -ENOMEM; - goto free_dev; - } - - /* Initialize cdev and device structures for control device */ - rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops, - name, &hdev->cdev_ctrl, &hdev->dev_ctrl); - - kfree(name); - - if (rc) - goto free_dev; - - /* Initialize ASIC function pointers and perform early init */ - rc = device_early_init(hdev); - if (rc) - goto free_dev_ctrl; - - user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count + - hdev->asic_prop.user_interrupt_count; - - if (user_interrupt_cnt) { - hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt), - GFP_KERNEL); - if (!hdev->user_interrupt) { - rc = -ENOMEM; - goto early_fini; - } - } - - /* - * Start calling ASIC initialization. First S/W then H/W and finally - * late init - */ - rc = hdev->asic_funcs->sw_init(hdev); - if (rc) - goto free_usr_intr_mem; - - - /* initialize completion structure for multi CS wait */ - hl_multi_cs_completion_init(hdev); - - /* - * Initialize the H/W queues. Must be done before hw_init, because - * there the addresses of the kernel queue are being written to the - * registers of the device - */ - rc = hl_hw_queues_create(hdev); - if (rc) { - dev_err(hdev->dev, "failed to initialize kernel queues\n"); - goto sw_fini; - } - - cq_cnt = hdev->asic_prop.completion_queues_count; - - /* - * Initialize the completion queues. Must be done before hw_init, - * because there the addresses of the completion queues are being - * passed as arguments to request_irq - */ - if (cq_cnt) { - hdev->completion_queue = kcalloc(cq_cnt, - sizeof(*hdev->completion_queue), - GFP_KERNEL); - - if (!hdev->completion_queue) { - dev_err(hdev->dev, - "failed to allocate completion queues\n"); - rc = -ENOMEM; - goto hw_queues_destroy; - } - } - - for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) { - rc = hl_cq_init(hdev, &hdev->completion_queue[i], - hdev->asic_funcs->get_queue_id_for_cq(hdev, i)); - if (rc) { - dev_err(hdev->dev, - "failed to initialize completion queue\n"); - goto cq_fini; - } - hdev->completion_queue[i].cq_idx = i; - } - - hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs, - sizeof(struct hl_cs *), GFP_KERNEL); - if (!hdev->shadow_cs_queue) { - rc = -ENOMEM; - goto cq_fini; - } - - /* - * Initialize the event queue. Must be done before hw_init, - * because there the address of the event queue is being - * passed as argument to request_irq - */ - rc = hl_eq_init(hdev, &hdev->event_queue); - if (rc) { - dev_err(hdev->dev, "failed to initialize event queue\n"); - goto free_shadow_cs_queue; - } - - /* MMU S/W must be initialized before kernel context is created */ - rc = hl_mmu_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); - goto eq_fini; - } - - /* Allocate the kernel context */ - hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); - if (!hdev->kernel_ctx) { - rc = -ENOMEM; - goto mmu_fini; - } - - hdev->is_compute_ctx_active = false; - - hdev->asic_funcs->state_dump_init(hdev); - - hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC; - - hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL; - hl_debugfs_add_device(hdev); - - /* debugfs nodes are created in hl_ctx_init so it must be called after - * hl_debugfs_add_device. - */ - rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); - if (rc) { - dev_err(hdev->dev, "failed to initialize kernel context\n"); - kfree(hdev->kernel_ctx); - goto remove_device_from_debugfs; - } - - rc = hl_cb_pool_init(hdev); - if (rc) { - dev_err(hdev->dev, "failed to initialize CB pool\n"); - goto release_ctx; - } - - rc = hl_dec_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to initialize the decoder module\n"); - goto cb_pool_fini; - } - - /* - * From this point, override rc (=0) in case of an error to allow - * debugging (by adding char devices and create sysfs nodes as part of - * the error flow). - */ - add_cdev_sysfs_on_err = true; - - /* Device is now enabled as part of the initialization requires - * communication with the device firmware to get information that - * is required for the initialization itself - */ - hdev->disabled = false; - - rc = hdev->asic_funcs->hw_init(hdev); - if (rc) { - dev_err(hdev->dev, "failed to initialize the H/W\n"); - rc = 0; - goto out_disabled; - } - - /* Check that the communication with the device is working */ - rc = hdev->asic_funcs->test_queues(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to detect if device is alive\n"); - rc = 0; - goto out_disabled; - } - - rc = device_late_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed late initialization\n"); - rc = 0; - goto out_disabled; - } - - dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", - hdev->asic_name, - hdev->asic_prop.dram_size / SZ_1G); - - rc = hl_vm_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to initialize memory module\n"); - rc = 0; - goto out_disabled; - } - - /* - * Expose devices and sysfs nodes to user. - * From here there is no need to add char devices and create sysfs nodes - * in case of an error. - */ - add_cdev_sysfs_on_err = false; - rc = device_cdev_sysfs_add(hdev); - if (rc) { - dev_err(hdev->dev, - "Failed to add char devices and sysfs nodes\n"); - rc = 0; - goto out_disabled; - } - - /* Need to call this again because the max power might change, - * depending on card type for certain ASICs - */ - if (hdev->asic_prop.set_max_power_on_device_init && - !hdev->asic_prop.fw_security_enabled) - hl_fw_set_max_power(hdev); - - /* - * hl_hwmon_init() must be called after device_late_init(), because only - * there we get the information from the device about which - * hwmon-related sensors the device supports. - * Furthermore, it must be done after adding the device to the system. - */ - rc = hl_hwmon_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to initialize hwmon\n"); - rc = 0; - goto out_disabled; - } - - dev_notice(hdev->dev, - "Successfully added device to habanalabs driver\n"); - - hdev->init_done = true; - - /* After initialization is done, we are ready to receive events from - * the F/W. We can't do it before because we will ignore events and if - * those events are fatal, we won't know about it and the device will - * be operational although it shouldn't be - */ - hdev->asic_funcs->enable_events_from_fw(hdev); - - return 0; - -cb_pool_fini: - hl_cb_pool_fini(hdev); -release_ctx: - if (hl_ctx_put(hdev->kernel_ctx) != 1) - dev_err(hdev->dev, - "kernel ctx is still alive on initialization failure\n"); -remove_device_from_debugfs: - hl_debugfs_remove_device(hdev); -mmu_fini: - hl_mmu_fini(hdev); -eq_fini: - hl_eq_fini(hdev, &hdev->event_queue); -free_shadow_cs_queue: - kfree(hdev->shadow_cs_queue); -cq_fini: - for (i = 0 ; i < cq_ready_cnt ; i++) - hl_cq_fini(hdev, &hdev->completion_queue[i]); - kfree(hdev->completion_queue); -hw_queues_destroy: - hl_hw_queues_destroy(hdev); -sw_fini: - hdev->asic_funcs->sw_fini(hdev); -free_usr_intr_mem: - kfree(hdev->user_interrupt); -early_fini: - device_early_fini(hdev); -free_dev_ctrl: - put_device(hdev->dev_ctrl); -free_dev: - put_device(hdev->dev); -out_disabled: - hdev->disabled = true; - if (add_cdev_sysfs_on_err) - device_cdev_sysfs_add(hdev); - if (hdev->pdev) - dev_err(&hdev->pdev->dev, - "Failed to initialize hl%d. Device is NOT usable !\n", - hdev->cdev_idx); - else - pr_err("Failed to initialize hl%d. Device is NOT usable !\n", - hdev->cdev_idx); - - return rc; -} - -/* - * hl_device_fini - main tear-down function for habanalabs device - * - * @hdev: pointer to habanalabs device structure - * - * Destroy the device, call ASIC fini functions and release the id - */ -void hl_device_fini(struct hl_device *hdev) -{ - bool device_in_reset; - ktime_t timeout; - u64 reset_sec; - int i, rc; - - dev_info(hdev->dev, "Removing device\n"); - - hdev->device_fini_pending = 1; - flush_delayed_work(&hdev->device_reset_work.reset_work); - - if (hdev->pldm) - reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT; - else - reset_sec = HL_HARD_RESET_MAX_TIMEOUT; - - /* - * This function is competing with the reset function, so try to - * take the reset atomic and if we are already in middle of reset, - * wait until reset function is finished. Reset function is designed - * to always finish. However, in Gaudi, because of all the network - * ports, the hard reset could take between 10-30 seconds - */ - - timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000); - - spin_lock(&hdev->reset_info.lock); - device_in_reset = !!hdev->reset_info.in_reset; - if (!device_in_reset) - hdev->reset_info.in_reset = 1; - spin_unlock(&hdev->reset_info.lock); - - while (device_in_reset) { - usleep_range(50, 200); - - spin_lock(&hdev->reset_info.lock); - device_in_reset = !!hdev->reset_info.in_reset; - if (!device_in_reset) - hdev->reset_info.in_reset = 1; - spin_unlock(&hdev->reset_info.lock); - - if (ktime_compare(ktime_get(), timeout) > 0) { - dev_crit(hdev->dev, - "Failed to remove device because reset function did not finish\n"); - return; - } - } - - cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work); - - /* Disable PCI access from device F/W so it won't send us additional - * interrupts. We disable MSI/MSI-X at the halt_engines function and we - * can't have the F/W sending us interrupts after that. We need to - * disable the access here because if the device is marked disable, the - * message won't be send. Also, in case of heartbeat, the device CPU is - * marked as disable so this message won't be sent - */ - hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); - - /* Mark device as disabled */ - hdev->disabled = true; - - take_release_locks(hdev); - - hdev->reset_info.hard_reset_pending = true; - - hl_hwmon_fini(hdev); - - cleanup_resources(hdev, true, false, false); - - /* Kill processes here after CS rollback. This is because the process - * can't really exit until all its CSs are done, which is what we - * do in cs rollback - */ - dev_info(hdev->dev, - "Waiting for all processes to exit (timeout of %u seconds)", - HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI); - - hdev->process_kill_trial_cnt = 0; - rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false); - if (rc) { - dev_crit(hdev->dev, "Failed to kill all open processes\n"); - device_disable_open_processes(hdev, false); - } - - hdev->process_kill_trial_cnt = 0; - rc = device_kill_open_processes(hdev, 0, true); - if (rc) { - dev_crit(hdev->dev, "Failed to kill all control device open processes\n"); - device_disable_open_processes(hdev, true); - } - - hl_cb_pool_fini(hdev); - - /* Reset the H/W. It will be in idle state after this returns */ - hdev->asic_funcs->hw_fini(hdev, true, false); - - hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; - - /* Release kernel context */ - if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) - dev_err(hdev->dev, "kernel ctx is still alive\n"); - - hl_debugfs_remove_device(hdev); - - hl_dec_fini(hdev); - - hl_vm_fini(hdev); - - hl_mmu_fini(hdev); - - vfree(hdev->captured_err_info.pgf_info.user_mappings); - - hl_eq_fini(hdev, &hdev->event_queue); - - kfree(hdev->shadow_cs_queue); - - for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) - hl_cq_fini(hdev, &hdev->completion_queue[i]); - kfree(hdev->completion_queue); - kfree(hdev->user_interrupt); - - hl_hw_queues_destroy(hdev); - - /* Call ASIC S/W finalize function */ - hdev->asic_funcs->sw_fini(hdev); - - device_early_fini(hdev); - - /* Hide devices and sysfs nodes from user */ - device_cdev_sysfs_del(hdev); - - pr_info("removed device successfully\n"); -} - -/* - * MMIO register access helper functions. - */ - -/* - * hl_rreg - Read an MMIO register - * - * @hdev: pointer to habanalabs device structure - * @reg: MMIO register offset (in bytes) - * - * Returns the value of the MMIO register we are asked to read - * - */ -inline u32 hl_rreg(struct hl_device *hdev, u32 reg) -{ - return readl(hdev->rmmio + reg); -} - -/* - * hl_wreg - Write to an MMIO register - * - * @hdev: pointer to habanalabs device structure - * @reg: MMIO register offset (in bytes) - * @val: 32-bit value - * - * Writes the 32-bit value into the MMIO register - * - */ -inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val) -{ - writel(val, hdev->rmmio + reg); -} - -void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, - u8 flags) -{ - if (num_of_engines > HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR) { - dev_err(hdev->dev, - "Number of possible razwi initiators (%u) exceeded limit (%u)\n", - num_of_engines, HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR); - return; - } - - /* In case it's the first razwi since the device was opened, capture its parameters */ - if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info_recorded, 0, 1)) - return; - - hdev->captured_err_info.razwi.timestamp = ktime_to_ns(ktime_get()); - hdev->captured_err_info.razwi.addr = addr; - hdev->captured_err_info.razwi.num_of_possible_engines = num_of_engines; - memcpy(&hdev->captured_err_info.razwi.engine_id[0], &engine_id[0], - num_of_engines * sizeof(u16)); - hdev->captured_err_info.razwi.flags = flags; -} - -void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, - u8 flags, u64 *event_mask) -{ - hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags); - - if (event_mask) - *event_mask |= HL_NOTIFIER_EVENT_RAZWI; -} - -static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu) -{ - struct page_fault_info *pgf_info = &hdev->captured_err_info.pgf_info; - struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; - struct hl_vm_hash_node *hnode; - struct hl_userptr *userptr; - enum vm_type *vm_type; - struct hl_ctx *ctx; - u32 map_idx = 0; - int i; - - /* Reset previous session count*/ - pgf_info->num_of_user_mappings = 0; - - ctx = hl_get_compute_ctx(hdev); - if (!ctx) { - dev_err(hdev->dev, "Can't get user context for user mappings\n"); - return; - } - - mutex_lock(&ctx->mem_hash_lock); - hash_for_each(ctx->mem_hash, i, hnode, node) { - vm_type = hnode->ptr; - if (((*vm_type == VM_TYPE_USERPTR) && is_pmmu) || - ((*vm_type == VM_TYPE_PHYS_PACK) && !is_pmmu)) - pgf_info->num_of_user_mappings++; - - } - - if (!pgf_info->num_of_user_mappings) - goto finish; - - /* In case we already allocated in previous session, need to release it before - * allocating new buffer. - */ - vfree(pgf_info->user_mappings); - pgf_info->user_mappings = - vzalloc(pgf_info->num_of_user_mappings * sizeof(struct hl_user_mapping)); - if (!pgf_info->user_mappings) { - pgf_info->num_of_user_mappings = 0; - goto finish; - } - - hash_for_each(ctx->mem_hash, i, hnode, node) { - vm_type = hnode->ptr; - if ((*vm_type == VM_TYPE_USERPTR) && (is_pmmu)) { - userptr = hnode->ptr; - pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr; - pgf_info->user_mappings[map_idx].size = userptr->size; - map_idx++; - } else if ((*vm_type == VM_TYPE_PHYS_PACK) && (!is_pmmu)) { - phys_pg_pack = hnode->ptr; - pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr; - pgf_info->user_mappings[map_idx].size = phys_pg_pack->total_size; - map_idx++; - } - } -finish: - mutex_unlock(&ctx->mem_hash_lock); - hl_ctx_put(ctx); -} - -void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu) -{ - /* Capture only the first page fault */ - if (atomic_cmpxchg(&hdev->captured_err_info.pgf_info_recorded, 0, 1)) - return; - - hdev->captured_err_info.pgf_info.pgf.timestamp = ktime_to_ns(ktime_get()); - hdev->captured_err_info.pgf_info.pgf.addr = addr; - hdev->captured_err_info.pgf_info.pgf.engine_id = eng_id; - hl_capture_user_mappings(hdev, is_pmmu); -} - -void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu, - u64 *event_mask) -{ - hl_capture_page_fault(hdev, addr, eng_id, is_pmmu); - - if (event_mask) - *event_mask |= HL_NOTIFIER_EVENT_PAGE_FAULT; -} |