summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/ttm/ttm_bo_vm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo_vm.c')
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_vm.c454
1 files changed, 454 insertions, 0 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
new file mode 100644
index 000000000000..27b146c54fbc
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -0,0 +1,454 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#include <ttm/ttm_module.h>
+#include <ttm/ttm_bo_driver.h>
+#include <ttm/ttm_placement.h>
+#include <linux/mm.h>
+#include <linux/version.h>
+#include <linux/rbtree.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+
+#define TTM_BO_VM_NUM_PREFAULT 16
+
+static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev,
+ unsigned long page_start,
+ unsigned long num_pages)
+{
+ struct rb_node *cur = bdev->addr_space_rb.rb_node;
+ unsigned long cur_offset;
+ struct ttm_buffer_object *bo;
+ struct ttm_buffer_object *best_bo = NULL;
+
+ while (likely(cur != NULL)) {
+ bo = rb_entry(cur, struct ttm_buffer_object, vm_rb);
+ cur_offset = bo->vm_node->start;
+ if (page_start >= cur_offset) {
+ cur = cur->rb_right;
+ best_bo = bo;
+ if (page_start == cur_offset)
+ break;
+ } else
+ cur = cur->rb_left;
+ }
+
+ if (unlikely(best_bo == NULL))
+ return NULL;
+
+ if (unlikely((best_bo->vm_node->start + best_bo->num_pages) <
+ (page_start + num_pages)))
+ return NULL;
+
+ return best_bo;
+}
+
+static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
+ vma->vm_private_data;
+ struct ttm_bo_device *bdev = bo->bdev;
+ unsigned long bus_base;
+ unsigned long bus_offset;
+ unsigned long bus_size;
+ unsigned long page_offset;
+ unsigned long page_last;
+ unsigned long pfn;
+ struct ttm_tt *ttm = NULL;
+ struct page *page;
+ int ret;
+ int i;
+ bool is_iomem;
+ unsigned long address = (unsigned long)vmf->virtual_address;
+ int retval = VM_FAULT_NOPAGE;
+
+ /*
+ * Work around locking order reversal in fault / nopfn
+ * between mmap_sem and bo_reserve: Perform a trylock operation
+ * for reserve, and if it fails, retry the fault after scheduling.
+ */
+
+ ret = ttm_bo_reserve(bo, true, true, false, 0);
+ if (unlikely(ret != 0)) {
+ if (ret == -EBUSY)
+ set_need_resched();
+ return VM_FAULT_NOPAGE;
+ }
+
+ /*
+ * Wait for buffer data in transit, due to a pipelined
+ * move.
+ */
+
+ spin_lock(&bo->lock);
+ if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
+ ret = ttm_bo_wait(bo, false, true, false);
+ spin_unlock(&bo->lock);
+ if (unlikely(ret != 0)) {
+ retval = (ret != -ERESTART) ?
+ VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
+ goto out_unlock;
+ }
+ } else
+ spin_unlock(&bo->lock);
+
+
+ ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset,
+ &bus_size);
+ if (unlikely(ret != 0)) {
+ retval = VM_FAULT_SIGBUS;
+ goto out_unlock;
+ }
+
+ is_iomem = (bus_size != 0);
+
+ page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
+ bo->vm_node->start - vma->vm_pgoff;
+ page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) +
+ bo->vm_node->start - vma->vm_pgoff;
+
+ if (unlikely(page_offset >= bo->num_pages)) {
+ retval = VM_FAULT_SIGBUS;
+ goto out_unlock;
+ }
+
+ /*
+ * Strictly, we're not allowed to modify vma->vm_page_prot here,
+ * since the mmap_sem is only held in read mode. However, we
+ * modify only the caching bits of vma->vm_page_prot and
+ * consider those bits protected by
+ * the bo->mutex, as we should be the only writers.
+ * There shouldn't really be any readers of these bits except
+ * within vm_insert_mixed()? fork?
+ *
+ * TODO: Add a list of vmas to the bo, and change the
+ * vma->vm_page_prot when the object changes caching policy, with
+ * the correct locks held.
+ */
+
+ if (is_iomem) {
+ vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
+ vma->vm_page_prot);
+ } else {
+ ttm = bo->ttm;
+ vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
+ vm_get_page_prot(vma->vm_flags) :
+ ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
+ }
+
+ /*
+ * Speculatively prefault a number of pages. Only error on
+ * first page.
+ */
+
+ for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
+
+ if (is_iomem)
+ pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) +
+ page_offset;
+ else {
+ page = ttm_tt_get_page(ttm, page_offset);
+ if (unlikely(!page && i == 0)) {
+ retval = VM_FAULT_OOM;
+ goto out_unlock;
+ } else if (unlikely(!page)) {
+ break;
+ }
+ pfn = page_to_pfn(page);
+ }
+
+ ret = vm_insert_mixed(vma, address, pfn);
+ /*
+ * Somebody beat us to this PTE or prefaulting to
+ * an already populated PTE, or prefaulting error.
+ */
+
+ if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
+ break;
+ else if (unlikely(ret != 0)) {
+ retval =
+ (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
+ goto out_unlock;
+
+ }
+
+ address += PAGE_SIZE;
+ if (unlikely(++page_offset >= page_last))
+ break;
+ }
+
+out_unlock:
+ ttm_bo_unreserve(bo);
+ return retval;
+}
+
+static void ttm_bo_vm_open(struct vm_area_struct *vma)
+{
+ struct ttm_buffer_object *bo =
+ (struct ttm_buffer_object *)vma->vm_private_data;
+
+ (void)ttm_bo_reference(bo);
+}
+
+static void ttm_bo_vm_close(struct vm_area_struct *vma)
+{
+ struct ttm_buffer_object *bo =
+ (struct ttm_buffer_object *)vma->vm_private_data;
+
+ ttm_bo_unref(&bo);
+ vma->vm_private_data = NULL;
+}
+
+static struct vm_operations_struct ttm_bo_vm_ops = {
+ .fault = ttm_bo_vm_fault,
+ .open = ttm_bo_vm_open,
+ .close = ttm_bo_vm_close
+};
+
+int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
+ struct ttm_bo_device *bdev)
+{
+ struct ttm_bo_driver *driver;
+ struct ttm_buffer_object *bo;
+ int ret;
+
+ read_lock(&bdev->vm_lock);
+ bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff,
+ (vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
+ if (likely(bo != NULL))
+ ttm_bo_reference(bo);
+ read_unlock(&bdev->vm_lock);
+
+ if (unlikely(bo == NULL)) {
+ printk(KERN_ERR TTM_PFX
+ "Could not find buffer object to map.\n");
+ return -EINVAL;
+ }
+
+ driver = bo->bdev->driver;
+ if (unlikely(!driver->verify_access)) {
+ ret = -EPERM;
+ goto out_unref;
+ }
+ ret = driver->verify_access(bo, filp);
+ if (unlikely(ret != 0))
+ goto out_unref;
+
+ vma->vm_ops = &ttm_bo_vm_ops;
+
+ /*
+ * Note: We're transferring the bo reference to
+ * vma->vm_private_data here.
+ */
+
+ vma->vm_private_data = bo;
+ vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
+ return 0;
+out_unref:
+ ttm_bo_unref(&bo);
+ return ret;
+}
+EXPORT_SYMBOL(ttm_bo_mmap);
+
+int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
+{
+ if (vma->vm_pgoff != 0)
+ return -EACCES;
+
+ vma->vm_ops = &ttm_bo_vm_ops;
+ vma->vm_private_data = ttm_bo_reference(bo);
+ vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
+ return 0;
+}
+EXPORT_SYMBOL(ttm_fbdev_mmap);
+
+
+ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
+ const char __user *wbuf, char __user *rbuf, size_t count,
+ loff_t *f_pos, bool write)
+{
+ struct ttm_buffer_object *bo;
+ struct ttm_bo_driver *driver;
+ struct ttm_bo_kmap_obj map;
+ unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
+ unsigned long kmap_offset;
+ unsigned long kmap_end;
+ unsigned long kmap_num;
+ size_t io_size;
+ unsigned int page_offset;
+ char *virtual;
+ int ret;
+ bool no_wait = false;
+ bool dummy;
+
+ read_lock(&bdev->vm_lock);
+ bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1);
+ if (likely(bo != NULL))
+ ttm_bo_reference(bo);
+ read_unlock(&bdev->vm_lock);
+
+ if (unlikely(bo == NULL))
+ return -EFAULT;
+
+ driver = bo->bdev->driver;
+ if (unlikely(driver->verify_access)) {
+ ret = -EPERM;
+ goto out_unref;
+ }
+
+ ret = driver->verify_access(bo, filp);
+ if (unlikely(ret != 0))
+ goto out_unref;
+
+ kmap_offset = dev_offset - bo->vm_node->start;
+ if (unlikely(kmap_offset) >= bo->num_pages) {
+ ret = -EFBIG;
+ goto out_unref;
+ }
+
+ page_offset = *f_pos & ~PAGE_MASK;
+ io_size = bo->num_pages - kmap_offset;
+ io_size = (io_size << PAGE_SHIFT) - page_offset;
+ if (count < io_size)
+ io_size = count;
+
+ kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
+ kmap_num = kmap_end - kmap_offset + 1;
+
+ ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
+
+ switch (ret) {
+ case 0:
+ break;
+ case -ERESTART:
+ ret = -EINTR;
+ goto out_unref;
+ case -EBUSY:
+ ret = -EAGAIN;
+ goto out_unref;
+ default:
+ goto out_unref;
+ }
+
+ ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
+ if (unlikely(ret != 0)) {
+ ttm_bo_unreserve(bo);
+ goto out_unref;
+ }
+
+ virtual = ttm_kmap_obj_virtual(&map, &dummy);
+ virtual += page_offset;
+
+ if (write)
+ ret = copy_from_user(virtual, wbuf, io_size);
+ else
+ ret = copy_to_user(rbuf, virtual, io_size);
+
+ ttm_bo_kunmap(&map);
+ ttm_bo_unreserve(bo);
+ ttm_bo_unref(&bo);
+
+ if (unlikely(ret != 0))
+ return -EFBIG;
+
+ *f_pos += io_size;
+
+ return io_size;
+out_unref:
+ ttm_bo_unref(&bo);
+ return ret;
+}
+
+ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
+ char __user *rbuf, size_t count, loff_t *f_pos,
+ bool write)
+{
+ struct ttm_bo_kmap_obj map;
+ unsigned long kmap_offset;
+ unsigned long kmap_end;
+ unsigned long kmap_num;
+ size_t io_size;
+ unsigned int page_offset;
+ char *virtual;
+ int ret;
+ bool no_wait = false;
+ bool dummy;
+
+ kmap_offset = (*f_pos >> PAGE_SHIFT);
+ if (unlikely(kmap_offset) >= bo->num_pages)
+ return -EFBIG;
+
+ page_offset = *f_pos & ~PAGE_MASK;
+ io_size = bo->num_pages - kmap_offset;
+ io_size = (io_size << PAGE_SHIFT) - page_offset;
+ if (count < io_size)
+ io_size = count;
+
+ kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
+ kmap_num = kmap_end - kmap_offset + 1;
+
+ ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
+
+ switch (ret) {
+ case 0:
+ break;
+ case -ERESTART:
+ return -EINTR;
+ case -EBUSY:
+ return -EAGAIN;
+ default:
+ return ret;
+ }
+
+ ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
+ if (unlikely(ret != 0)) {
+ ttm_bo_unreserve(bo);
+ return ret;
+ }
+
+ virtual = ttm_kmap_obj_virtual(&map, &dummy);
+ virtual += page_offset;
+
+ if (write)
+ ret = copy_from_user(virtual, wbuf, io_size);
+ else
+ ret = copy_to_user(rbuf, virtual, io_size);
+
+ ttm_bo_kunmap(&map);
+ ttm_bo_unreserve(bo);
+ ttm_bo_unref(&bo);
+
+ if (unlikely(ret != 0))
+ return ret;
+
+ *f_pos += io_size;
+
+ return io_size;
+}