diff options
Diffstat (limited to 'drivers/gpu/drm')
| -rw-r--r-- | drivers/gpu/drm/nouveau/include/nvif/class.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/include/nvif/if000c.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 16 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 76 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 147 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 111 |
9 files changed, 350 insertions, 19 deletions
diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index d08da82ba7ed..7f0839615b0c 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -14,6 +14,8 @@ #define NVIF_CLASS_SW_NV50 /* if0005.h */ -0x00000006 #define NVIF_CLASS_SW_GF100 /* if0005.h */ -0x00000007 +#define NVIF_CLASS_VMM /* if000c.h */ 0x0000000c + /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000c.h b/drivers/gpu/drm/nouveau/include/nvif/if000c.h new file mode 100644 index 000000000000..6363e22f9dcc --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if000c.h @@ -0,0 +1,3 @@ +#ifndef __NVIF_IF000C_H__ +#define __NVIF_IF000C_H__ +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 40e21f23b41d..8ff71e7f2f5b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -26,20 +26,28 @@ struct nvkm_vma { }; struct nvkm_vm { + const struct nvkm_vmm_func *func; struct nvkm_mmu *mmu; - + const char *name; + struct kref kref; struct mutex mutex; + + u64 start; + u64 limit; + + struct nvkm_vmm_pt *pd; + u16 pd_offset; + struct list_head join; + struct nvkm_mm mm; struct kref refcount; - struct list_head pgd_list; - atomic_t engref[NVKM_SUBDEV_NR]; - struct nvkm_vm_pgt *pgt; u32 fpde; u32 lpde; bool bootstrapped; + atomic_t engref[NVKM_SUBDEV_NR]; }; int nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index e84ef36eb7e7..c786a9f90776 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -11,3 +11,5 @@ nvkm-y += nvkm/subdev/mmu/gm200.o nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o + +nvkm-y += nvkm/subdev/mmu/vmm.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 1bdae020057e..d55ec0e85dca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "priv.h" +#include "vmm.h" #include <core/gpuobj.h> #include <subdev/fb.h> @@ -584,22 +585,14 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size) return ret; } -int -nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, - u32 block, struct lock_class_key *key, struct nvkm_vm **pvm) +static int +nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, + u32 block, struct nvkm_vm *vm) { - static struct lock_class_key _key; - struct nvkm_vm *vm; u64 mm_length = (offset + length) - mm_offset; int ret; - vm = kzalloc(sizeof(*vm), GFP_KERNEL); - if (!vm) - return -ENOMEM; - - __mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key); INIT_LIST_HEAD(&vm->pgd_list); - vm->mmu = mmu; kref_init(&vm->refcount); vm->fpde = offset >> (mmu->func->pgt_bits + 12); vm->lpde = (offset + length - 1) >> (mmu->func->pgt_bits + 12); @@ -610,16 +603,41 @@ nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, return -ENOMEM; } + if (block > length) + block = length; + ret = nvkm_mm_init(&vm->mm, 0, mm_offset >> 12, mm_length >> 12, block >> 12); if (ret) { vfree(vm->pgt); + return ret; + } + + return 0; +} + +int +nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, + u32 block, struct lock_class_key *key, struct nvkm_vm **pvm) +{ + static struct lock_class_key _key; + struct nvkm_vm *vm; + int ret; + + vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return -ENOMEM; + + __mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key); + vm->mmu = mmu; + + ret = nvkm_vm_legacy(mmu, offset, length, mm_offset, block, vm); + if (ret) { kfree(vm); return ret; } *pvm = vm; - return 0; } @@ -628,8 +646,29 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *key, struct nvkm_vm **pvm) { struct nvkm_mmu *mmu = device->mmu; + + *pvm = NULL; + if (mmu->func->vmm.ctor) { + int ret = mmu->func->vmm.ctor(mmu, mm_offset, + offset + length - mm_offset, + NULL, 0, key, "legacy", pvm); + if (ret) { + nvkm_vm_ref(NULL, pvm, NULL); + return ret; + } + + ret = nvkm_vm_legacy(mmu, offset, length, mm_offset, + (*pvm)->func->page_block ? + (*pvm)->func->page_block : 4096, *pvm); + if (ret) + nvkm_vm_ref(NULL, pvm, NULL); + + return ret; + } + if (!mmu->func->create) return -EINVAL; + return mmu->func->create(mmu, offset, length, mm_offset, key, pvm); } @@ -688,6 +727,9 @@ nvkm_vm_del(struct kref *kref) nvkm_mm_fini(&vm->mm); vfree(vm->pgt); + + if (vm->func) + nvkm_vmm_dtor(vm); kfree(vm); } @@ -717,8 +759,17 @@ static int nvkm_mmu_oneinit(struct nvkm_subdev *subdev) { struct nvkm_mmu *mmu = nvkm_mmu(subdev); + + if (mmu->func->vmm.global) { + int ret = nvkm_vm_new(subdev->device, 0, mmu->limit, 0, + NULL, &mmu->vmm); + if (ret) + return ret; + } + if (mmu->func->oneinit) return mmu->func->oneinit(mmu); + return 0; } @@ -739,6 +790,7 @@ nvkm_mmu_dtor(struct nvkm_subdev *subdev) if (mmu->func->dtor) data = mmu->func->dtor(mmu); + nvkm_vm_ref(NULL, &mmu->vmm, NULL); nvkm_mmu_ptc_fini(mmu); return data; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 43e3b4a77583..5bf757541f7b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -105,10 +105,8 @@ nv04_mmu_dtor(struct nvkm_mmu *base) { struct nv04_mmu *mmu = nv04_mmu(base); struct nvkm_device *device = mmu->base.subdev.device; - if (mmu->base.vmm) { + if (mmu->base.vmm) nvkm_memory_unref(&mmu->base.vmm->pgt[0].mem[0]); - nvkm_vm_ref(NULL, &mmu->base.vmm, NULL); - } if (mmu->nullp) { dma_free_coherent(device->dev, 16 * 1024, mmu->nullp, mmu->null); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 0f3bb316022a..ee0a8416df19 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -32,6 +32,14 @@ struct nvkm_mmu_func { void (*unmap)(struct nvkm_vma *, struct nvkm_memory *pgt, u32 pte, u32 cnt); void (*flush)(struct nvkm_vm *); + + struct { + struct nvkm_sclass base; + int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *, + const char *name, struct nvkm_vmm **); + bool global; + } vmm; }; int nvkm_vm_create(struct nvkm_mmu *, u64, u64, u64, u32, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c new file mode 100644 index 000000000000..93c8398b36a5 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -0,0 +1,147 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define NVKM_VMM_LEVELS_MAX 5 +#include "vmm.h" + +static void +nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt) +{ + struct nvkm_vmm_pt *pgt = *ppgt; + if (pgt) { + kvfree(pgt->pde); + kfree(pgt); + *ppgt = NULL; + } +} + + +static struct nvkm_vmm_pt * +nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse, + const struct nvkm_vmm_page *page) +{ + const u32 pten = 1 << desc->bits; + struct nvkm_vmm_pt *pgt; + u32 lpte = 0; + + if (desc->type > PGT) { + if (desc->type == SPT) { + const struct nvkm_vmm_desc *pair = page[-1].desc; + lpte = pten >> (desc->bits - pair->bits); + } else { + lpte = pten; + } + } + + if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL))) + return NULL; + pgt->page = page ? page->shift : 0; + pgt->sparse = sparse; + + if (desc->type == PGD) { + pgt->pde = kvzalloc(sizeof(*pgt->pde) * pten, GFP_KERNEL); + if (!pgt->pde) { + kfree(pgt); + return NULL; + } + } + + return pgt; +} + +void +nvkm_vmm_dtor(struct nvkm_vmm *vmm) +{ + if (vmm->pd) { + nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]); + nvkm_vmm_pt_del(&vmm->pd); + } +} + +int +nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, + u32 pd_header, u64 addr, u64 size, struct lock_class_key *key, + const char *name, struct nvkm_vmm *vmm) +{ + static struct lock_class_key _key; + const struct nvkm_vmm_page *page = func->page; + const struct nvkm_vmm_desc *desc; + int levels, bits = 0; + + vmm->func = func; + vmm->mmu = mmu; + vmm->name = name; + kref_init(&vmm->kref); + + __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key); + + /* Locate the smallest page size supported by the backend, it will + * have the the deepest nesting of page tables. + */ + while (page[1].shift) + page++; + + /* Locate the structure that describes the layout of the top-level + * page table, and determine the number of valid bits in a virtual + * address. + */ + for (levels = 0, desc = page->desc; desc->bits; desc++, levels++) + bits += desc->bits; + bits += page->shift; + desc--; + + if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX)) + return -EINVAL; + + vmm->start = addr; + vmm->limit = size ? (addr + size) : (1ULL << bits); + if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) + return -EINVAL; + + /* Allocate top-level page table. */ + vmm->pd = nvkm_vmm_pt_new(desc, false, NULL); + if (!vmm->pd) + return -ENOMEM; + vmm->pd->refs[0] = 1; + INIT_LIST_HEAD(&vmm->join); + + /* ... and the GPU storage for it, except on Tesla-class GPUs that + * have the PD embedded in the instance structure. + */ + if (desc->size && mmu->func->vmm.global) { + const u32 size = pd_header + desc->size * (1 << desc->bits); + vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true); + if (!vmm->pd->pt[0]) + return -ENOMEM; + } + + return 0; +} + +int +nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, + u32 hdr, u64 addr, u64 size, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL))) + return -ENOMEM; + return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h new file mode 100644 index 000000000000..46cc194b2003 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -0,0 +1,111 @@ +#ifndef __NVKM_VMM_H__ +#define __NVKM_VMM_H__ +#include "priv.h" +#include <core/memory.h> + +struct nvkm_vmm_pt { + /* Some GPUs have a mapping level with a dual page tables to + * support large and small pages in the same address-range. + * + * We track the state of both page tables in one place, which + * is why there's multiple PT pointers/refcounts here. + */ + struct nvkm_mmu_pt *pt[2]; + u32 refs[2]; + + /* Page size handled by this PT. + * + * Tesla backend needs to know this when writinge PDEs, + * otherwise unnecessary. + */ + u8 page; + + /* Entire page table sparse. + * + * Used to propagate sparseness to child page tables. + */ + bool sparse:1; + + /* Tracking for page directories. + * + * The array is indexed by PDE, and will either point to the + * child page table, or indicate the PDE is marked as sparse. + **/ +#define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) +#define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) +#define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) + struct nvkm_vmm_pt **pde; + + /* Tracking for dual page tables. + * + * There's one entry for each LPTE, keeping track of whether + * there are valid SPTEs in the same address-range. + * + * This information is used to manage LPTE state transitions. + */ +#define NVKM_VMM_PTE_SPARSE 0x80 +#define NVKM_VMM_PTE_VALID 0x40 +#define NVKM_VMM_PTE_SPTES 0x3f + u8 pte[]; +}; + +struct nvkm_vmm_desc_func { +}; + +struct nvkm_vmm_desc { + enum { + PGD, + PGT, + SPT, + LPT, + } type; + u8 bits; /* VMA bits covered by PT. */ + u8 size; /* Bytes-per-PTE. */ + u32 align; /* PT address alignment. */ + const struct nvkm_vmm_desc_func *func; +}; + +struct nvkm_vmm_page { + u8 shift; + const struct nvkm_vmm_desc *desc; +#define NVKM_VMM_PAGE_SPARSE 0x01 +#define NVKM_VMM_PAGE_VRAM 0x02 +#define NVKM_VMM_PAGE_HOST 0x04 +#define NVKM_VMM_PAGE_COMP 0x08 +#define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) +#define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) +#define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) +#define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) +#define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) +#define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) +#define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) + u8 type; +}; + +struct nvkm_vmm_func { + int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); + void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); + + u64 page_block; + const struct nvkm_vmm_page page[]; +}; + +int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, + u32 pd_header, u64 addr, u64 size, struct lock_class_key *, + const char *name, struct nvkm_vmm **); +int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, + u32 pd_header, u64 addr, u64 size, struct lock_class_key *, + const char *name, struct nvkm_vmm *); +void nvkm_vmm_dtor(struct nvkm_vmm *); + +struct nvkm_vmm_user { + struct nvkm_sclass base; + int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, void *args, u32 argc, + struct lock_class_key *, const char *name, + struct nvkm_vmm **); +}; +#endif |