diff options
Diffstat (limited to 'drivers/vfio')
| -rw-r--r-- | drivers/vfio/fsl-mc/vfio_fsl_mc.c | 127 | ||||
| -rw-r--r-- | drivers/vfio/fsl-mc/vfio_fsl_mc_private.h | 1 | ||||
| -rw-r--r-- | drivers/vfio/mdev/mdev_private.h | 7 | ||||
| -rw-r--r-- | drivers/vfio/mdev/vfio_mdev.c | 53 | ||||
| -rw-r--r-- | drivers/vfio/pci/vfio_pci.c | 256 | ||||
| -rw-r--r-- | drivers/vfio/pci/vfio_pci_config.c | 2 | ||||
| -rw-r--r-- | drivers/vfio/pci/vfio_pci_igd.c | 53 | ||||
| -rw-r--r-- | drivers/vfio/pci/vfio_pci_private.h | 1 | ||||
| -rw-r--r-- | drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c | 2 | ||||
| -rw-r--r-- | drivers/vfio/platform/vfio_amba.c | 8 | ||||
| -rw-r--r-- | drivers/vfio/platform/vfio_platform.c | 20 | ||||
| -rw-r--r-- | drivers/vfio/platform/vfio_platform_common.c | 56 | ||||
| -rw-r--r-- | drivers/vfio/platform/vfio_platform_private.h | 5 | ||||
| -rw-r--r-- | drivers/vfio/vfio.c | 210 | ||||
| -rw-r--r-- | drivers/vfio/vfio_iommu_type1.c | 14 |
15 files changed, 431 insertions, 384 deletions
diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c index f27e25112c40..980e59551301 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c @@ -75,7 +75,8 @@ static int vfio_fsl_mc_reflck_attach(struct vfio_fsl_mc_device *vdev) goto unlock; } - cont_vdev = vfio_device_data(device); + cont_vdev = + container_of(device, struct vfio_fsl_mc_device, vdev); if (!cont_vdev || !cont_vdev->reflck) { vfio_device_put(device); ret = -ENODEV; @@ -135,9 +136,10 @@ static void vfio_fsl_mc_regions_cleanup(struct vfio_fsl_mc_device *vdev) kfree(vdev->regions); } -static int vfio_fsl_mc_open(void *device_data) +static int vfio_fsl_mc_open(struct vfio_device *core_vdev) { - struct vfio_fsl_mc_device *vdev = device_data; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); int ret; if (!try_module_get(THIS_MODULE)) @@ -161,9 +163,10 @@ err_reg_init: return ret; } -static void vfio_fsl_mc_release(void *device_data) +static void vfio_fsl_mc_release(struct vfio_device *core_vdev) { - struct vfio_fsl_mc_device *vdev = device_data; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); int ret; mutex_lock(&vdev->reflck->lock); @@ -197,11 +200,12 @@ static void vfio_fsl_mc_release(void *device_data) module_put(THIS_MODULE); } -static long vfio_fsl_mc_ioctl(void *device_data, unsigned int cmd, - unsigned long arg) +static long vfio_fsl_mc_ioctl(struct vfio_device *core_vdev, + unsigned int cmd, unsigned long arg) { unsigned long minsz; - struct vfio_fsl_mc_device *vdev = device_data; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); struct fsl_mc_device *mc_dev = vdev->mc_dev; switch (cmd) { @@ -327,10 +331,11 @@ static long vfio_fsl_mc_ioctl(void *device_data, unsigned int cmd, } } -static ssize_t vfio_fsl_mc_read(void *device_data, char __user *buf, +static ssize_t vfio_fsl_mc_read(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos) { - struct vfio_fsl_mc_device *vdev = device_data; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; struct fsl_mc_device *mc_dev = vdev->mc_dev; @@ -404,10 +409,12 @@ static int vfio_fsl_mc_send_command(void __iomem *ioaddr, uint64_t *cmd_data) return 0; } -static ssize_t vfio_fsl_mc_write(void *device_data, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t vfio_fsl_mc_write(struct vfio_device *core_vdev, + const char __user *buf, size_t count, + loff_t *ppos) { - struct vfio_fsl_mc_device *vdev = device_data; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; struct fsl_mc_device *mc_dev = vdev->mc_dev; @@ -468,9 +475,11 @@ static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region, size, vma->vm_page_prot); } -static int vfio_fsl_mc_mmap(void *device_data, struct vm_area_struct *vma) +static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev, + struct vm_area_struct *vma) { - struct vfio_fsl_mc_device *vdev = device_data; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); struct fsl_mc_device *mc_dev = vdev->mc_dev; unsigned int index; @@ -568,23 +577,39 @@ static int vfio_fsl_mc_init_device(struct vfio_fsl_mc_device *vdev) dev_err(&mc_dev->dev, "VFIO_FSL_MC: Failed to setup DPRC (%d)\n", ret); goto out_nc_unreg; } + return 0; + +out_nc_unreg: + bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); + return ret; +} +static int vfio_fsl_mc_scan_container(struct fsl_mc_device *mc_dev) +{ + int ret; + + /* non dprc devices do not scan for other devices */ + if (!is_fsl_mc_bus_dprc(mc_dev)) + return 0; ret = dprc_scan_container(mc_dev, false); if (ret) { - dev_err(&mc_dev->dev, "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); - goto out_dprc_cleanup; + dev_err(&mc_dev->dev, + "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); + dprc_remove_devices(mc_dev, NULL, 0); + return ret; } - return 0; +} + +static void vfio_fsl_uninit_device(struct vfio_fsl_mc_device *vdev) +{ + struct fsl_mc_device *mc_dev = vdev->mc_dev; + + if (!is_fsl_mc_bus_dprc(mc_dev)) + return; -out_dprc_cleanup: - dprc_remove_devices(mc_dev, NULL, 0); dprc_cleanup(mc_dev); -out_nc_unreg: bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); - vdev->nb.notifier_call = NULL; - - return ret; } static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) @@ -600,36 +625,50 @@ static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) return -EINVAL; } - vdev = devm_kzalloc(dev, sizeof(*vdev), GFP_KERNEL); + vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); if (!vdev) { ret = -ENOMEM; goto out_group_put; } + vfio_init_group_dev(&vdev->vdev, dev, &vfio_fsl_mc_ops); vdev->mc_dev = mc_dev; - - ret = vfio_add_group_dev(dev, &vfio_fsl_mc_ops, vdev); - if (ret) { - dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); - goto out_group_put; - } + mutex_init(&vdev->igate); ret = vfio_fsl_mc_reflck_attach(vdev); if (ret) - goto out_group_dev; + goto out_kfree; ret = vfio_fsl_mc_init_device(vdev); if (ret) goto out_reflck; - mutex_init(&vdev->igate); + ret = vfio_register_group_dev(&vdev->vdev); + if (ret) { + dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); + goto out_device; + } + /* + * This triggers recursion into vfio_fsl_mc_probe() on another device + * and the vfio_fsl_mc_reflck_attach() must succeed, which relies on the + * vfio_add_group_dev() above. It has no impact on this vdev, so it is + * safe to be after the vfio device is made live. + */ + ret = vfio_fsl_mc_scan_container(mc_dev); + if (ret) + goto out_group_dev; + dev_set_drvdata(dev, vdev); return 0; +out_group_dev: + vfio_unregister_group_dev(&vdev->vdev); +out_device: + vfio_fsl_uninit_device(vdev); out_reflck: vfio_fsl_mc_reflck_put(vdev->reflck); -out_group_dev: - vfio_del_group_dev(dev); +out_kfree: + kfree(vdev); out_group_put: vfio_iommu_group_put(group, dev); return ret; @@ -637,25 +676,17 @@ out_group_put: static int vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) { - struct vfio_fsl_mc_device *vdev; struct device *dev = &mc_dev->dev; + struct vfio_fsl_mc_device *vdev = dev_get_drvdata(dev); - vdev = vfio_del_group_dev(dev); - if (!vdev) - return -EINVAL; - + vfio_unregister_group_dev(&vdev->vdev); mutex_destroy(&vdev->igate); + dprc_remove_devices(mc_dev, NULL, 0); + vfio_fsl_uninit_device(vdev); vfio_fsl_mc_reflck_put(vdev->reflck); - if (is_fsl_mc_bus_dprc(mc_dev)) { - dprc_remove_devices(mc_dev, NULL, 0); - dprc_cleanup(mc_dev); - } - - if (vdev->nb.notifier_call) - bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); - + kfree(vdev); vfio_iommu_group_put(mc_dev->dev.iommu_group, dev); return 0; diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h b/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h index a97ee691ed47..89700e00e77d 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc_private.h @@ -36,6 +36,7 @@ struct vfio_fsl_mc_region { }; struct vfio_fsl_mc_device { + struct vfio_device vdev; struct fsl_mc_device *mc_dev; struct notifier_block nb; int refcnt; diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h index 7d922950caaf..fdbb3bee99a9 100644 --- a/drivers/vfio/mdev/mdev_private.h +++ b/drivers/vfio/mdev/mdev_private.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Mediated device interal definitions + * Mediated device internal definitions * * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. * Author: Neo Jia <cjia@nvidia.com> @@ -35,7 +35,10 @@ struct mdev_device { bool active; }; -#define to_mdev_device(dev) container_of(dev, struct mdev_device, dev) +static inline struct mdev_device *to_mdev_device(struct device *dev) +{ + return container_of(dev, struct mdev_device, dev); +} #define dev_is_mdev(d) ((d)->bus == &mdev_bus_type) struct mdev_type { diff --git a/drivers/vfio/mdev/vfio_mdev.c b/drivers/vfio/mdev/vfio_mdev.c index b52eea128549..ae7e322fbe3c 100644 --- a/drivers/vfio/mdev/vfio_mdev.c +++ b/drivers/vfio/mdev/vfio_mdev.c @@ -21,10 +21,11 @@ #define DRIVER_AUTHOR "NVIDIA Corporation" #define DRIVER_DESC "VFIO based driver for Mediated device" -static int vfio_mdev_open(void *device_data) +static int vfio_mdev_open(struct vfio_device *core_vdev) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; + int ret; if (unlikely(!parent->ops->open)) @@ -40,9 +41,9 @@ static int vfio_mdev_open(void *device_data) return ret; } -static void vfio_mdev_release(void *device_data) +static void vfio_mdev_release(struct vfio_device *core_vdev) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; if (likely(parent->ops->release)) @@ -51,10 +52,10 @@ static void vfio_mdev_release(void *device_data) module_put(THIS_MODULE); } -static long vfio_mdev_unlocked_ioctl(void *device_data, +static long vfio_mdev_unlocked_ioctl(struct vfio_device *core_vdev, unsigned int cmd, unsigned long arg) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; if (unlikely(!parent->ops->ioctl)) @@ -63,10 +64,10 @@ static long vfio_mdev_unlocked_ioctl(void *device_data, return parent->ops->ioctl(mdev, cmd, arg); } -static ssize_t vfio_mdev_read(void *device_data, char __user *buf, +static ssize_t vfio_mdev_read(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; if (unlikely(!parent->ops->read)) @@ -75,10 +76,11 @@ static ssize_t vfio_mdev_read(void *device_data, char __user *buf, return parent->ops->read(mdev, buf, count, ppos); } -static ssize_t vfio_mdev_write(void *device_data, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t vfio_mdev_write(struct vfio_device *core_vdev, + const char __user *buf, size_t count, + loff_t *ppos) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; if (unlikely(!parent->ops->write)) @@ -87,9 +89,10 @@ static ssize_t vfio_mdev_write(void *device_data, const char __user *buf, return parent->ops->write(mdev, buf, count, ppos); } -static int vfio_mdev_mmap(void *device_data, struct vm_area_struct *vma) +static int vfio_mdev_mmap(struct vfio_device *core_vdev, + struct vm_area_struct *vma) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; if (unlikely(!parent->ops->mmap)) @@ -98,9 +101,9 @@ static int vfio_mdev_mmap(void *device_data, struct vm_area_struct *vma) return parent->ops->mmap(mdev, vma); } -static void vfio_mdev_request(void *device_data, unsigned int count) +static void vfio_mdev_request(struct vfio_device *core_vdev, unsigned int count) { - struct mdev_device *mdev = device_data; + struct mdev_device *mdev = to_mdev_device(core_vdev->dev); struct mdev_parent *parent = mdev->parent; if (parent->ops->request) @@ -124,13 +127,29 @@ static const struct vfio_device_ops vfio_mdev_dev_ops = { static int vfio_mdev_probe(struct device *dev) { struct mdev_device *mdev = to_mdev_device(dev); + struct vfio_device *vdev; + int ret; - return vfio_add_group_dev(dev, &vfio_mdev_dev_ops, mdev); + vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); + if (!vdev) + return -ENOMEM; + + vfio_init_group_dev(vdev, &mdev->dev, &vfio_mdev_dev_ops); + ret = vfio_register_group_dev(vdev); + if (ret) { + kfree(vdev); + return ret; + } + dev_set_drvdata(&mdev->dev, vdev); + return 0; } static void vfio_mdev_remove(struct device *dev) { - vfio_del_group_dev(dev); + struct vfio_device *vdev = dev_get_drvdata(dev); + + vfio_unregister_group_dev(vdev); + kfree(vdev); } static struct mdev_driver vfio_mdev_driver = { diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index d691006b6428..882e890dca54 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -378,7 +378,6 @@ static int vfio_pci_enable(struct vfio_pci_device *vdev) if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) vdev->has_vga = true; - if (vfio_pci_is_vga(pdev) && pdev->vendor == PCI_VENDOR_ID_INTEL && IS_ENABLED(CONFIG_VFIO_PCI_IGD)) { @@ -499,30 +498,29 @@ out: static struct pci_driver vfio_pci_driver; -static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev, - struct vfio_device **pf_dev) +static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev) { struct pci_dev *physfn = pci_physfn(vdev->pdev); + struct vfio_device *pf_dev; if (!vdev->pdev->is_virtfn) return NULL; - *pf_dev = vfio_device_get_from_dev(&physfn->dev); - if (!*pf_dev) + pf_dev = vfio_device_get_from_dev(&physfn->dev); + if (!pf_dev) return NULL; if (pci_dev_driver(physfn) != &vfio_pci_driver) { - vfio_device_put(*pf_dev); + vfio_device_put(pf_dev); return NULL; } - return vfio_device_data(*pf_dev); + return container_of(pf_dev, struct vfio_pci_device, vdev); } static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val) { - struct vfio_device *pf_dev; - struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev); + struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev); if (!pf_vdev) return; @@ -532,12 +530,13 @@ static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val) WARN_ON(pf_vdev->vf_token->users < 0); mutex_unlock(&pf_vdev->vf_token->lock); - vfio_device_put(pf_dev); + vfio_device_put(&pf_vdev->vdev); } -static void vfio_pci_release(void *device_data) +static void vfio_pci_release(struct vfio_device *core_vdev) { - struct vfio_pci_device *vdev = device_data; + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); mutex_lock(&vdev->reflck->lock); @@ -563,9 +562,10 @@ static void vfio_pci_release(void *device_data) module_put(THIS_MODULE); } -static int vfio_pci_open(void *device_data) +static int vfio_pci_open(struct vfio_device *core_vdev) { - struct vfio_pci_device *vdev = device_data; + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); int ret = 0; if (!try_module_get(THIS_MODULE)) @@ -774,15 +774,16 @@ int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, } struct vfio_devices { - struct vfio_device **devices; + struct vfio_pci_device **devices; int cur_index; int max_index; }; -static long vfio_pci_ioctl(void *device_data, +static long vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd, unsigned long arg) { - struct vfio_pci_device *vdev = device_data; + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); unsigned long minsz; if (cmd == VFIO_DEVICE_GET_INFO) { @@ -1262,9 +1263,7 @@ reset_info_exit: goto hot_reset_release; for (; mem_idx < devs.cur_index; mem_idx++) { - struct vfio_pci_device *tmp; - - tmp = vfio_device_data(devs.devices[mem_idx]); + struct vfio_pci_device *tmp = devs.devices[mem_idx]; ret = down_write_trylock(&tmp->memory_lock); if (!ret) { @@ -1279,17 +1278,13 @@ reset_info_exit: hot_reset_release: for (i = 0; i < devs.cur_index; i++) { - struct vfio_device *device; - struct vfio_pci_device *tmp; - - device = devs.devices[i]; - tmp = vfio_device_data(device); + struct vfio_pci_device *tmp = devs.devices[i]; if (i < mem_idx) up_write(&tmp->memory_lock); else mutex_unlock(&tmp->vma_lock); - vfio_device_put(device); + vfio_device_put(&tmp->vdev); } kfree(devs.devices); @@ -1384,11 +1379,10 @@ hot_reset_release: return -ENOTTY; } -static ssize_t vfio_pci_rw(void *device_data, char __user *buf, +static ssize_t vfio_pci_rw(struct vfio_pci_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) { unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); - struct vfio_pci_device *vdev = device_data; if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) return -EINVAL; @@ -1416,22 +1410,28 @@ static ssize_t vfio_pci_rw(void *device_data, char __user *buf, return -EINVAL; } -static ssize_t vfio_pci_read(void *device_data, char __user *buf, +static ssize_t vfio_pci_read(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos) { + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); + if (!count) return 0; - return vfio_pci_rw(device_data, buf, count, ppos, false); + return vfio_pci_rw(vdev, buf, count, ppos, false); } -static ssize_t vfio_pci_write(void *device_data, const char __user *buf, +static ssize_t vfio_pci_write(struct vfio_device *core_vdev, const char __user *buf, size_t count, loff_t *ppos) { + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); + if (!count) return 0; - return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); + return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true); } /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ @@ -1628,9 +1628,10 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = { .fault = vfio_pci_mmap_fault, }; -static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) +static int vfio_pci_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) { - struct vfio_pci_device *vdev = device_data; + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); struct pci_dev *pdev = vdev->pdev; unsigned int index; u64 phys_len, req_len, pgoff, req_start; @@ -1696,9 +1697,10 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) return 0; } -static void vfio_pci_request(void *device_data, unsigned int count) +static void vfio_pci_request(struct vfio_device *core_vdev, unsigned int count) { - struct vfio_pci_device *vdev = device_data; + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); struct pci_dev *pdev = vdev->pdev; mutex_lock(&vdev->igate); @@ -1749,8 +1751,7 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev, return 0; /* No VF token provided or required */ if (vdev->pdev->is_virtfn) { - struct vfio_device *pf_dev; - struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev); + struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev); bool match; if (!pf_vdev) { @@ -1763,7 +1764,7 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev, } if (!vf_token) { - vfio_device_put(pf_dev); + vfio_device_put(&pf_vdev->vdev); pci_info_ratelimited(vdev->pdev, "VF token required to access device\n"); return -EACCES; @@ -1773,7 +1774,7 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev, match = uuid_equal(uuid, &pf_vdev->vf_token->uuid); mutex_unlock(&pf_vdev->vf_token->lock); - vfio_device_put(pf_dev); + vfio_device_put(&pf_vdev->vdev); if (!match) { pci_info_ratelimited(vdev->pdev, @@ -1812,9 +1813,10 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev, #define VF_TOKEN_ARG "vf_token=" -static int vfio_pci_match(void *device_data, char *buf) +static int vfio_pci_match(struct vfio_device *core_vdev, char *buf) { - struct vfio_pci_device *vdev = device_data; + struct vfio_pci_device *vdev = + container_of(core_vdev, struct vfio_pci_device, vdev); bool vf_token = false; uuid_t uuid; int ret; @@ -1904,6 +1906,68 @@ static int vfio_pci_bus_notifier(struct notifier_block *nb, return 0; } +static int vfio_pci_vf_init(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + int ret; + + if (!pdev->is_physfn) + return 0; + + vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); + if (!vdev->vf_token) + return -ENOMEM; + + mutex_init(&vdev->vf_token->lock); + uuid_gen(&vdev->vf_token->uuid); + + vdev->nb.notifier_call = vfio_pci_bus_notifier; + ret = bus_register_notifier(&pci_bus_type, &vdev->nb); + if (ret) { + kfree(vdev->vf_token); + return ret; + } + return 0; +} + +static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev) +{ + if (!vdev->vf_token) + return; + + bus_unregister_notifier(&pci_bus_type, &vdev->nb); + WARN_ON(vdev->vf_token->users); + mutex_destroy(&vdev->vf_token->lock); + kfree(vdev->vf_token); +} + +static int vfio_pci_vga_init(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + int ret; + + if (!vfio_pci_is_vga(pdev)) + return 0; + + ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); + if (ret) + return ret; + vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false)); + return 0; +} + +static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + + if (!vfio_pci_is_vga(pdev)) + return; + vga_client_register(pdev, NULL, NULL, NULL); + vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | + VGA_RSRC_LEGACY_IO | + VGA_RSRC_LEGACY_MEM); +} + static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct vfio_pci_device *vdev; @@ -1939,6 +2003,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto out_group_put; } + vfio_init_group_dev(&vdev->vdev, &pdev->dev, &vfio_pci_ops); vdev->pdev = pdev; vdev->irq_type = VFIO_PCI_NUM_IRQS; mutex_init(&vdev->igate); @@ -1950,35 +2015,15 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) INIT_LIST_HEAD(&vdev->vma_list); init_rwsem(&vdev->memory_lock); - ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + ret = vfio_pci_reflck_attach(vdev); if (ret) goto out_free; - - ret = vfio_pci_reflck_attach(vdev); + ret = vfio_pci_vf_init(vdev); if (ret) - goto out_del_group_dev; - - if (pdev->is_physfn) { - vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); - if (!vdev->vf_token) { - ret = -ENOMEM; - goto out_reflck; - } - - mutex_init(&vdev->vf_token->lock); - uuid_gen(&vdev->vf_token->uuid); - - vdev->nb.notifier_call = vfio_pci_bus_notifier; - ret = bus_register_notifier(&pci_bus_type, &vdev->nb); - if (ret) - goto out_vf_token; - } - - if (vfio_pci_is_vga(pdev)) { - vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); - vga_set_legacy_decoding(pdev, - vfio_pci_set_vga_decode(vdev, false)); - } + goto out_reflck; + ret = vfio_pci_vga_init(vdev); + if (ret) + goto out_vf; vfio_pci_probe_power_state(vdev); @@ -1996,15 +2041,21 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) vfio_pci_set_power_state(vdev, PCI_D3hot); } - return ret; + ret = vfio_register_group_dev(&vdev->vdev); + if (ret) + goto out_power; + dev_set_drvdata(&pdev->dev, vdev); + return 0; -out_vf_token: - kfree(vdev->vf_token); +out_power: + if (!disable_idle_d3) + vfio_pci_set_power_state(vdev, PCI_D0); +out_vf: + vfio_pci_vf_uninit(vdev); out_reflck: vfio_pci_reflck_put(vdev->reflck); -out_del_group_dev: - vfio_del_group_dev(&pdev->dev); out_free: + kfree(vdev->pm_save); kfree(vdev); out_group_put: vfio_iommu_group_put(group, &pdev->dev); @@ -2013,41 +2064,25 @@ out_group_put: static void vfio_pci_remove(struct pci_dev *pdev) { - struct vfio_pci_device *vdev; + struct vfio_pci_device *vdev = dev_get_drvdata(&pdev->dev); pci_disable_sriov(pdev); - vdev = vfio_del_group_dev(&pdev->dev); - if (!vdev) - return; - - if (vdev->vf_token) { - WARN_ON(vdev->vf_token->users); - mutex_destroy(&vdev->vf_token->lock); - kfree(vdev->vf_token); - } - - if (vdev->nb.notifier_call) - bus_unregister_notifier(&pci_bus_type, &vdev->nb); + vfio_unregister_group_dev(&vdev->vdev); + vfio_pci_vf_uninit(vdev); vfio_pci_reflck_put(vdev->reflck); + vfio_pci_vga_uninit(vdev); vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev); - kfree(vdev->region); - mutex_destroy(&vdev->ioeventfds_lock); if (!disable_idle_d3) vfio_pci_set_power_state(vdev, PCI_D0); + mutex_destroy(&vdev->ioeventfds_lock); + kfree(vdev->region); kfree(vdev->pm_save); kfree(vdev); - - if (vfio_pci_is_vga(pdev)) { - vga_client_register(pdev, NULL, NULL, NULL); - vga_set_legacy_decoding(pdev, - VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | - VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM); - } } static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, @@ -2060,11 +2095,7 @@ static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, if (device == NULL) return PCI_ERS_RESULT_DISCONNECT; - vdev = vfio_device_data(device); - if (vdev == NULL) { - vfio_device_put(device); - return PCI_ERS_RESULT_DISCONNECT; - } + vdev = container_of(device, struct vfio_pci_device, vdev); mutex_lock(&vdev->igate); @@ -2080,7 +2111,6 @@ static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn) { - struct vfio_pci_device *vdev; struct vfio_device *device; int ret = 0; @@ -2093,12 +2123,6 @@ static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn) if (!device) return -ENODEV; - vdev = vfio_device_data(device); - if (!vdev) { - vfio_device_put(device); - return -ENODEV; - } - if (nr_virtfn == 0) pci_disable_sriov(pdev); else @@ -2158,7 +2182,7 @@ static int vfio_pci_reflck_find(struct pci_dev *pdev, void *data) return 0; } - vdev = vfio_device_data(device); + vdev = container_of(device, struct vfio_pci_device, vdev); if (vdev->reflck) { vfio_pci_reflck_get(vdev->reflck); @@ -2220,7 +2244,7 @@ static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data) return -EBUSY; } - vdev = vfio_device_data(device); + vdev = container_of(device, struct vfio_pci_device, vdev); /* Fault if the device is not unused */ if (vdev->refcnt) { @@ -2228,7 +2252,7 @@ static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data) return -EBUSY; } - devs->devices[devs->cur_index++] = device; + devs->devices[devs->cur_index++] = vdev; return 0; } @@ -2250,7 +2274,7 @@ static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data) return -EBUSY; } - vdev = vfio_device_data(device); + vdev = container_of(device, struct vfio_pci_device, vdev); /* * Locking multiple devices is prone to deadlock, runaway and @@ -2261,7 +2285,7 @@ static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data) return -EBUSY; } - devs->devices[devs->cur_index++] = device; + devs->devices[devs->cur_index++] = vdev; return 0; } @@ -2309,7 +2333,7 @@ static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev) /* Does at least one need a reset? */ for (i = 0; i < devs.cur_index; i++) { - tmp = vfio_device_data(devs.devices[i]); + tmp = devs.devices[i]; if (tmp->needs_reset) { ret = pci_reset_bus(vdev->pdev); break; @@ -2318,7 +2342,7 @@ static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev) put_devs: for (i = 0; i < devs.cur_index; i++) { - tmp = vfio_device_data(devs.devices[i]); + tmp = devs.devices[i]; /* * If reset was successful, affected devices no longer need @@ -2334,7 +2358,7 @@ put_devs: vfio_pci_set_power_state(tmp, PCI_D3hot); } - vfio_device_put(devs.devices[i]); + vfio_device_put(&tmp->vdev); } kfree(devs.devices); @@ -2391,7 +2415,7 @@ static int __init vfio_pci_init(void) { int ret; - /* Allocate shared config space permision data used by all devices */ + /* Allocate shared config space permission data used by all devices */ ret = vfio_pci_init_perm_bits(); if (ret) return ret; diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index a402adee8a21..d57f037f65b8 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -101,7 +101,7 @@ static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = { /* * Read/Write Permission Bits - one bit for each bit in capability * Any field can be read if it exists, but what is read depends on - * whether the field is 'virtualized', or just pass thru to the + * whether the field is 'virtualized', or just pass through to the * hardware. Any virtualized field is also virtualized for writes. * Writes are only permitted if they have a 1 bit here. */ diff --git a/drivers/vfio/pci/vfio_pci_igd.c b/drivers/vfio/pci/vfio_pci_igd.c index e66dfb0178ed..228df565e9bc 100644 --- a/drivers/vfio/pci/vfio_pci_igd.c +++ b/drivers/vfio/pci/vfio_pci_igd.c @@ -21,6 +21,10 @@ #define OPREGION_SIZE (8 * 1024) #define OPREGION_PCI_ADDR 0xfc +#define OPREGION_RVDA 0x3ba +#define OPREGION_RVDS 0x3c2 +#define OPREGION_VERSION 0x16 + static size_t vfio_pci_igd_rw(struct vfio_pci_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) { @@ -58,6 +62,7 @@ static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev) u32 addr, size; void *base; int ret; + u16 version; ret = pci_read_config_dword(vdev->pdev, OPREGION_PCI_ADDR, &addr); if (ret) @@ -83,6 +88,54 @@ static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev) size *= 1024; /* In KB */ + /* + * Support opregion v2.1+ + * When VBT data exceeds 6KB size and cannot be within mailbox #4, then + * the Extended VBT region next to opregion is used to hold the VBT data. + * RVDA (Relative Address of VBT Data from Opregion Base) and RVDS + * (Raw VBT Data Size) from opregion structure member are used to hold the + * address from region base and size of VBT data. RVDA/RVDS are not + * defined before opregion 2.0. + * + * opregion 2.1+: RVDA is unsigned, relative offset from + * opregion base, and should point to the end of opregion. + * otherwise, exposing to userspace to allow read access to everything between + * the OpRegion and VBT is not safe. + * RVDS is defined as size in bytes. + * + * opregion 2.0: rvda is the physical VBT address. + * Since rvda is HPA it cannot be directly used in guest. + * And it should not be practically available for end user,so it is not supported. + */ + version = le16_to_cpu(*(__le16 *)(base + OPREGION_VERSION)); + if (version >= 0x0200) { + u64 rvda; + u32 rvds; + + rvda = le64_to_cpu(*(__le64 *)(base + OPREGION_RVDA)); + rvds = le32_to_cpu(*(__le32 *)(base + OPREGION_RVDS)); + if (rvda && rvds) { + /* no support for opregion v2.0 with physical VBT address */ + if (version == 0x0200) { + memunmap(base); + pci_err(vdev->pdev, + "IGD assignment does not support opregion v2.0 with an extended VBT region\n"); + return -EINVAL; + } + + if (rvda != size) { + memunmap(base); + pci_err(vdev->pdev, + "Extended VBT does not follow opregion on version 0x%04x\n", + version); + return -EINVAL; + } + + /* region size for opregion v2.0+: opregion and VBT size. */ + size += rvds; + } + } + if (size != OPREGION_SIZE) { memunmap(base); base = memremap(addr, size, MEMREMAP_WB); diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index cdae2e4cf11c..5a36272cecbf 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -100,6 +100,7 @@ struct vfio_pci_mmap_vma { }; struct vfio_pci_device { + struct vfio_device vdev; struct pci_dev *pdev; void __iomem *barmap[PCI_STD_NUM_BARS]; bool bar_mmap_supported[PCI_STD_NUM_BARS]; diff --git a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c index 09a9453b75c5..63cc7f0b2e4a 100644 --- a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c +++ b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c @@ -26,7 +26,7 @@ #define XGMAC_DMA_CONTROL 0x00000f18 /* Ctrl (Operational Mode) */ #define XGMAC_DMA_INTR_ENA 0x00000f1c /* Interrupt Enable */ -/* DMA Control registe defines */ +/* DMA Control register defines */ #define DMA_CONTROL_ST 0x00002000 /* Start/Stop Transmission */ #define DMA_CONTROL_SR 0x00000002 /* Start/Stop Receive */ diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c index 3626c2150101..f970eb2a999f 100644 --- a/drivers/vfio/platform/vfio_amba.c +++ b/drivers/vfio/platform/vfio_amba.c @@ -66,16 +66,18 @@ static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id) if (ret) { kfree(vdev->name); kfree(vdev); + return ret; } - return ret; + dev_set_drvdata(&adev->dev, vdev); + return 0; } static void vfio_amba_remove(struct amba_device *adev) { - struct vfio_platform_device *vdev = - vfio_platform_remove_common(&adev->dev); + struct vfio_platform_device *vdev = dev_get_drvdata(&adev->dev); + vfio_platform_remove_common(vdev); kfree(vdev->name); kfree(vdev); } diff --git a/drivers/vfio/platform/vfio_platform.c b/drivers/vfio/platform/vfio_platform.c index 9fb6818cea12..e4027799a154 100644 --- a/drivers/vfio/platform/vfio_platform.c +++ b/drivers/vfio/platform/vfio_platform.c @@ -54,23 +54,21 @@ static int vfio_platform_probe(struct platform_device *pdev) vdev->reset_required = reset_required; ret = vfio_platform_probe_common(vdev, &pdev->dev); - if (ret) + if (ret) { kfree(vdev); - - return ret; + return ret; + } + dev_set_drvdata(&pdev->dev, vdev); + return 0; } static int vfio_platform_remove(struct platform_device *pdev) { - struct vfio_platform_device *vdev; - - vdev = vfio_platform_remove_common(&pdev->dev); - if (vdev) { - kfree(vdev); - return 0; - } + struct vfio_platform_device *vdev = dev_get_drvdata(&pdev->dev); - return -EINVAL; + vfio_platform_remove_common(vdev); + kfree(vdev); + return 0; } static struct platform_driver vfio_platform_driver = { diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c index fb4b385191f2..361e5b57e369 100644 --- a/drivers/vfio/platform/vfio_platform_common.c +++ b/drivers/vfio/platform/vfio_platform_common.c @@ -218,9 +218,10 @@ static int vfio_platform_call_reset(struct vfio_platform_device *vdev, return -EINVAL; } -static void vfio_platform_release(void *device_data) +static void vfio_platform_release(struct vfio_device *core_vdev) { - struct vfio_platform_device *vdev = device_data; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); mutex_lock(&driver_lock); @@ -244,9 +245,10 @@ static void vfio_platform_release(void *device_data) module_put(vdev->parent_module); } -static int vfio_platform_open(void *device_data) +static int vfio_platform_open(struct vfio_device *core_vdev) { - struct vfio_platform_device *vdev = device_data; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); int ret; if (!try_module_get(vdev->parent_module)) @@ -293,10 +295,12 @@ err_reg: return ret; } -static long vfio_platform_ioctl(void *device_data, +static long vfio_platform_ioctl(struct vfio_device *core_vdev, unsigned int cmd, unsigned long arg) { - struct vfio_platform_device *vdev = device_data; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); + unsigned long minsz; if (cmd == VFIO_DEVICE_GET_INFO) { @@ -455,10 +459,11 @@ err: return -EFAULT; } -static ssize_t vfio_platform_read(void *device_data, char __user *buf, - size_t count, loff_t *ppos) +static ssize_t vfio_platform_read(struct vfio_device *core_vdev, + char __user *buf, size_t count, loff_t *ppos) { - struct vfio_platform_device *vdev = device_data; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); unsigned int index = VFIO_PLATFORM_OFFSET_TO_INDEX(*ppos); loff_t off = *ppos & VFIO_PLATFORM_OFFSET_MASK; @@ -531,10 +536,11 @@ err: return -EFAULT; } -static ssize_t vfio_platform_write(void *device_data, const char __user *buf, +static ssize_t vfio_platform_write(struct vfio_device *core_vdev, const char __user *buf, size_t count, loff_t *ppos) { - struct vfio_platform_device *vdev = device_data; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); unsigned int index = VFIO_PLATFORM_OFFSET_TO_INDEX(*ppos); loff_t off = *ppos & VFIO_PLATFORM_OFFSET_MASK; @@ -573,9 +579,10 @@ static int vfio_platform_mmap_mmio(struct vfio_platform_region region, req_len, vma->vm_page_prot); } -static int vfio_platform_mmap(void *device_data, struct vm_area_struct *vma) +static int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) { - struct vfio_platform_device *vdev = device_data; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); unsigned int index; index = vma->vm_pgoff >> (VFIO_PLATFORM_OFFSET_SHIFT - PAGE_SHIFT); @@ -659,8 +666,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, struct iommu_group *group; int ret; - if (!vdev) - return -EINVAL; + vfio_init_group_dev(&vdev->vdev, dev, &vfio_platform_ops); ret = vfio_platform_acpi_probe(vdev, dev); if (ret) @@ -685,13 +691,13 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev, goto put_reset; } - ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev); + ret = vfio_register_group_dev(&vdev->vdev); if (ret) goto put_iommu; mutex_init(&vdev->igate); - pm_runtime_enable(vdev->device); + pm_runtime_enable(dev); return 0; put_iommu: @@ -702,19 +708,13 @@ put_reset: } EXPORT_SYMBOL_GPL(vfio_platform_probe_common); -struct vfio_platform_device *vfio_platform_remove_common(struct device *dev) +void vfio_platform_remove_common(struct vfio_platform_device *vdev) { - struct vfio_platform_device *vdev; - - vdev = vfio_del_group_dev(dev); + vfio_unregister_group_dev(&vdev->vdev); - if (vdev) { - pm_runtime_disable(vdev->device); - vfio_platform_put_reset(vdev); - vfio_iommu_group_put(dev->iommu_group, dev); - } - - return vdev; + pm_runtime_disable(vdev->device); + vfio_platform_put_reset(vdev); + vfio_iommu_group_put(vdev->vdev.dev->iommu_group, vdev->vdev.dev); } EXPORT_SYMBOL_GPL(vfio_platform_remove_common); diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h index 289089910643..a5ba82c8cbc3 100644 --- a/drivers/vfio/platform/vfio_platform_private.h +++ b/drivers/vfio/platform/vfio_platform_private.h @@ -9,6 +9,7 @@ #include <linux/types.h> #include <linux/interrupt.h> +#include <linux/vfio.h> #define VFIO_PLATFORM_OFFSET_SHIFT 40 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1) @@ -42,6 +43,7 @@ struct vfio_platform_region { }; struct vfio_platform_device { + struct vfio_device vdev; struct vfio_platform_region *regions; u32 num_regions; struct vfio_platform_irq *irqs; @@ -80,8 +82,7 @@ struct vfio_platform_reset_node { extern int vfio_platform_probe_common(struct vfio_platform_device *vdev, struct device *dev); -extern struct vfio_platform_device *vfio_platform_remove_common - (struct device *dev); +void vfio_platform_remove_common(struct vfio_platform_device *vdev); extern int vfio_platform_irq_init(struct vfio_platform_device *vdev); extern void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev); diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 38779e6fd80c..5e631c359ef2 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -46,7 +46,6 @@ static struct vfio { struct mutex group_lock; struct cdev group_cdev; dev_t group_devt; - wait_queue_head_t release_q; } vfio; struct vfio_iommu_driver { @@ -90,15 +89,6 @@ struct vfio_group { struct blocking_notifier_head notifier; }; -struct vfio_device { - struct kref kref; - struct device *dev; - const struct vfio_device_ops *ops; - struct vfio_group *group; - struct list_head group_next; - void *device_data; -}; - #ifdef CONFIG_VFIO_NOIOMMU static bool noiommu __read_mostly; module_param_named(enable_unsafe_noiommu_mode, @@ -109,8 +99,8 @@ MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. Thi /* * vfio_iommu_group_{get,put} are only intended for VFIO bus driver probe * and remove functions, any use cases other than acquiring the first - * reference for the purpose of calling vfio_add_group_dev() or removing - * that symmetric reference after vfio_del_group_dev() should use the raw + * reference for the purpose of calling vfio_register_group_dev() or removing + * that symmetric reference after vfio_unregister_group_dev() should use the raw * iommu_group_{get,put} functions. In particular, vfio_iommu_group_put() * removes the device from the dummy group and cannot be nested. */ @@ -532,67 +522,17 @@ static struct vfio_group *vfio_group_get_from_dev(struct device *dev) /** * Device objects - create, release, get, put, search */ -static -struct vfio_device *vfio_group_create_device(struct vfio_group *group, - struct device *dev, - const struct vfio_device_ops *ops, - void *device_data) -{ - struct vfio_device *device; - - device = kzalloc(sizeof(*device), GFP_KERNEL); - if (!device) - return ERR_PTR(-ENOMEM); - - kref_init(&device->kref); - device->dev = dev; - device->group = group; - device->ops = ops; - device->device_data = device_data; - dev_set_drvdata(dev, device); - - /* No need to get group_lock, caller has group reference */ - vfio_group_get(group); - - mutex_lock(&group->device_lock); - list_add(&device->group_next, &group->device_list); - group->dev_counter++; - mutex_unlock(&group->device_lock); - - return device; -} - -static void vfio_device_release(struct kref *kref) -{ - struct vfio_device *device = container_of(kref, - struct vfio_device, kref); - struct vfio_group *group = device->group; - - list_del(&device->group_next); - group->dev_counter--; - mutex_unlock(&group->device_lock); - - dev_set_drvdata(device->dev, NULL); - - kfree(device); - - /* vfio_del_group_dev may be waiting for this device */ - wake_up(&vfio.release_q); -} - /* Device reference always implies a group reference */ void vfio_device_put(struct vfio_device *device) { - struct vfio_group *group = device->group; - kref_put_mutex(&device->kref, vfio_device_release, &group->device_lock); - vfio_group_put(group); + if (refcount_dec_and_test(&device->refcount)) + complete(&device->comp); } EXPORT_SYMBOL_GPL(vfio_device_put); -static void vfio_device_get(struct vfio_device *device) +static bool vfio_device_try_get(struct vfio_device *device) { - vfio_group_get(device->group); - kref_get(&device->kref); + return refcount_inc_not_zero(&device->refcount); } static struct vfio_device *vfio_group_get_device(struct vfio_group *group, @@ -602,8 +542,7 @@ static struct vfio_device *vfio_group_get_device(struct vfio_group *group, mutex_lock(&group->device_lock); list_for_each_entry(device, &group->device_list, group_next) { - if (device->dev == dev) { - vfio_device_get(device); + if (device->dev == dev && vfio_device_try_get(device)) { mutex_unlock(&group->device_lock); return device; } @@ -801,14 +740,22 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb, /** * VFIO driver API */ -int vfio_add_group_dev(struct device *dev, - const struct vfio_device_ops *ops, void *device_data) +void vfio_init_group_dev(struct vfio_device *device, struct device *dev, + const struct vfio_device_ops *ops) +{ + init_completion(&device->comp); + device->dev = dev; + device->ops = ops; +} +EXPORT_SYMBOL_GPL(vfio_init_group_dev); + +int vfio_register_group_dev(struct vfio_device *device) { + struct vfio_device *existing_device; struct iommu_group *iommu_group; struct vfio_group *group; - struct vfio_device *device; - iommu_group = iommu_group_get(dev); + iommu_group = iommu_group_get(device->dev); if (!iommu_group) return -EINVAL; @@ -827,31 +774,29 @@ int vfio_add_group_dev(struct device *dev, iommu_group_put(iommu_group); } - device = vfio_group_get_device(group, dev); - if (device) { - dev_WARN(dev, "Device already exists on group %d\n", + existing_device = vfio_group_get_device(group, device->dev); + if (existing_device) { + dev_WARN(device->dev, "Device already exists on group %d\n", iommu_group_id(iommu_group)); - vfio_device_put(device); + vfio_device_put(existing_device); vfio_group_put(group); return -EBUSY; } - device = vfio_group_create_device(group, dev, ops, device_data); - if (IS_ERR(device)) { - vfio_group_put(group); - return PTR_ERR(device); - } + /* Our reference on group is moved to the device */ + device->group = group; - /* - * Drop all but the vfio_device reference. The vfio_device holds - * a reference to the vfio_group, which holds a reference to the - * iommu_group. - */ - vfio_group_put(group); + /* Refcounting can't start until the driver calls register */ + refcount_set(&device->refcount, 1); + + mutex_lock(&group->device_lock); + list_add(&device->group_next, &group->device_list); + group->dev_counter++; + mutex_unlock(&group->device_lock); return 0; } -EXPORT_SYMBOL_GPL(vfio_add_group_dev); +EXPORT_SYMBOL_GPL(vfio_register_group_dev); /** * Get a reference to the vfio_device for a device. Even if the @@ -886,7 +831,7 @@ static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group, int ret; if (it->ops->match) { - ret = it->ops->match(it->device_data, buf); + ret = it->ops->match(it, buf); if (ret < 0) { device = ERR_PTR(ret); break; @@ -895,9 +840,8 @@ static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group, ret = !strcmp(dev_name(it->dev), buf); } - if (ret) { + if (ret && vfio_device_try_get(it)) { device = it; - vfio_device_get(device); break; } } @@ -907,32 +851,15 @@ static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group, } /* - * Caller must hold a reference to the vfio_device - */ -void *vfio_device_data(struct vfio_device *device) -{ - return device->device_data; -} -EXPORT_SYMBOL_GPL(vfio_device_data); - -/* * Decrement the device reference count and wait for the device to be * removed. Open file descriptors for the device... */ -void *vfio_del_group_dev(struct device *dev) +void vfio_unregister_group_dev(struct vfio_device *device) { - DEFINE_WAIT_FUNC(wait, woken_wake_function); - struct vfio_device *device = dev_get_drvdata(dev); struct vfio_group *group = device->group; - void *device_data = device->device_data; struct vfio_unbound_dev *unbound; unsigned int i = 0; bool interrupted = false; - - /* - * The group exists so long as we have a device reference. Get - * a group reference and use it to scan for the device going away. - */ - vfio_group_get(group); + long rc; /* * When the device is removed from the group, the group suddenly @@ -945,7 +872,7 @@ void *vfio_del_group_dev(struct device *dev) */ unbound = kzalloc(sizeof(*unbound), GFP_KERNEL); if (unbound) { - unbound->dev = dev; + unbound->dev = device->dev; mutex_lock(&group->unbound_lock); list_add(&unbound->unbound_next, &group->unbound_list); mutex_unlock(&group->unbound_lock); @@ -953,44 +880,33 @@ void *vfio_del_group_dev(struct device *dev) WARN_ON(!unbound); vfio_device_put(device); - - /* - * If the device is still present in the group after the above - * 'put', then it is in use and we need to request it from the - * bus driver. The driver may in turn need to request the - * device from the user. We send the request on an arbitrary - * interval with counter to allow the driver to take escalating - * measures to release the device if it has the ability to do so. - */ - add_wait_queue(&vfio.release_q, &wait); - - do { - device = vfio_group_get_device(group, dev); - if (!device) - break; - + rc = try_wait_for_completion(&device->comp); + while (rc <= 0) { if (device->ops->request) - device->ops->request(device_data, i++); - - vfio_device_put(device); + device->ops->request(device, i++); if (interrupted) { - wait_woken(&wait, TASK_UNINTERRUPTIBLE, HZ * 10); + rc = wait_for_completion_timeout(&device->comp, + HZ * 10); } else { - wait_woken(&wait, TASK_INTERRUPTIBLE, HZ * 10); - if (signal_pending(current)) { + rc = wait_for_completion_interruptible_timeout( + &device->comp, HZ * 10); + if (rc < 0) { interrupted = true; - dev_warn(dev, + dev_warn(device->dev, "Device is currently in use, task" " \"%s\" (%d) " "blocked until device is released", current->comm, task_pid_nr(current)); } } + } - } while (1); + mutex_lock(&group->device_lock); + list_del(&device->group_next); + group->dev_counter--; + mutex_unlock(&group->device_lock); - remove_wait_queue(&vfio.release_q, &wait); /* * In order to support multiple devices per group, devices can be * plucked from the group while other devices in the group are still @@ -1008,11 +924,10 @@ void *vfio_del_group_dev(struct device *dev) if (list_empty(&group->device_list)) wait_event(group->container_q, !group->container); + /* Matches the get in vfio_register_group_dev() */ vfio_group_put(group); - - return device_data; } -EXPORT_SYMBOL_GPL(vfio_del_group_dev); +EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); /** * VFIO base fd, /dev/vfio/vfio @@ -1454,7 +1369,7 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf) if (IS_ERR(device)) return PTR_ERR(device); - ret = device->ops->open(device->device_data); + ret = device->ops->open(device); if (ret) { vfio_device_put(device); return ret; @@ -1466,7 +1381,7 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf) */ ret = get_unused_fd_flags(O_CLOEXEC); if (ret < 0) { - device->ops->release(device->device_data); + device->ops->release(device); vfio_device_put(device); return ret; } @@ -1476,7 +1391,7 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf) if (IS_ERR(filep)) { put_unused_fd(ret); ret = PTR_ERR(filep); - device->ops->release(device->device_data); + device->ops->release(device); vfio_device_put(device); return ret; } @@ -1633,7 +1548,7 @@ static int vfio_device_fops_release(struct inode *inode, struct file *filep) { struct vfio_device *device = filep->private_data; - device->ops->release(device->device_data); + device->ops->release(device); vfio_group_try_dissolve_container(device->group); @@ -1650,7 +1565,7 @@ static long vfio_device_fops_unl_ioctl(struct file *filep, if (unlikely(!device->ops->ioctl)) return -EINVAL; - return device->ops->ioctl(device->device_data, cmd, arg); + return device->ops->ioctl(device, cmd, arg); } static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, @@ -1661,7 +1576,7 @@ static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, if (unlikely(!device->ops->read)) return -EINVAL; - return device->ops->read(device->device_data, buf, count, ppos); + return device->ops->read(device, buf, count, ppos); } static ssize_t vfio_device_fops_write(struct file *filep, @@ -1673,7 +1588,7 @@ static ssize_t vfio_device_fops_write(struct file *filep, if (unlikely(!device->ops->write)) return -EINVAL; - return device->ops->write(device->device_data, buf, count, ppos); + return device->ops->write(device, buf, count, ppos); } static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) @@ -1683,7 +1598,7 @@ static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) if (unlikely(!device->ops->mmap)) return -EINVAL; - return device->ops->mmap(device->device_data, vma); + return device->ops->mmap(device, vma); } static const struct file_operations vfio_device_fops = { @@ -2379,7 +2294,6 @@ static int __init vfio_init(void) mutex_init(&vfio.iommu_drivers_lock); INIT_LIST_HEAD(&vfio.group_list); INIT_LIST_HEAD(&vfio.iommu_drivers_list); - init_waitqueue_head(&vfio.release_q); ret = misc_register(&vfio_dev); if (ret) { diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 45cbfd4879a5..5fb1bb52b057 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -16,7 +16,7 @@ * IOMMU to support the IOMMU API and have few to no restrictions around * the IOVA range that can be mapped. The Type1 IOMMU is currently * optimized for relatively static mappings of a userspace process with - * userpsace pages pinned into memory. We also assume devices and IOMMU + * userspace pages pinned into memory. We also assume devices and IOMMU * domains are PCI based as the IOMMU API is still centered around a * device/bus interface rather than a group interface. */ @@ -877,7 +877,7 @@ again: /* * If iommu capable domain exist in the container then all pages are - * already pinned and accounted. Accouting should be done if there is no + * already pinned and accounted. Accounting should be done if there is no * iommu capable domain in the container. */ do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); @@ -960,7 +960,7 @@ static int vfio_iommu_type1_unpin_pages(void *iommu_data, bool do_accounting; int i; - if (!iommu || !user_pfn) + if (!iommu || !user_pfn || npage <= 0) return -EINVAL; /* Supported for v2 version only */ @@ -977,13 +977,13 @@ static int vfio_iommu_type1_unpin_pages(void *iommu_data, iova = user_pfn[i] << PAGE_SHIFT; dma = vfio_find_dma(iommu, iova, PAGE_SIZE); if (!dma) - goto unpin_exit; + break; + vfio_unpin_page_external(dma, iova, do_accounting); } -unpin_exit: mutex_unlock(&iommu->lock); - return i > npage ? npage : (i > 0 ? i : -EINVAL); + return i > 0 ? i : -EINVAL; } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, @@ -2177,7 +2177,7 @@ static int vfio_iommu_resv_exclude(struct list_head *iova, continue; /* * Insert a new node if current node overlaps with the - * reserve region to exlude that from valid iova range. + * reserve region to exclude that from valid iova range. * Note that, new node is inserted before the current * node and finally the current node is deleted keeping * the list updated and sorted. |