2019-06-04 08:11:33 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2012-07-31 14:16:22 +00:00
|
|
|
/*
|
|
|
|
* VFIO core
|
|
|
|
*
|
|
|
|
* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
|
|
|
|
* Author: Alex Williamson <alex.williamson@redhat.com>
|
|
|
|
*
|
|
|
|
* Derived from original vfio:
|
|
|
|
* Copyright 2010 Cisco Systems, Inc. All rights reserved.
|
|
|
|
* Author: Tom Lyon, pugs@cisco.com
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/cdev.h>
|
|
|
|
#include <linux/compat.h>
|
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/anon_inodes.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/idr.h>
|
|
|
|
#include <linux/iommu.h>
|
|
|
|
#include <linux/list.h>
|
2013-12-19 17:17:13 +00:00
|
|
|
#include <linux/miscdevice.h>
|
2012-07-31 14:16:22 +00:00
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/mutex.h>
|
2015-10-27 20:53:04 +00:00
|
|
|
#include <linux/pci.h>
|
2013-04-25 22:12:38 +00:00
|
|
|
#include <linux/rwsem.h>
|
2012-07-31 14:16:22 +00:00
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/slab.h>
|
2013-04-30 21:42:28 +00:00
|
|
|
#include <linux/stat.h>
|
2012-07-31 14:16:22 +00:00
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/vfio.h>
|
|
|
|
#include <linux/wait.h>
|
2019-04-03 18:22:27 +00:00
|
|
|
#include <linux/sched/signal.h>
|
2022-08-29 11:48:47 +00:00
|
|
|
#include <linux/pm_runtime.h>
|
2022-09-08 18:34:43 +00:00
|
|
|
#include <linux/interval_tree.h>
|
|
|
|
#include <linux/iova_bitmap.h>
|
2021-09-24 15:56:59 +00:00
|
|
|
#include "vfio.h"
|
2012-07-31 14:16:22 +00:00
|
|
|
|
|
|
|
#define DRIVER_VERSION "0.3"
|
|
|
|
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
|
|
|
|
#define DRIVER_DESC "VFIO - User Level meta-driver"
|
|
|
|
|
|
|
|
static struct vfio {
|
|
|
|
struct class *class;
|
|
|
|
struct list_head group_list;
|
2021-10-15 11:40:54 +00:00
|
|
|
struct mutex group_lock; /* locks group_list */
|
|
|
|
struct ida group_ida;
|
2013-12-19 17:17:13 +00:00
|
|
|
dev_t group_devt;
|
2022-09-21 10:44:01 +00:00
|
|
|
struct class *device_class;
|
|
|
|
struct ida device_ida;
|
2012-07-31 14:16:22 +00:00
|
|
|
} vfio;
|
|
|
|
|
2021-08-06 01:19:00 +00:00
|
|
|
static DEFINE_XARRAY(vfio_device_set_xa);
|
2021-10-15 11:40:54 +00:00
|
|
|
static const struct file_operations vfio_group_fops;
|
2021-08-06 01:19:00 +00:00
|
|
|
|
|
|
|
int vfio_assign_device_set(struct vfio_device *device, void *set_id)
|
|
|
|
{
|
|
|
|
unsigned long idx = (unsigned long)set_id;
|
|
|
|
struct vfio_device_set *new_dev_set;
|
|
|
|
struct vfio_device_set *dev_set;
|
|
|
|
|
|
|
|
if (WARN_ON(!set_id))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Atomically acquire a singleton object in the xarray for this set_id
|
|
|
|
*/
|
|
|
|
xa_lock(&vfio_device_set_xa);
|
|
|
|
dev_set = xa_load(&vfio_device_set_xa, idx);
|
|
|
|
if (dev_set)
|
|
|
|
goto found_get_ref;
|
|
|
|
xa_unlock(&vfio_device_set_xa);
|
|
|
|
|
|
|
|
new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL);
|
|
|
|
if (!new_dev_set)
|
|
|
|
return -ENOMEM;
|
|
|
|
mutex_init(&new_dev_set->lock);
|
|
|
|
INIT_LIST_HEAD(&new_dev_set->device_list);
|
|
|
|
new_dev_set->set_id = set_id;
|
|
|
|
|
|
|
|
xa_lock(&vfio_device_set_xa);
|
|
|
|
dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!dev_set) {
|
|
|
|
dev_set = new_dev_set;
|
|
|
|
goto found_get_ref;
|
|
|
|
}
|
|
|
|
|
|
|
|
kfree(new_dev_set);
|
|
|
|
if (xa_is_err(dev_set)) {
|
|
|
|
xa_unlock(&vfio_device_set_xa);
|
|
|
|
return xa_err(dev_set);
|
|
|
|
}
|
|
|
|
|
|
|
|
found_get_ref:
|
|
|
|
dev_set->device_count++;
|
|
|
|
xa_unlock(&vfio_device_set_xa);
|
|
|
|
mutex_lock(&dev_set->lock);
|
|
|
|
device->dev_set = dev_set;
|
|
|
|
list_add_tail(&device->dev_set_list, &dev_set->device_list);
|
|
|
|
mutex_unlock(&dev_set->lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_assign_device_set);
|
|
|
|
|
|
|
|
static void vfio_release_device_set(struct vfio_device *device)
|
|
|
|
{
|
|
|
|
struct vfio_device_set *dev_set = device->dev_set;
|
|
|
|
|
|
|
|
if (!dev_set)
|
|
|
|
return;
|
|
|
|
|
|
|
|
mutex_lock(&dev_set->lock);
|
|
|
|
list_del(&device->dev_set_list);
|
|
|
|
mutex_unlock(&dev_set->lock);
|
|
|
|
|
|
|
|
xa_lock(&vfio_device_set_xa);
|
|
|
|
if (!--dev_set->device_count) {
|
|
|
|
__xa_erase(&vfio_device_set_xa,
|
|
|
|
(unsigned long)dev_set->set_id);
|
|
|
|
mutex_destroy(&dev_set->lock);
|
|
|
|
kfree(dev_set);
|
|
|
|
}
|
|
|
|
xa_unlock(&vfio_device_set_xa);
|
|
|
|
}
|
|
|
|
|
2012-07-31 14:16:22 +00:00
|
|
|
static void vfio_group_get(struct vfio_group *group);
|
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2012-07-31 14:16:22 +00:00
|
|
|
* Group objects - create, release, get, put, search
|
|
|
|
*/
|
2021-10-15 11:40:51 +00:00
|
|
|
static struct vfio_group *
|
|
|
|
__vfio_group_get_from_iommu(struct iommu_group *iommu_group)
|
|
|
|
{
|
|
|
|
struct vfio_group *group;
|
|
|
|
|
|
|
|
list_for_each_entry(group, &vfio.group_list, vfio_next) {
|
|
|
|
if (group->iommu_group == iommu_group) {
|
|
|
|
vfio_group_get(group);
|
|
|
|
return group;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct vfio_group *
|
|
|
|
vfio_group_get_from_iommu(struct iommu_group *iommu_group)
|
|
|
|
{
|
|
|
|
struct vfio_group *group;
|
|
|
|
|
|
|
|
mutex_lock(&vfio.group_lock);
|
|
|
|
group = __vfio_group_get_from_iommu(iommu_group);
|
|
|
|
mutex_unlock(&vfio.group_lock);
|
|
|
|
return group;
|
|
|
|
}
|
|
|
|
|
2021-10-15 11:40:54 +00:00
|
|
|
static void vfio_group_release(struct device *dev)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
2021-10-15 11:40:54 +00:00
|
|
|
struct vfio_group *group = container_of(dev, struct vfio_group, dev);
|
|
|
|
|
|
|
|
mutex_destroy(&group->device_lock);
|
|
|
|
iommu_group_put(group->iommu_group);
|
|
|
|
ida_free(&vfio.group_ida, MINOR(group->dev.devt));
|
|
|
|
kfree(group);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct vfio_group *vfio_group_alloc(struct iommu_group *iommu_group,
|
|
|
|
enum vfio_group_type type)
|
|
|
|
{
|
|
|
|
struct vfio_group *group;
|
|
|
|
int minor;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
|
|
|
group = kzalloc(sizeof(*group), GFP_KERNEL);
|
|
|
|
if (!group)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
2021-10-15 11:40:54 +00:00
|
|
|
minor = ida_alloc_max(&vfio.group_ida, MINORMASK, GFP_KERNEL);
|
|
|
|
if (minor < 0) {
|
|
|
|
kfree(group);
|
|
|
|
return ERR_PTR(minor);
|
|
|
|
}
|
|
|
|
|
|
|
|
device_initialize(&group->dev);
|
|
|
|
group->dev.devt = MKDEV(MAJOR(vfio.group_devt), minor);
|
|
|
|
group->dev.class = vfio.class;
|
|
|
|
group->dev.release = vfio_group_release;
|
|
|
|
cdev_init(&group->cdev, &vfio_group_fops);
|
|
|
|
group->cdev.owner = THIS_MODULE;
|
|
|
|
|
2021-10-15 11:40:53 +00:00
|
|
|
refcount_set(&group->users, 1);
|
2022-05-16 23:41:17 +00:00
|
|
|
init_rwsem(&group->group_rwsem);
|
2012-07-31 14:16:22 +00:00
|
|
|
INIT_LIST_HEAD(&group->device_list);
|
|
|
|
mutex_init(&group->device_lock);
|
|
|
|
group->iommu_group = iommu_group;
|
2021-10-15 11:40:54 +00:00
|
|
|
/* put in vfio_group_release() */
|
2021-10-15 11:40:52 +00:00
|
|
|
iommu_group_ref_get(iommu_group);
|
2021-09-24 15:56:57 +00:00
|
|
|
group->type = type;
|
2016-12-01 05:20:06 +00:00
|
|
|
BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2021-10-15 11:40:54 +00:00
|
|
|
return group;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group,
|
|
|
|
enum vfio_group_type type)
|
|
|
|
{
|
|
|
|
struct vfio_group *group;
|
|
|
|
struct vfio_group *ret;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
group = vfio_group_alloc(iommu_group, type);
|
|
|
|
if (IS_ERR(group))
|
|
|
|
return group;
|
|
|
|
|
|
|
|
err = dev_set_name(&group->dev, "%s%d",
|
|
|
|
group->type == VFIO_NO_IOMMU ? "noiommu-" : "",
|
|
|
|
iommu_group_id(iommu_group));
|
|
|
|
if (err) {
|
|
|
|
ret = ERR_PTR(err);
|
|
|
|
goto err_put;
|
|
|
|
}
|
|
|
|
|
2012-07-31 14:16:22 +00:00
|
|
|
mutex_lock(&vfio.group_lock);
|
|
|
|
|
|
|
|
/* Did we race creating this group? */
|
2021-10-15 11:40:54 +00:00
|
|
|
ret = __vfio_group_get_from_iommu(iommu_group);
|
|
|
|
if (ret)
|
|
|
|
goto err_unlock;
|
2015-03-16 20:08:56 +00:00
|
|
|
|
2021-10-15 11:40:54 +00:00
|
|
|
err = cdev_device_add(&group->cdev, &group->dev);
|
|
|
|
if (err) {
|
|
|
|
ret = ERR_PTR(err);
|
|
|
|
goto err_unlock;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
list_add(&group->vfio_next, &vfio.group_list);
|
|
|
|
|
|
|
|
mutex_unlock(&vfio.group_lock);
|
|
|
|
return group;
|
2021-10-15 11:40:54 +00:00
|
|
|
|
|
|
|
err_unlock:
|
|
|
|
mutex_unlock(&vfio.group_lock);
|
|
|
|
err_put:
|
|
|
|
put_device(&group->dev);
|
|
|
|
return ret;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
2021-10-15 11:40:53 +00:00
|
|
|
static void vfio_group_put(struct vfio_group *group)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
2021-10-15 11:40:53 +00:00
|
|
|
if (!refcount_dec_and_mutex_lock(&group->users, &vfio.group_lock))
|
|
|
|
return;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2021-10-15 11:40:50 +00:00
|
|
|
/*
|
|
|
|
* These data structures all have paired operations that can only be
|
|
|
|
* undone when the caller holds a live reference on the group. Since all
|
|
|
|
* pairs must be undone these WARN_ON's indicate some caller did not
|
|
|
|
* properly hold the group reference.
|
|
|
|
*/
|
2012-07-31 14:16:22 +00:00
|
|
|
WARN_ON(!list_empty(&group->device_list));
|
2022-05-16 23:41:22 +00:00
|
|
|
WARN_ON(group->container || group->container_users);
|
2017-03-21 19:19:09 +00:00
|
|
|
WARN_ON(group->notifier.head);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
|
|
|
list_del(&group->vfio_next);
|
2021-10-15 11:40:54 +00:00
|
|
|
cdev_device_del(&group->cdev, &group->dev);
|
|
|
|
mutex_unlock(&vfio.group_lock);
|
|
|
|
|
|
|
|
put_device(&group->dev);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void vfio_group_get(struct vfio_group *group)
|
|
|
|
{
|
2021-10-15 11:40:53 +00:00
|
|
|
refcount_inc(&group->users);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2012-07-31 14:16:22 +00:00
|
|
|
* Device objects - create, release, get, put, search
|
|
|
|
*/
|
|
|
|
/* Device reference always implies a group reference */
|
2022-09-21 10:44:00 +00:00
|
|
|
static void vfio_device_put_registration(struct vfio_device *device)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
2021-03-30 15:53:05 +00:00
|
|
|
if (refcount_dec_and_test(&device->refcount))
|
|
|
|
complete(&device->comp);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
2022-09-21 10:44:00 +00:00
|
|
|
static bool vfio_device_try_get_registration(struct vfio_device *device)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
2021-03-30 15:53:05 +00:00
|
|
|
return refcount_inc_not_zero(&device->refcount);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct vfio_device *vfio_group_get_device(struct vfio_group *group,
|
|
|
|
struct device *dev)
|
|
|
|
{
|
|
|
|
struct vfio_device *device;
|
|
|
|
|
|
|
|
mutex_lock(&group->device_lock);
|
|
|
|
list_for_each_entry(device, &group->device_list, group_next) {
|
2022-09-21 10:44:00 +00:00
|
|
|
if (device->dev == dev &&
|
|
|
|
vfio_device_try_get_registration(device)) {
|
2012-07-31 14:16:22 +00:00
|
|
|
mutex_unlock(&group->device_lock);
|
|
|
|
return device;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
mutex_unlock(&group->device_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2012-07-31 14:16:22 +00:00
|
|
|
* VFIO driver API
|
|
|
|
*/
|
2022-09-21 10:43:47 +00:00
|
|
|
/* Release helper called by vfio_put_device() */
|
2022-09-21 10:44:01 +00:00
|
|
|
static void vfio_device_release(struct device *dev)
|
2022-09-21 10:43:47 +00:00
|
|
|
{
|
|
|
|
struct vfio_device *device =
|
2022-09-21 10:44:01 +00:00
|
|
|
container_of(dev, struct vfio_device, device);
|
2022-09-21 10:43:47 +00:00
|
|
|
|
2022-09-21 10:43:59 +00:00
|
|
|
vfio_release_device_set(device);
|
2022-09-21 10:44:01 +00:00
|
|
|
ida_free(&vfio.device_ida, device->index);
|
2022-09-21 10:43:47 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* kvfree() cannot be done here due to a life cycle mess in
|
|
|
|
* vfio-ccw. Before the ccw part is fixed all drivers are
|
|
|
|
* required to support @release and call vfio_free_device()
|
|
|
|
* from there.
|
|
|
|
*/
|
|
|
|
device->ops->release(device);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate and initialize vfio_device so it can be registered to vfio
|
|
|
|
* core.
|
|
|
|
*
|
|
|
|
* Drivers should use the wrapper vfio_alloc_device() for allocation.
|
|
|
|
* @size is the size of the structure to be allocated, including any
|
|
|
|
* private data used by the driver.
|
|
|
|
*
|
|
|
|
* Driver may provide an @init callback to cover device private data.
|
|
|
|
*
|
|
|
|
* Use vfio_put_device() to release the structure after success return.
|
|
|
|
*/
|
|
|
|
struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
|
|
|
|
const struct vfio_device_ops *ops)
|
|
|
|
{
|
|
|
|
struct vfio_device *device;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (WARN_ON(size < sizeof(struct vfio_device)))
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
device = kvzalloc(size, GFP_KERNEL);
|
|
|
|
if (!device)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
|
|
ret = vfio_init_device(device, dev, ops);
|
|
|
|
if (ret)
|
|
|
|
goto out_free;
|
|
|
|
return device;
|
|
|
|
|
|
|
|
out_free:
|
|
|
|
kvfree(device);
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(_vfio_alloc_device);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize a vfio_device so it can be registered to vfio core.
|
|
|
|
*
|
|
|
|
* Only vfio-ccw driver should call this interface.
|
|
|
|
*/
|
|
|
|
int vfio_init_device(struct vfio_device *device, struct device *dev,
|
|
|
|
const struct vfio_device_ops *ops)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2022-09-21 10:44:01 +00:00
|
|
|
ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
|
|
|
|
if (ret < 0) {
|
|
|
|
dev_dbg(dev, "Error to alloc index\n");
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
device->index = ret;
|
2022-09-21 10:43:59 +00:00
|
|
|
init_completion(&device->comp);
|
|
|
|
device->dev = dev;
|
|
|
|
device->ops = ops;
|
2022-09-21 10:43:47 +00:00
|
|
|
|
|
|
|
if (ops->init) {
|
|
|
|
ret = ops->init(device);
|
|
|
|
if (ret)
|
|
|
|
goto out_uninit;
|
|
|
|
}
|
|
|
|
|
2022-09-21 10:44:01 +00:00
|
|
|
device_initialize(&device->device);
|
|
|
|
device->device.release = vfio_device_release;
|
|
|
|
device->device.class = vfio.device_class;
|
|
|
|
device->device.parent = device->dev;
|
2022-09-21 10:43:47 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
out_uninit:
|
2022-09-21 10:43:59 +00:00
|
|
|
vfio_release_device_set(device);
|
2022-09-21 10:44:01 +00:00
|
|
|
ida_free(&vfio.device_ida, device->index);
|
2022-09-21 10:43:47 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_init_device);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The helper called by driver @release callback to free the device
|
|
|
|
* structure. Drivers which don't have private data to clean can
|
|
|
|
* simply use this helper as its @release.
|
|
|
|
*/
|
|
|
|
void vfio_free_device(struct vfio_device *device)
|
|
|
|
{
|
|
|
|
kvfree(device);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_free_device);
|
|
|
|
|
2021-09-24 15:56:57 +00:00
|
|
|
static struct vfio_group *vfio_noiommu_group_alloc(struct device *dev,
|
|
|
|
enum vfio_group_type type)
|
2021-09-24 15:56:54 +00:00
|
|
|
{
|
|
|
|
struct iommu_group *iommu_group;
|
|
|
|
struct vfio_group *group;
|
2021-09-24 15:56:55 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
iommu_group = iommu_group_alloc();
|
|
|
|
if (IS_ERR(iommu_group))
|
|
|
|
return ERR_CAST(iommu_group);
|
|
|
|
|
2022-06-25 11:42:39 +00:00
|
|
|
ret = iommu_group_set_name(iommu_group, "vfio-noiommu");
|
|
|
|
if (ret)
|
|
|
|
goto out_put_group;
|
2021-09-24 15:56:55 +00:00
|
|
|
ret = iommu_group_add_device(iommu_group, dev);
|
|
|
|
if (ret)
|
|
|
|
goto out_put_group;
|
2021-09-24 15:56:54 +00:00
|
|
|
|
2021-09-24 15:56:57 +00:00
|
|
|
group = vfio_create_group(iommu_group, type);
|
2021-09-24 15:56:55 +00:00
|
|
|
if (IS_ERR(group)) {
|
|
|
|
ret = PTR_ERR(group);
|
|
|
|
goto out_remove_device;
|
|
|
|
}
|
2021-10-15 11:40:52 +00:00
|
|
|
iommu_group_put(iommu_group);
|
2021-09-24 15:56:55 +00:00
|
|
|
return group;
|
|
|
|
|
|
|
|
out_remove_device:
|
|
|
|
iommu_group_remove_device(dev);
|
|
|
|
out_put_group:
|
|
|
|
iommu_group_put(iommu_group);
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct vfio_group *vfio_group_find_or_alloc(struct device *dev)
|
|
|
|
{
|
|
|
|
struct iommu_group *iommu_group;
|
|
|
|
struct vfio_group *group;
|
|
|
|
|
|
|
|
iommu_group = iommu_group_get(dev);
|
2022-09-22 19:20:22 +00:00
|
|
|
if (!iommu_group && vfio_noiommu) {
|
2021-09-24 15:56:55 +00:00
|
|
|
/*
|
|
|
|
* With noiommu enabled, create an IOMMU group for devices that
|
2022-05-11 19:12:58 +00:00
|
|
|
* don't already have one, implying no IOMMU hardware/driver
|
|
|
|
* exists. Taint the kernel because we're about to give a DMA
|
2021-09-24 15:56:55 +00:00
|
|
|
* capable device to a user without IOMMU protection.
|
|
|
|
*/
|
2021-09-24 15:56:57 +00:00
|
|
|
group = vfio_noiommu_group_alloc(dev, VFIO_NO_IOMMU);
|
2021-09-24 15:56:55 +00:00
|
|
|
if (!IS_ERR(group)) {
|
|
|
|
add_taint(TAINT_USER, LOCKDEP_STILL_OK);
|
|
|
|
dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
|
|
|
|
}
|
|
|
|
return group;
|
|
|
|
}
|
2022-09-22 19:20:22 +00:00
|
|
|
|
2021-09-24 15:56:54 +00:00
|
|
|
if (!iommu_group)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
2022-07-05 01:10:50 +00:00
|
|
|
/*
|
|
|
|
* VFIO always sets IOMMU_CACHE because we offer no way for userspace to
|
|
|
|
* restore cache coherency. It has to be checked here because it is only
|
|
|
|
* valid for cases where we are using iommu groups.
|
|
|
|
*/
|
2022-08-06 15:59:35 +00:00
|
|
|
if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY)) {
|
2022-07-05 01:10:50 +00:00
|
|
|
iommu_group_put(iommu_group);
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
}
|
|
|
|
|
2021-09-24 15:56:54 +00:00
|
|
|
group = vfio_group_get_from_iommu(iommu_group);
|
2021-10-15 11:40:52 +00:00
|
|
|
if (!group)
|
|
|
|
group = vfio_create_group(iommu_group, VFIO_IOMMU);
|
2021-09-24 15:56:54 +00:00
|
|
|
|
2021-10-15 11:40:52 +00:00
|
|
|
/* The vfio_group holds a reference to the iommu_group */
|
2021-09-24 15:56:54 +00:00
|
|
|
iommu_group_put(iommu_group);
|
|
|
|
return group;
|
|
|
|
}
|
|
|
|
|
2021-09-24 15:56:57 +00:00
|
|
|
static int __vfio_register_dev(struct vfio_device *device,
|
|
|
|
struct vfio_group *group)
|
2021-03-30 15:53:05 +00:00
|
|
|
{
|
|
|
|
struct vfio_device *existing_device;
|
2022-09-21 10:44:01 +00:00
|
|
|
int ret;
|
2021-09-24 15:56:57 +00:00
|
|
|
|
|
|
|
if (IS_ERR(group))
|
|
|
|
return PTR_ERR(group);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2021-08-06 01:19:00 +00:00
|
|
|
/*
|
|
|
|
* If the driver doesn't specify a set then the device is added to a
|
|
|
|
* singleton set just for itself.
|
|
|
|
*/
|
|
|
|
if (!device->dev_set)
|
|
|
|
vfio_assign_device_set(device, device);
|
|
|
|
|
2021-03-30 15:53:05 +00:00
|
|
|
existing_device = vfio_group_get_device(group, device->dev);
|
|
|
|
if (existing_device) {
|
|
|
|
dev_WARN(device->dev, "Device already exists on group %d\n",
|
2021-09-24 15:56:54 +00:00
|
|
|
iommu_group_id(group->iommu_group));
|
2022-09-21 10:44:00 +00:00
|
|
|
vfio_device_put_registration(existing_device);
|
2022-09-21 10:44:01 +00:00
|
|
|
ret = -EBUSY;
|
|
|
|
goto err_out;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
2021-03-30 15:53:05 +00:00
|
|
|
/* Our reference on group is moved to the device */
|
|
|
|
device->group = group;
|
|
|
|
|
2022-09-21 10:44:01 +00:00
|
|
|
ret = dev_set_name(&device->device, "vfio%d", device->index);
|
|
|
|
if (ret)
|
|
|
|
goto err_out;
|
|
|
|
|
|
|
|
ret = device_add(&device->device);
|
|
|
|
if (ret)
|
|
|
|
goto err_out;
|
|
|
|
|
2021-03-30 15:53:05 +00:00
|
|
|
/* Refcounting can't start until the driver calls register */
|
|
|
|
refcount_set(&device->refcount, 1);
|
|
|
|
|
|
|
|
mutex_lock(&group->device_lock);
|
|
|
|
list_add(&device->group_next, &group->device_list);
|
|
|
|
mutex_unlock(&group->device_lock);
|
|
|
|
|
|
|
|
return 0;
|
2022-09-21 10:44:01 +00:00
|
|
|
err_out:
|
|
|
|
if (group->type == VFIO_NO_IOMMU ||
|
|
|
|
group->type == VFIO_EMULATED_IOMMU)
|
|
|
|
iommu_group_remove_device(device->dev);
|
|
|
|
vfio_group_put(group);
|
|
|
|
return ret;
|
2021-03-30 15:53:05 +00:00
|
|
|
}
|
2021-09-24 15:56:57 +00:00
|
|
|
|
|
|
|
int vfio_register_group_dev(struct vfio_device *device)
|
|
|
|
{
|
|
|
|
return __vfio_register_dev(device,
|
|
|
|
vfio_group_find_or_alloc(device->dev));
|
|
|
|
}
|
2021-03-30 15:53:05 +00:00
|
|
|
EXPORT_SYMBOL_GPL(vfio_register_group_dev);
|
|
|
|
|
2021-09-24 15:56:57 +00:00
|
|
|
/*
|
|
|
|
* Register a virtual device without IOMMU backing. The user of this
|
|
|
|
* device must not be able to directly trigger unmediated DMA.
|
|
|
|
*/
|
|
|
|
int vfio_register_emulated_iommu_dev(struct vfio_device *device)
|
|
|
|
{
|
|
|
|
return __vfio_register_dev(device,
|
|
|
|
vfio_noiommu_group_alloc(device->dev, VFIO_EMULATED_IOMMU));
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
|
|
|
|
|
2015-07-24 21:14:04 +00:00
|
|
|
static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,
|
|
|
|
char *buf)
|
|
|
|
{
|
2020-03-24 15:28:25 +00:00
|
|
|
struct vfio_device *it, *device = ERR_PTR(-ENODEV);
|
2015-07-24 21:14:04 +00:00
|
|
|
|
|
|
|
mutex_lock(&group->device_lock);
|
2015-11-04 12:53:26 +00:00
|
|
|
list_for_each_entry(it, &group->device_list, group_next) {
|
2020-03-24 15:28:25 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (it->ops->match) {
|
2021-03-30 15:53:08 +00:00
|
|
|
ret = it->ops->match(it, buf);
|
2020-03-24 15:28:25 +00:00
|
|
|
if (ret < 0) {
|
|
|
|
device = ERR_PTR(ret);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
ret = !strcmp(dev_name(it->dev), buf);
|
|
|
|
}
|
|
|
|
|
2022-09-21 10:44:00 +00:00
|
|
|
if (ret && vfio_device_try_get_registration(it)) {
|
2015-11-04 12:53:26 +00:00
|
|
|
device = it;
|
2015-07-24 21:14:04 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
mutex_unlock(&group->device_lock);
|
|
|
|
|
|
|
|
return device;
|
|
|
|
}
|
|
|
|
|
2012-07-31 14:16:22 +00:00
|
|
|
/*
|
|
|
|
* Decrement the device reference count and wait for the device to be
|
|
|
|
* removed. Open file descriptors for the device... */
|
2021-03-30 15:53:05 +00:00
|
|
|
void vfio_unregister_group_dev(struct vfio_device *device)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
|
|
|
struct vfio_group *group = device->group;
|
2015-02-06 22:05:07 +00:00
|
|
|
unsigned int i = 0;
|
2015-05-01 22:31:41 +00:00
|
|
|
bool interrupted = false;
|
2021-03-30 15:53:05 +00:00
|
|
|
long rc;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-09-21 10:44:00 +00:00
|
|
|
vfio_device_put_registration(device);
|
2021-03-30 15:53:05 +00:00
|
|
|
rc = try_wait_for_completion(&device->comp);
|
|
|
|
while (rc <= 0) {
|
2015-02-06 22:05:07 +00:00
|
|
|
if (device->ops->request)
|
2021-03-30 15:53:08 +00:00
|
|
|
device->ops->request(device, i++);
|
2015-02-06 22:05:07 +00:00
|
|
|
|
2015-05-01 22:31:41 +00:00
|
|
|
if (interrupted) {
|
2021-03-30 15:53:05 +00:00
|
|
|
rc = wait_for_completion_timeout(&device->comp,
|
|
|
|
HZ * 10);
|
2015-05-01 22:31:41 +00:00
|
|
|
} else {
|
2021-03-30 15:53:05 +00:00
|
|
|
rc = wait_for_completion_interruptible_timeout(
|
|
|
|
&device->comp, HZ * 10);
|
|
|
|
if (rc < 0) {
|
2015-05-01 22:31:41 +00:00
|
|
|
interrupted = true;
|
2021-03-30 15:53:05 +00:00
|
|
|
dev_warn(device->dev,
|
2015-05-01 22:31:41 +00:00
|
|
|
"Device is currently in use, task"
|
|
|
|
" \"%s\" (%d) "
|
|
|
|
"blocked until device is released",
|
|
|
|
current->comm, task_pid_nr(current));
|
|
|
|
}
|
|
|
|
}
|
2021-03-30 15:53:05 +00:00
|
|
|
}
|
2013-02-14 21:02:13 +00:00
|
|
|
|
2021-03-30 15:53:05 +00:00
|
|
|
mutex_lock(&group->device_lock);
|
|
|
|
list_del(&device->group_next);
|
|
|
|
mutex_unlock(&group->device_lock);
|
2019-04-03 18:22:27 +00:00
|
|
|
|
2022-09-21 10:44:01 +00:00
|
|
|
/* Balances device_add in register path */
|
|
|
|
device_del(&device->device);
|
|
|
|
|
2021-09-24 15:56:57 +00:00
|
|
|
if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU)
|
2021-09-24 15:56:51 +00:00
|
|
|
iommu_group_remove_device(device->dev);
|
2021-09-24 15:56:56 +00:00
|
|
|
|
2021-03-30 15:53:05 +00:00
|
|
|
/* Matches the get in vfio_register_group_dev() */
|
2013-02-14 21:02:13 +00:00
|
|
|
vfio_group_put(group);
|
2021-03-30 15:53:05 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
|
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2012-07-31 14:16:22 +00:00
|
|
|
* VFIO Group fd, /dev/vfio/$GROUP
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
|
|
|
|
* if there was no container to unset. Since the ioctl is called on
|
|
|
|
* the group, we know that still exists, therefore the only valid
|
|
|
|
* transition here is 1->0.
|
|
|
|
*/
|
2022-08-31 20:16:02 +00:00
|
|
|
static int vfio_group_ioctl_unset_container(struct vfio_group *group)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
2022-08-31 20:16:02 +00:00
|
|
|
int ret = 0;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-08-31 20:16:02 +00:00
|
|
|
down_write(&group->group_rwsem);
|
|
|
|
if (!group->container) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
if (group->container_users != 1) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
2022-09-22 19:20:20 +00:00
|
|
|
vfio_group_detach_container(group);
|
2022-08-31 20:16:02 +00:00
|
|
|
|
|
|
|
out_unlock:
|
|
|
|
up_write(&group->group_rwsem);
|
|
|
|
return ret;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
2022-09-22 19:20:21 +00:00
|
|
|
static int vfio_group_ioctl_set_container(struct vfio_group *group,
|
|
|
|
int __user *arg)
|
|
|
|
{
|
|
|
|
struct vfio_container *container;
|
|
|
|
struct fd f;
|
|
|
|
int ret;
|
|
|
|
int fd;
|
|
|
|
|
|
|
|
if (get_user(fd, arg))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
f = fdget(fd);
|
|
|
|
if (!f.file)
|
|
|
|
return -EBADF;
|
|
|
|
|
|
|
|
down_write(&group->group_rwsem);
|
|
|
|
if (group->container || WARN_ON(group->container_users)) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
container = vfio_container_from_file(f.file);
|
|
|
|
ret = -EINVAL;
|
|
|
|
if (container) {
|
|
|
|
ret = vfio_container_attach_group(container, group);
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
out_unlock:
|
2022-08-31 20:16:01 +00:00
|
|
|
up_write(&group->group_rwsem);
|
2012-08-28 16:52:22 +00:00
|
|
|
fdput(f);
|
2012-07-31 14:16:22 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct file_operations vfio_device_fops;
|
|
|
|
|
2022-05-11 19:13:00 +00:00
|
|
|
/* true if the vfio_device has open_device() called but not close_device() */
|
2022-09-22 19:20:26 +00:00
|
|
|
bool vfio_assert_device_open(struct vfio_device *device)
|
2016-11-16 20:46:16 +00:00
|
|
|
{
|
2022-05-11 19:13:00 +00:00
|
|
|
return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
|
|
|
|
}
|
|
|
|
|
2022-05-16 23:41:19 +00:00
|
|
|
static struct file *vfio_device_open(struct vfio_device *device)
|
2012-07-31 14:16:22 +00:00
|
|
|
{
|
|
|
|
struct file *filep;
|
2022-05-16 23:41:19 +00:00
|
|
|
int ret;
|
2015-12-21 22:13:33 +00:00
|
|
|
|
2022-05-16 23:41:20 +00:00
|
|
|
down_write(&device->group->group_rwsem);
|
2022-05-16 23:41:19 +00:00
|
|
|
ret = vfio_device_assign_container(device);
|
2022-05-16 23:41:20 +00:00
|
|
|
up_write(&device->group->group_rwsem);
|
2022-05-16 23:41:19 +00:00
|
|
|
if (ret)
|
|
|
|
return ERR_PTR(ret);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2021-05-18 19:21:32 +00:00
|
|
|
if (!try_module_get(device->dev->driver->owner)) {
|
2021-08-06 01:19:00 +00:00
|
|
|
ret = -ENODEV;
|
2022-05-16 23:41:19 +00:00
|
|
|
goto err_unassign_container;
|
2021-05-18 19:21:32 +00:00
|
|
|
}
|
|
|
|
|
2021-08-06 01:19:00 +00:00
|
|
|
mutex_lock(&device->dev_set->lock);
|
|
|
|
device->open_count++;
|
2022-05-19 18:33:11 +00:00
|
|
|
if (device->open_count == 1) {
|
|
|
|
/*
|
|
|
|
* Here we pass the KVM pointer with the group under the read
|
|
|
|
* lock. If the device driver will use it, it must obtain a
|
|
|
|
* reference and release it during close_device.
|
|
|
|
*/
|
|
|
|
down_read(&device->group->group_rwsem);
|
|
|
|
device->kvm = device->group->kvm;
|
|
|
|
|
|
|
|
if (device->ops->open_device) {
|
|
|
|
ret = device->ops->open_device(device);
|
|
|
|
if (ret)
|
|
|
|
goto err_undo_count;
|
|
|
|
}
|
2022-09-22 19:20:25 +00:00
|
|
|
vfio_device_container_register(device);
|
2022-05-19 18:33:11 +00:00
|
|
|
up_read(&device->group->group_rwsem);
|
2021-08-06 01:19:00 +00:00
|
|
|
}
|
|
|
|
mutex_unlock(&device->dev_set->lock);
|
|
|
|
|
2015-07-24 21:14:04 +00:00
|
|
|
/*
|
|
|
|
* We can't use anon_inode_getfd() because we need to modify
|
|
|
|
* the f_mode flags directly to allow more than just ioctls
|
|
|
|
*/
|
|
|
|
filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
|
|
|
|
device, O_RDWR);
|
|
|
|
if (IS_ERR(filep)) {
|
|
|
|
ret = PTR_ERR(filep);
|
2022-05-16 23:41:19 +00:00
|
|
|
goto err_close_device;
|
2015-07-24 21:14:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* TODO: add an anon_inode interface to do this.
|
|
|
|
* Appears to be missing by lack of need rather than
|
|
|
|
* explicitly prevented. Now there's need.
|
|
|
|
*/
|
2022-06-29 13:07:02 +00:00
|
|
|
filep->f_mode |= (FMODE_PREAD | FMODE_PWRITE);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-05-16 23:41:19 +00:00
|
|
|
if (device->group->type == VFIO_NO_IOMMU)
|
2015-12-21 22:13:33 +00:00
|
|
|
dev_warn(device->dev, "vfio-noiommu device opened by user "
|
|
|
|
"(%s:%d)\n", current->comm, task_pid_nr(current));
|
2022-05-16 23:41:19 +00:00
|
|
|
/*
|
|
|
|
* On success the ref of device is moved to the file and
|
|
|
|
* put in vfio_device_fops_release()
|
|
|
|
*/
|
|
|
|
return filep;
|
2015-12-21 22:13:33 +00:00
|
|
|
|
2021-08-06 01:19:00 +00:00
|
|
|
err_close_device:
|
|
|
|
mutex_lock(&device->dev_set->lock);
|
2022-05-19 18:33:11 +00:00
|
|
|
down_read(&device->group->group_rwsem);
|
2022-07-20 00:02:48 +00:00
|
|
|
if (device->open_count == 1 && device->ops->close_device) {
|
2021-08-06 01:19:00 +00:00
|
|
|
device->ops->close_device(device);
|
2022-07-20 00:02:48 +00:00
|
|
|
|
2022-09-22 19:20:25 +00:00
|
|
|
vfio_device_container_unregister(device);
|
2022-07-20 00:02:48 +00:00
|
|
|
}
|
2021-08-06 01:19:00 +00:00
|
|
|
err_undo_count:
|
2022-06-27 07:41:19 +00:00
|
|
|
up_read(&device->group->group_rwsem);
|
2021-08-06 01:19:00 +00:00
|
|
|
device->open_count--;
|
2022-05-19 18:33:11 +00:00
|
|
|
if (device->open_count == 0 && device->kvm)
|
|
|
|
device->kvm = NULL;
|
2021-08-06 01:19:00 +00:00
|
|
|
mutex_unlock(&device->dev_set->lock);
|
|
|
|
module_put(device->dev->driver->owner);
|
2022-05-16 23:41:19 +00:00
|
|
|
err_unassign_container:
|
2022-05-16 23:41:21 +00:00
|
|
|
vfio_device_unassign_container(device);
|
2022-05-16 23:41:19 +00:00
|
|
|
return ERR_PTR(ret);
|
|
|
|
}
|
|
|
|
|
2022-08-31 20:16:00 +00:00
|
|
|
static int vfio_group_ioctl_get_device_fd(struct vfio_group *group,
|
|
|
|
char __user *arg)
|
2022-05-16 23:41:19 +00:00
|
|
|
{
|
|
|
|
struct vfio_device *device;
|
|
|
|
struct file *filep;
|
2022-08-31 20:16:00 +00:00
|
|
|
char *buf;
|
2022-05-16 23:41:19 +00:00
|
|
|
int fdno;
|
|
|
|
int ret;
|
|
|
|
|
2022-08-31 20:16:00 +00:00
|
|
|
buf = strndup_user(arg, PAGE_SIZE);
|
|
|
|
if (IS_ERR(buf))
|
|
|
|
return PTR_ERR(buf);
|
|
|
|
|
2022-05-16 23:41:19 +00:00
|
|
|
device = vfio_device_get_from_name(group, buf);
|
2022-08-31 20:16:00 +00:00
|
|
|
kfree(buf);
|
2022-05-16 23:41:19 +00:00
|
|
|
if (IS_ERR(device))
|
|
|
|
return PTR_ERR(device);
|
|
|
|
|
|
|
|
fdno = get_unused_fd_flags(O_CLOEXEC);
|
|
|
|
if (fdno < 0) {
|
|
|
|
ret = fdno;
|
|
|
|
goto err_put_device;
|
|
|
|
}
|
|
|
|
|
|
|
|
filep = vfio_device_open(device);
|
|
|
|
if (IS_ERR(filep)) {
|
|
|
|
ret = PTR_ERR(filep);
|
|
|
|
goto err_put_fdno;
|
|
|
|
}
|
|
|
|
|
|
|
|
fd_install(fdno, filep);
|
|
|
|
return fdno;
|
|
|
|
|
|
|
|
err_put_fdno:
|
|
|
|
put_unused_fd(fdno);
|
|
|
|
err_put_device:
|
2022-09-21 10:44:00 +00:00
|
|
|
vfio_device_put_registration(device);
|
2012-07-31 14:16:22 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2022-08-31 20:16:03 +00:00
|
|
|
static int vfio_group_ioctl_get_status(struct vfio_group *group,
|
|
|
|
struct vfio_group_status __user *arg)
|
|
|
|
{
|
|
|
|
unsigned long minsz = offsetofend(struct vfio_group_status, flags);
|
|
|
|
struct vfio_group_status status;
|
|
|
|
|
|
|
|
if (copy_from_user(&status, arg, minsz))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (status.argsz < minsz)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
status.flags = 0;
|
|
|
|
|
|
|
|
down_read(&group->group_rwsem);
|
|
|
|
if (group->container)
|
|
|
|
status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET |
|
|
|
|
VFIO_GROUP_FLAGS_VIABLE;
|
|
|
|
else if (!iommu_group_dma_owner_claimed(group->iommu_group))
|
|
|
|
status.flags |= VFIO_GROUP_FLAGS_VIABLE;
|
|
|
|
up_read(&group->group_rwsem);
|
|
|
|
|
|
|
|
if (copy_to_user(arg, &status, minsz))
|
|
|
|
return -EFAULT;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-07-31 14:16:22 +00:00
|
|
|
static long vfio_group_fops_unl_ioctl(struct file *filep,
|
|
|
|
unsigned int cmd, unsigned long arg)
|
|
|
|
{
|
|
|
|
struct vfio_group *group = filep->private_data;
|
2022-08-31 20:16:00 +00:00
|
|
|
void __user *uarg = (void __user *)arg;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
|
|
|
switch (cmd) {
|
2022-08-31 20:16:00 +00:00
|
|
|
case VFIO_GROUP_GET_DEVICE_FD:
|
|
|
|
return vfio_group_ioctl_get_device_fd(group, uarg);
|
2012-07-31 14:16:22 +00:00
|
|
|
case VFIO_GROUP_GET_STATUS:
|
2022-08-31 20:16:03 +00:00
|
|
|
return vfio_group_ioctl_get_status(group, uarg);
|
2012-07-31 14:16:22 +00:00
|
|
|
case VFIO_GROUP_SET_CONTAINER:
|
2022-08-31 20:16:01 +00:00
|
|
|
return vfio_group_ioctl_set_container(group, uarg);
|
2012-07-31 14:16:22 +00:00
|
|
|
case VFIO_GROUP_UNSET_CONTAINER:
|
2022-08-31 20:16:02 +00:00
|
|
|
return vfio_group_ioctl_unset_container(group);
|
2022-08-31 20:16:03 +00:00
|
|
|
default:
|
|
|
|
return -ENOTTY;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int vfio_group_fops_open(struct inode *inode, struct file *filep)
|
|
|
|
{
|
2021-10-15 11:40:54 +00:00
|
|
|
struct vfio_group *group =
|
|
|
|
container_of(inode->i_cdev, struct vfio_group, cdev);
|
2022-05-16 23:41:18 +00:00
|
|
|
int ret;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-05-16 23:41:18 +00:00
|
|
|
down_write(&group->group_rwsem);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-05-16 23:41:18 +00:00
|
|
|
/* users can be zero if this races with vfio_group_put() */
|
|
|
|
if (!refcount_inc_not_zero(&group->users)) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto err_unlock;
|
2015-12-21 22:13:33 +00:00
|
|
|
}
|
|
|
|
|
2022-05-16 23:41:18 +00:00
|
|
|
if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) {
|
|
|
|
ret = -EPERM;
|
|
|
|
goto err_put;
|
2013-06-25 22:06:54 +00:00
|
|
|
}
|
|
|
|
|
2022-05-16 23:41:18 +00:00
|
|
|
/*
|
|
|
|
* Do we need multiple instances of the group open? Seems not.
|
|
|
|
*/
|
2022-05-16 23:41:21 +00:00
|
|
|
if (group->opened_file) {
|
2022-05-16 23:41:18 +00:00
|
|
|
ret = -EBUSY;
|
|
|
|
goto err_put;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
2022-05-16 23:41:21 +00:00
|
|
|
group->opened_file = filep;
|
2012-07-31 14:16:22 +00:00
|
|
|
filep->private_data = group;
|
|
|
|
|
2022-05-16 23:41:18 +00:00
|
|
|
up_write(&group->group_rwsem);
|
2012-07-31 14:16:22 +00:00
|
|
|
return 0;
|
2022-05-16 23:41:18 +00:00
|
|
|
err_put:
|
|
|
|
vfio_group_put(group);
|
|
|
|
err_unlock:
|
|
|
|
up_write(&group->group_rwsem);
|
|
|
|
return ret;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int vfio_group_fops_release(struct inode *inode, struct file *filep)
|
|
|
|
{
|
|
|
|
struct vfio_group *group = filep->private_data;
|
|
|
|
|
|
|
|
filep->private_data = NULL;
|
|
|
|
|
2022-05-16 23:41:18 +00:00
|
|
|
down_write(&group->group_rwsem);
|
2022-05-16 23:41:21 +00:00
|
|
|
/*
|
|
|
|
* Device FDs hold a group file reference, therefore the group release
|
|
|
|
* is only called when there are no open devices.
|
|
|
|
*/
|
|
|
|
WARN_ON(group->notifier.head);
|
2022-09-22 19:20:20 +00:00
|
|
|
if (group->container)
|
|
|
|
vfio_group_detach_container(group);
|
2022-05-16 23:41:21 +00:00
|
|
|
group->opened_file = NULL;
|
2022-05-16 23:41:18 +00:00
|
|
|
up_write(&group->group_rwsem);
|
2013-06-25 22:06:54 +00:00
|
|
|
|
2012-07-31 14:16:22 +00:00
|
|
|
vfio_group_put(group);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct file_operations vfio_group_fops = {
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.unlocked_ioctl = vfio_group_fops_unl_ioctl,
|
2018-09-11 15:23:00 +00:00
|
|
|
.compat_ioctl = compat_ptr_ioctl,
|
2012-07-31 14:16:22 +00:00
|
|
|
.open = vfio_group_fops_open,
|
|
|
|
.release = vfio_group_fops_release,
|
|
|
|
};
|
|
|
|
|
2022-08-29 11:48:47 +00:00
|
|
|
/*
|
|
|
|
* Wrapper around pm_runtime_resume_and_get().
|
|
|
|
* Return error code on failure or 0 on success.
|
|
|
|
*/
|
|
|
|
static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
|
|
|
|
{
|
|
|
|
struct device *dev = device->dev;
|
|
|
|
|
|
|
|
if (dev->driver && dev->driver->pm) {
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = pm_runtime_resume_and_get(dev);
|
|
|
|
if (ret) {
|
|
|
|
dev_info_ratelimited(dev,
|
|
|
|
"vfio: runtime resume failed %d\n", ret);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Wrapper around pm_runtime_put().
|
|
|
|
*/
|
|
|
|
static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
|
|
|
|
{
|
|
|
|
struct device *dev = device->dev;
|
|
|
|
|
|
|
|
if (dev->driver && dev->driver->pm)
|
|
|
|
pm_runtime_put(dev);
|
|
|
|
}
|
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2012-07-31 14:16:22 +00:00
|
|
|
* VFIO Device fd
|
|
|
|
*/
|
|
|
|
static int vfio_device_fops_release(struct inode *inode, struct file *filep)
|
|
|
|
{
|
|
|
|
struct vfio_device *device = filep->private_data;
|
|
|
|
|
2021-08-06 01:19:00 +00:00
|
|
|
mutex_lock(&device->dev_set->lock);
|
2022-05-11 19:13:00 +00:00
|
|
|
vfio_assert_device_open(device);
|
2022-05-19 18:33:11 +00:00
|
|
|
down_read(&device->group->group_rwsem);
|
2022-05-11 19:13:00 +00:00
|
|
|
if (device->open_count == 1 && device->ops->close_device)
|
2021-08-06 01:19:00 +00:00
|
|
|
device->ops->close_device(device);
|
2022-07-20 00:02:48 +00:00
|
|
|
|
2022-09-22 19:20:25 +00:00
|
|
|
vfio_device_container_unregister(device);
|
2022-05-19 18:33:11 +00:00
|
|
|
up_read(&device->group->group_rwsem);
|
2022-05-11 19:13:00 +00:00
|
|
|
device->open_count--;
|
2022-05-19 18:33:11 +00:00
|
|
|
if (device->open_count == 0)
|
|
|
|
device->kvm = NULL;
|
2021-08-06 01:19:00 +00:00
|
|
|
mutex_unlock(&device->dev_set->lock);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2021-05-18 19:21:32 +00:00
|
|
|
module_put(device->dev->driver->owner);
|
|
|
|
|
2022-05-16 23:41:21 +00:00
|
|
|
vfio_device_unassign_container(device);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-09-21 10:44:00 +00:00
|
|
|
vfio_device_put_registration(device);
|
2012-07-31 14:16:22 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
/*
|
|
|
|
* vfio_mig_get_next_state - Compute the next step in the FSM
|
|
|
|
* @cur_fsm - The current state the device is in
|
|
|
|
* @new_fsm - The target state to reach
|
|
|
|
* @next_fsm - Pointer to the next step to get to new_fsm
|
|
|
|
*
|
|
|
|
* Return 0 upon success, otherwise -errno
|
|
|
|
* Upon success the next step in the state progression between cur_fsm and
|
|
|
|
* new_fsm will be set in next_fsm.
|
|
|
|
*
|
|
|
|
* This breaks down requests for combination transitions into smaller steps and
|
|
|
|
* returns the next step to get to new_fsm. The function may need to be called
|
|
|
|
* multiple times before reaching new_fsm.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
int vfio_mig_get_next_state(struct vfio_device *device,
|
|
|
|
enum vfio_device_mig_state cur_fsm,
|
|
|
|
enum vfio_device_mig_state new_fsm,
|
|
|
|
enum vfio_device_mig_state *next_fsm)
|
|
|
|
{
|
2022-02-24 14:20:19 +00:00
|
|
|
enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_RUNNING_P2P + 1 };
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
/*
|
2022-02-24 14:20:19 +00:00
|
|
|
* The coding in this table requires the driver to implement the
|
|
|
|
* following FSM arcs:
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
* RESUMING -> STOP
|
|
|
|
* STOP -> RESUMING
|
|
|
|
* STOP -> STOP_COPY
|
|
|
|
* STOP_COPY -> STOP
|
|
|
|
*
|
2022-02-24 14:20:19 +00:00
|
|
|
* If P2P is supported then the driver must also implement these FSM
|
|
|
|
* arcs:
|
|
|
|
* RUNNING -> RUNNING_P2P
|
|
|
|
* RUNNING_P2P -> RUNNING
|
|
|
|
* RUNNING_P2P -> STOP
|
|
|
|
* STOP -> RUNNING_P2P
|
|
|
|
* Without P2P the driver must implement:
|
|
|
|
* RUNNING -> STOP
|
|
|
|
* STOP -> RUNNING
|
|
|
|
*
|
|
|
|
* The coding will step through multiple states for some combination
|
|
|
|
* transitions; if all optional features are supported, this means the
|
|
|
|
* following ones:
|
|
|
|
* RESUMING -> STOP -> RUNNING_P2P
|
|
|
|
* RESUMING -> STOP -> RUNNING_P2P -> RUNNING
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
* RESUMING -> STOP -> STOP_COPY
|
2022-02-24 14:20:19 +00:00
|
|
|
* RUNNING -> RUNNING_P2P -> STOP
|
|
|
|
* RUNNING -> RUNNING_P2P -> STOP -> RESUMING
|
|
|
|
* RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
|
|
|
|
* RUNNING_P2P -> STOP -> RESUMING
|
|
|
|
* RUNNING_P2P -> STOP -> STOP_COPY
|
|
|
|
* STOP -> RUNNING_P2P -> RUNNING
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
* STOP_COPY -> STOP -> RESUMING
|
2022-02-24 14:20:19 +00:00
|
|
|
* STOP_COPY -> STOP -> RUNNING_P2P
|
|
|
|
* STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
*/
|
|
|
|
static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
},
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = {
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
},
|
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
},
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
},
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
|
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
},
|
|
|
|
[VFIO_DEVICE_STATE_ERROR] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
|
2022-02-24 14:20:19 +00:00
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
2022-02-24 14:20:19 +00:00
|
|
|
static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
|
|
|
|
[VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
|
|
|
|
[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
|
|
|
|
[VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
|
|
|
|
[VFIO_DEVICE_STATE_RUNNING_P2P] =
|
|
|
|
VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
|
|
|
|
[VFIO_DEVICE_STATE_ERROR] = ~0U,
|
|
|
|
};
|
|
|
|
|
|
|
|
if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
|
|
|
|
(state_flags_table[cur_fsm] & device->migration_flags) !=
|
|
|
|
state_flags_table[cur_fsm]))
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2022-02-24 14:20:19 +00:00
|
|
|
if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
|
|
|
|
(state_flags_table[new_fsm] & device->migration_flags) !=
|
|
|
|
state_flags_table[new_fsm])
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2022-02-24 14:20:19 +00:00
|
|
|
/*
|
|
|
|
* Arcs touching optional and unsupported states are skipped over. The
|
|
|
|
* driver will instead see an arc from the original state to the next
|
|
|
|
* logical state, as per the above comment.
|
|
|
|
*/
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
*next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
|
2022-02-24 14:20:19 +00:00
|
|
|
while ((state_flags_table[*next_fsm] & device->migration_flags) !=
|
|
|
|
state_flags_table[*next_fsm])
|
|
|
|
*next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
|
|
|
|
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert the drivers's struct file into a FD number and return it to userspace
|
|
|
|
*/
|
|
|
|
static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
|
|
|
|
struct vfio_device_feature_mig_state *mig)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
int fd;
|
|
|
|
|
|
|
|
fd = get_unused_fd_flags(O_CLOEXEC);
|
|
|
|
if (fd < 0) {
|
|
|
|
ret = fd;
|
|
|
|
goto out_fput;
|
|
|
|
}
|
|
|
|
|
|
|
|
mig->data_fd = fd;
|
|
|
|
if (copy_to_user(arg, mig, sizeof(*mig))) {
|
|
|
|
ret = -EFAULT;
|
|
|
|
goto out_put_unused;
|
|
|
|
}
|
|
|
|
fd_install(fd, filp);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
out_put_unused:
|
|
|
|
put_unused_fd(fd);
|
|
|
|
out_fput:
|
|
|
|
fput(filp);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
|
|
|
|
u32 flags, void __user *arg,
|
|
|
|
size_t argsz)
|
|
|
|
{
|
|
|
|
size_t minsz =
|
|
|
|
offsetofend(struct vfio_device_feature_mig_state, data_fd);
|
|
|
|
struct vfio_device_feature_mig_state mig;
|
|
|
|
struct file *filp = NULL;
|
|
|
|
int ret;
|
|
|
|
|
2022-06-28 15:59:10 +00:00
|
|
|
if (!device->mig_ops)
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
return -ENOTTY;
|
|
|
|
|
|
|
|
ret = vfio_check_feature(flags, argsz,
|
|
|
|
VFIO_DEVICE_FEATURE_SET |
|
|
|
|
VFIO_DEVICE_FEATURE_GET,
|
|
|
|
sizeof(mig));
|
|
|
|
if (ret != 1)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (copy_from_user(&mig, arg, minsz))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (flags & VFIO_DEVICE_FEATURE_GET) {
|
|
|
|
enum vfio_device_mig_state curr_state;
|
|
|
|
|
2022-06-28 15:59:10 +00:00
|
|
|
ret = device->mig_ops->migration_get_state(device,
|
|
|
|
&curr_state);
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
mig.device_state = curr_state;
|
|
|
|
goto out_copy;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Handle the VFIO_DEVICE_FEATURE_SET */
|
2022-06-28 15:59:10 +00:00
|
|
|
filp = device->mig_ops->migration_set_state(device, mig.device_state);
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
if (IS_ERR(filp) || !filp)
|
|
|
|
goto out_copy;
|
|
|
|
|
|
|
|
return vfio_ioct_mig_return_fd(filp, arg, &mig);
|
|
|
|
out_copy:
|
|
|
|
mig.data_fd = -1;
|
|
|
|
if (copy_to_user(arg, &mig, sizeof(mig)))
|
|
|
|
return -EFAULT;
|
|
|
|
if (IS_ERR(filp))
|
|
|
|
return PTR_ERR(filp);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
|
|
|
|
u32 flags, void __user *arg,
|
|
|
|
size_t argsz)
|
|
|
|
{
|
|
|
|
struct vfio_device_feature_migration mig = {
|
2022-02-24 14:20:19 +00:00
|
|
|
.flags = device->migration_flags,
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
};
|
|
|
|
int ret;
|
|
|
|
|
2022-06-28 15:59:10 +00:00
|
|
|
if (!device->mig_ops)
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
return -ENOTTY;
|
|
|
|
|
|
|
|
ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
|
|
|
|
sizeof(mig));
|
|
|
|
if (ret != 1)
|
|
|
|
return ret;
|
|
|
|
if (copy_to_user(arg, &mig, sizeof(mig)))
|
|
|
|
return -EFAULT;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2022-09-08 18:34:43 +00:00
|
|
|
/* Ranges should fit into a single kernel page */
|
|
|
|
#define LOG_MAX_RANGES \
|
|
|
|
(PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
|
|
|
|
|
|
|
|
static int
|
|
|
|
vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
|
|
|
|
u32 flags, void __user *arg,
|
|
|
|
size_t argsz)
|
|
|
|
{
|
|
|
|
size_t minsz =
|
|
|
|
offsetofend(struct vfio_device_feature_dma_logging_control,
|
|
|
|
ranges);
|
|
|
|
struct vfio_device_feature_dma_logging_range __user *ranges;
|
|
|
|
struct vfio_device_feature_dma_logging_control control;
|
|
|
|
struct vfio_device_feature_dma_logging_range range;
|
|
|
|
struct rb_root_cached root = RB_ROOT_CACHED;
|
|
|
|
struct interval_tree_node *nodes;
|
|
|
|
u64 iova_end;
|
|
|
|
u32 nnodes;
|
|
|
|
int i, ret;
|
|
|
|
|
|
|
|
if (!device->log_ops)
|
|
|
|
return -ENOTTY;
|
|
|
|
|
|
|
|
ret = vfio_check_feature(flags, argsz,
|
|
|
|
VFIO_DEVICE_FEATURE_SET,
|
|
|
|
sizeof(control));
|
|
|
|
if (ret != 1)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (copy_from_user(&control, arg, minsz))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
nnodes = control.num_ranges;
|
|
|
|
if (!nnodes)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (nnodes > LOG_MAX_RANGES)
|
|
|
|
return -E2BIG;
|
|
|
|
|
|
|
|
ranges = u64_to_user_ptr(control.ranges);
|
|
|
|
nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!nodes)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for (i = 0; i < nnodes; i++) {
|
|
|
|
if (copy_from_user(&range, &ranges[i], sizeof(range))) {
|
|
|
|
ret = -EFAULT;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
if (!IS_ALIGNED(range.iova, control.page_size) ||
|
|
|
|
!IS_ALIGNED(range.length, control.page_size)) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (check_add_overflow(range.iova, range.length, &iova_end) ||
|
|
|
|
iova_end > ULONG_MAX) {
|
|
|
|
ret = -EOVERFLOW;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
|
|
|
|
nodes[i].start = range.iova;
|
|
|
|
nodes[i].last = range.iova + range.length - 1;
|
|
|
|
if (interval_tree_iter_first(&root, nodes[i].start,
|
|
|
|
nodes[i].last)) {
|
|
|
|
/* Range overlapping */
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
interval_tree_insert(nodes + i, &root);
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = device->log_ops->log_start(device, &root, nnodes,
|
|
|
|
&control.page_size);
|
|
|
|
if (ret)
|
|
|
|
goto end;
|
|
|
|
|
|
|
|
if (copy_to_user(arg, &control, sizeof(control))) {
|
|
|
|
ret = -EFAULT;
|
|
|
|
device->log_ops->log_stop(device);
|
|
|
|
}
|
|
|
|
|
|
|
|
end:
|
|
|
|
kfree(nodes);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
|
|
|
|
u32 flags, void __user *arg,
|
|
|
|
size_t argsz)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!device->log_ops)
|
|
|
|
return -ENOTTY;
|
|
|
|
|
|
|
|
ret = vfio_check_feature(flags, argsz,
|
|
|
|
VFIO_DEVICE_FEATURE_SET, 0);
|
|
|
|
if (ret != 1)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
return device->log_ops->log_stop(device);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
|
|
|
|
unsigned long iova, size_t length,
|
|
|
|
void *opaque)
|
|
|
|
{
|
|
|
|
struct vfio_device *device = opaque;
|
|
|
|
|
|
|
|
return device->log_ops->log_read_and_clear(device, iova, length, iter);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
|
|
|
|
u32 flags, void __user *arg,
|
|
|
|
size_t argsz)
|
|
|
|
{
|
|
|
|
size_t minsz =
|
|
|
|
offsetofend(struct vfio_device_feature_dma_logging_report,
|
|
|
|
bitmap);
|
|
|
|
struct vfio_device_feature_dma_logging_report report;
|
|
|
|
struct iova_bitmap *iter;
|
|
|
|
u64 iova_end;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!device->log_ops)
|
|
|
|
return -ENOTTY;
|
|
|
|
|
|
|
|
ret = vfio_check_feature(flags, argsz,
|
|
|
|
VFIO_DEVICE_FEATURE_GET,
|
|
|
|
sizeof(report));
|
|
|
|
if (ret != 1)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (copy_from_user(&report, arg, minsz))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (check_add_overflow(report.iova, report.length, &iova_end) ||
|
|
|
|
iova_end > ULONG_MAX)
|
|
|
|
return -EOVERFLOW;
|
|
|
|
|
|
|
|
iter = iova_bitmap_alloc(report.iova, report.length,
|
|
|
|
report.page_size,
|
|
|
|
u64_to_user_ptr(report.bitmap));
|
|
|
|
if (IS_ERR(iter))
|
|
|
|
return PTR_ERR(iter);
|
|
|
|
|
|
|
|
ret = iova_bitmap_for_each(iter, device,
|
|
|
|
vfio_device_log_read_and_clear);
|
|
|
|
|
|
|
|
iova_bitmap_free(iter);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2022-02-24 14:20:17 +00:00
|
|
|
static int vfio_ioctl_device_feature(struct vfio_device *device,
|
|
|
|
struct vfio_device_feature __user *arg)
|
|
|
|
{
|
|
|
|
size_t minsz = offsetofend(struct vfio_device_feature, flags);
|
|
|
|
struct vfio_device_feature feature;
|
|
|
|
|
|
|
|
if (copy_from_user(&feature, arg, minsz))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (feature.argsz < minsz)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Check unknown flags */
|
|
|
|
if (feature.flags &
|
|
|
|
~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
|
|
|
|
VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* GET & SET are mutually exclusive except with PROBE */
|
|
|
|
if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
|
|
|
|
(feature.flags & VFIO_DEVICE_FEATURE_SET) &&
|
|
|
|
(feature.flags & VFIO_DEVICE_FEATURE_GET))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
|
vfio: Define device migration protocol v2
Replace the existing region based migration protocol with an ioctl based
protocol. The two protocols have the same general semantic behaviors, but
the way the data is transported is changed.
This is the STOP_COPY portion of the new protocol, it defines the 5 states
for basic stop and copy migration and the protocol to move the migration
data in/out of the kernel.
Compared to the clarification of the v1 protocol Alex proposed:
https://lore.kernel.org/r/163909282574.728533.7460416142511440919.stgit@omen
This has a few deliberate functional differences:
- ERROR arcs allow the device function to remain unchanged.
- The protocol is not required to return to the original state on
transition failure. Instead userspace can execute an unwind back to
the original state, reset, or do something else without needing kernel
support. This simplifies the kernel design and should userspace choose
a policy like always reset, avoids doing useless work in the kernel
on error handling paths.
- PRE_COPY is made optional, userspace must discover it before using it.
This reflects the fact that the majority of drivers we are aware of
right now will not implement PRE_COPY.
- segmentation is not part of the data stream protocol, the receiver
does not have to reproduce the framing boundaries.
The hybrid FSM for the device_state is described as a Mealy machine by
documenting each of the arcs the driver is required to implement. Defining
the remaining set of old/new device_state transitions as 'combination
transitions' which are naturally defined as taking multiple FSM arcs along
the shortest path within the FSM's digraph allows a complete matrix of
transitions.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE is
defined to replace writing to the device_state field in the region. This
allows returning a brand new FD whenever the requested transition opens
a data transfer session.
The VFIO core code implements the new feature and provides a helper
function to the driver. Using the helper the driver only has to
implement 6 of the FSM arcs and the other combination transitions are
elaborated consistently from those arcs.
A new VFIO_DEVICE_FEATURE of VFIO_DEVICE_FEATURE_MIGRATION is defined to
report the capability for migration and indicate which set of states and
arcs are supported by the device. The FSM provides a lot of flexibility to
make backwards compatible extensions but the VFIO_DEVICE_FEATURE also
allows for future breaking extensions for scenarios that cannot support
even the basic STOP_COPY requirements.
The VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE with the GET option (i.e.
VFIO_DEVICE_FEATURE_GET) can be used to read the current migration state
of the VFIO device.
Data transfer sessions are now carried over a file descriptor, instead of
the region. The FD functions for the lifetime of the data transfer
session. read() and write() transfer the data with normal Linux stream FD
semantics. This design allows future expansion to support poll(),
io_uring, and other performance optimizations.
The complicated mmap mode for data transfer is discarded as current qemu
doesn't take meaningful advantage of it, and the new qemu implementation
avoids substantially all the performance penalty of using a read() on the
region.
Link: https://lore.kernel.org/all/20220224142024.147653-10-yishaih@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
2022-02-24 14:20:18 +00:00
|
|
|
case VFIO_DEVICE_FEATURE_MIGRATION:
|
|
|
|
return vfio_ioctl_device_feature_migration(
|
|
|
|
device, feature.flags, arg->data,
|
|
|
|
feature.argsz - minsz);
|
|
|
|
case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
|
|
|
|
return vfio_ioctl_device_feature_mig_device_state(
|
|
|
|
device, feature.flags, arg->data,
|
|
|
|
feature.argsz - minsz);
|
2022-09-08 18:34:43 +00:00
|
|
|
case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
|
|
|
|
return vfio_ioctl_device_feature_logging_start(
|
|
|
|
device, feature.flags, arg->data,
|
|
|
|
feature.argsz - minsz);
|
|
|
|
case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
|
|
|
|
return vfio_ioctl_device_feature_logging_stop(
|
|
|
|
device, feature.flags, arg->data,
|
|
|
|
feature.argsz - minsz);
|
|
|
|
case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
|
|
|
|
return vfio_ioctl_device_feature_logging_report(
|
|
|
|
device, feature.flags, arg->data,
|
|
|
|
feature.argsz - minsz);
|
2022-02-24 14:20:17 +00:00
|
|
|
default:
|
|
|
|
if (unlikely(!device->ops->device_feature))
|
|
|
|
return -EINVAL;
|
|
|
|
return device->ops->device_feature(device, feature.flags,
|
|
|
|
arg->data,
|
|
|
|
feature.argsz - minsz);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-07-31 14:16:22 +00:00
|
|
|
static long vfio_device_fops_unl_ioctl(struct file *filep,
|
|
|
|
unsigned int cmd, unsigned long arg)
|
|
|
|
{
|
|
|
|
struct vfio_device *device = filep->private_data;
|
2022-08-29 11:48:47 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = vfio_device_pm_runtime_get(device);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-02-24 14:20:17 +00:00
|
|
|
switch (cmd) {
|
|
|
|
case VFIO_DEVICE_FEATURE:
|
2022-08-29 11:48:47 +00:00
|
|
|
ret = vfio_ioctl_device_feature(device, (void __user *)arg);
|
|
|
|
break;
|
|
|
|
|
2022-02-24 14:20:17 +00:00
|
|
|
default:
|
|
|
|
if (unlikely(!device->ops->ioctl))
|
2022-08-29 11:48:47 +00:00
|
|
|
ret = -EINVAL;
|
|
|
|
else
|
|
|
|
ret = device->ops->ioctl(device, cmd, arg);
|
|
|
|
break;
|
2022-02-24 14:20:17 +00:00
|
|
|
}
|
2022-08-29 11:48:47 +00:00
|
|
|
|
|
|
|
vfio_device_pm_runtime_put(device);
|
|
|
|
return ret;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
|
|
|
|
size_t count, loff_t *ppos)
|
|
|
|
{
|
|
|
|
struct vfio_device *device = filep->private_data;
|
|
|
|
|
|
|
|
if (unlikely(!device->ops->read))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2021-03-30 15:53:08 +00:00
|
|
|
return device->ops->read(device, buf, count, ppos);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t vfio_device_fops_write(struct file *filep,
|
|
|
|
const char __user *buf,
|
|
|
|
size_t count, loff_t *ppos)
|
|
|
|
{
|
|
|
|
struct vfio_device *device = filep->private_data;
|
|
|
|
|
|
|
|
if (unlikely(!device->ops->write))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2021-03-30 15:53:08 +00:00
|
|
|
return device->ops->write(device, buf, count, ppos);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct vfio_device *device = filep->private_data;
|
|
|
|
|
|
|
|
if (unlikely(!device->ops->mmap))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2021-03-30 15:53:08 +00:00
|
|
|
return device->ops->mmap(device, vma);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct file_operations vfio_device_fops = {
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.release = vfio_device_fops_release,
|
|
|
|
.read = vfio_device_fops_read,
|
|
|
|
.write = vfio_device_fops_write,
|
|
|
|
.unlocked_ioctl = vfio_device_fops_unl_ioctl,
|
2018-09-11 15:23:00 +00:00
|
|
|
.compat_ioctl = compat_ptr_ioctl,
|
2012-07-31 14:16:22 +00:00
|
|
|
.mmap = vfio_device_fops_mmap,
|
|
|
|
};
|
|
|
|
|
2022-05-04 19:14:41 +00:00
|
|
|
/**
|
|
|
|
* vfio_file_iommu_group - Return the struct iommu_group for the vfio group file
|
|
|
|
* @file: VFIO group file
|
2013-08-05 16:52:36 +00:00
|
|
|
*
|
2022-05-04 19:14:41 +00:00
|
|
|
* The returned iommu_group is valid as long as a ref is held on the file.
|
2013-08-05 16:52:36 +00:00
|
|
|
*/
|
2022-05-04 19:14:41 +00:00
|
|
|
struct iommu_group *vfio_file_iommu_group(struct file *file)
|
2013-08-05 16:52:36 +00:00
|
|
|
{
|
2022-05-04 19:14:41 +00:00
|
|
|
struct vfio_group *group = file->private_data;
|
2013-08-05 16:52:36 +00:00
|
|
|
|
2022-05-04 19:14:41 +00:00
|
|
|
if (file->f_op != &vfio_group_fops)
|
|
|
|
return NULL;
|
|
|
|
return group->iommu_group;
|
2013-08-05 16:52:36 +00:00
|
|
|
}
|
2022-05-04 19:14:41 +00:00
|
|
|
EXPORT_SYMBOL_GPL(vfio_file_iommu_group);
|
2013-08-05 16:52:36 +00:00
|
|
|
|
2022-05-04 19:14:43 +00:00
|
|
|
/**
|
|
|
|
* vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
|
|
|
|
* is always CPU cache coherent
|
|
|
|
* @file: VFIO group file
|
2020-03-24 15:27:56 +00:00
|
|
|
*
|
2022-05-04 19:14:43 +00:00
|
|
|
* Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
|
|
|
|
* bit in DMA transactions. A return of false indicates that the user has
|
|
|
|
* rights to access additional instructions such as wbinvd on x86.
|
2020-03-24 15:27:56 +00:00
|
|
|
*/
|
2022-05-04 19:14:43 +00:00
|
|
|
bool vfio_file_enforced_coherent(struct file *file)
|
2020-03-24 15:27:56 +00:00
|
|
|
{
|
2022-05-04 19:14:43 +00:00
|
|
|
struct vfio_group *group = file->private_data;
|
|
|
|
bool ret;
|
2020-03-24 15:27:56 +00:00
|
|
|
|
2022-05-04 19:14:43 +00:00
|
|
|
if (file->f_op != &vfio_group_fops)
|
|
|
|
return true;
|
2020-03-24 15:27:56 +00:00
|
|
|
|
2022-05-16 23:41:20 +00:00
|
|
|
down_read(&group->group_rwsem);
|
|
|
|
if (group->container) {
|
2022-09-22 19:20:24 +00:00
|
|
|
ret = vfio_container_ioctl_check_extension(group->container,
|
|
|
|
VFIO_DMA_CC_IOMMU);
|
2022-05-16 23:41:20 +00:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Since the coherency state is determined only once a container
|
|
|
|
* is attached the user must do so before they can prove they
|
|
|
|
* have permission.
|
|
|
|
*/
|
|
|
|
ret = true;
|
2020-03-24 15:27:56 +00:00
|
|
|
}
|
2022-05-16 23:41:20 +00:00
|
|
|
up_read(&group->group_rwsem);
|
2022-05-04 19:14:43 +00:00
|
|
|
return ret;
|
2020-03-24 15:27:56 +00:00
|
|
|
}
|
2022-05-04 19:14:43 +00:00
|
|
|
EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
|
2020-03-24 15:27:56 +00:00
|
|
|
|
2022-05-04 19:14:44 +00:00
|
|
|
/**
|
|
|
|
* vfio_file_set_kvm - Link a kvm with VFIO drivers
|
|
|
|
* @file: VFIO group file
|
|
|
|
* @kvm: KVM to link
|
|
|
|
*
|
2022-05-19 18:33:11 +00:00
|
|
|
* When a VFIO device is first opened the KVM will be available in
|
|
|
|
* device->kvm if one was associated with the group.
|
2022-05-04 19:14:44 +00:00
|
|
|
*/
|
|
|
|
void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
|
2013-08-05 16:52:36 +00:00
|
|
|
{
|
2022-05-04 19:14:44 +00:00
|
|
|
struct vfio_group *group = file->private_data;
|
2013-08-05 16:52:36 +00:00
|
|
|
|
2022-05-04 19:14:44 +00:00
|
|
|
if (file->f_op != &vfio_group_fops)
|
|
|
|
return;
|
2017-06-28 19:50:05 +00:00
|
|
|
|
2022-05-16 23:41:17 +00:00
|
|
|
down_write(&group->group_rwsem);
|
2022-05-04 19:14:44 +00:00
|
|
|
group->kvm = kvm;
|
2022-05-16 23:41:17 +00:00
|
|
|
up_write(&group->group_rwsem);
|
2017-06-28 19:50:05 +00:00
|
|
|
}
|
2022-05-04 19:14:44 +00:00
|
|
|
EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
|
2017-06-28 19:50:05 +00:00
|
|
|
|
2022-05-04 19:14:46 +00:00
|
|
|
/**
|
|
|
|
* vfio_file_has_dev - True if the VFIO file is a handle for device
|
|
|
|
* @file: VFIO file to check
|
|
|
|
* @device: Device that must be part of the file
|
|
|
|
*
|
|
|
|
* Returns true if given file has permission to manipulate the given device.
|
|
|
|
*/
|
|
|
|
bool vfio_file_has_dev(struct file *file, struct vfio_device *device)
|
2013-08-05 16:52:36 +00:00
|
|
|
{
|
2022-05-04 19:14:46 +00:00
|
|
|
struct vfio_group *group = file->private_data;
|
2013-08-05 16:52:36 +00:00
|
|
|
|
2022-05-04 19:14:46 +00:00
|
|
|
if (file->f_op != &vfio_group_fops)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return group == device->group;
|
2014-02-26 18:38:39 +00:00
|
|
|
}
|
2022-05-04 19:14:46 +00:00
|
|
|
EXPORT_SYMBOL_GPL(vfio_file_has_dev);
|
2014-02-26 18:38:39 +00:00
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2016-02-22 23:02:33 +00:00
|
|
|
* Sub-module support
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* Helper for managing a buffer of info chain capabilities, allocate or
|
|
|
|
* reallocate a buffer with additional @size, filling in @id and @version
|
|
|
|
* of the capability. A pointer to the new capability is returned.
|
|
|
|
*
|
|
|
|
* NB. The chain is based at the head of the buffer, so new entries are
|
|
|
|
* added to the tail, vfio_info_cap_shift() should be called to fixup the
|
|
|
|
* next offsets prior to copying to the user buffer.
|
|
|
|
*/
|
|
|
|
struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
|
|
|
|
size_t size, u16 id, u16 version)
|
|
|
|
{
|
|
|
|
void *buf;
|
|
|
|
struct vfio_info_cap_header *header, *tmp;
|
|
|
|
|
|
|
|
buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
|
|
|
|
if (!buf) {
|
|
|
|
kfree(caps->buf);
|
2022-06-29 02:29:48 +00:00
|
|
|
caps->buf = NULL;
|
2016-02-22 23:02:33 +00:00
|
|
|
caps->size = 0;
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
}
|
|
|
|
|
|
|
|
caps->buf = buf;
|
|
|
|
header = buf + caps->size;
|
|
|
|
|
|
|
|
/* Eventually copied to user buffer, zero */
|
|
|
|
memset(header, 0, size);
|
|
|
|
|
|
|
|
header->id = id;
|
|
|
|
header->version = version;
|
|
|
|
|
|
|
|
/* Add to the end of the capability chain */
|
2016-11-21 06:21:02 +00:00
|
|
|
for (tmp = buf; tmp->next; tmp = buf + tmp->next)
|
2016-02-22 23:02:33 +00:00
|
|
|
; /* nothing */
|
|
|
|
|
|
|
|
tmp->next = caps->size;
|
|
|
|
caps->size += size;
|
|
|
|
|
|
|
|
return header;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(vfio_info_cap_add);
|
|
|
|
|
|
|
|
void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
|
|
|
|
{
|
|
|
|
struct vfio_info_cap_header *tmp;
|
2016-11-21 06:21:02 +00:00
|
|
|
void *buf = (void *)caps->buf;
|
2016-02-22 23:02:33 +00:00
|
|
|
|
2016-11-21 06:21:02 +00:00
|
|
|
for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
|
2016-02-22 23:02:33 +00:00
|
|
|
tmp->next += offset;
|
|
|
|
}
|
2016-11-16 20:46:25 +00:00
|
|
|
EXPORT_SYMBOL(vfio_info_cap_shift);
|
2016-02-22 23:02:33 +00:00
|
|
|
|
2017-12-12 19:59:39 +00:00
|
|
|
int vfio_info_add_capability(struct vfio_info_cap *caps,
|
|
|
|
struct vfio_info_cap_header *cap, size_t size)
|
2016-11-16 20:46:25 +00:00
|
|
|
{
|
|
|
|
struct vfio_info_cap_header *header;
|
|
|
|
|
2017-12-12 19:59:39 +00:00
|
|
|
header = vfio_info_cap_add(caps, size, cap->id, cap->version);
|
2016-11-16 20:46:25 +00:00
|
|
|
if (IS_ERR(header))
|
|
|
|
return PTR_ERR(header);
|
|
|
|
|
2017-12-12 19:59:39 +00:00
|
|
|
memcpy(header + 1, cap + 1, size - sizeof(*header));
|
2016-11-16 20:46:25 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(vfio_info_add_capability);
|
2016-11-16 20:46:17 +00:00
|
|
|
|
2016-11-16 20:46:27 +00:00
|
|
|
int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
|
|
|
|
int max_irq_type, size_t *data_size)
|
|
|
|
{
|
|
|
|
unsigned long minsz;
|
|
|
|
size_t size;
|
|
|
|
|
|
|
|
minsz = offsetofend(struct vfio_irq_set, count);
|
|
|
|
|
|
|
|
if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
|
|
|
|
(hdr->count >= (U32_MAX - hdr->start)) ||
|
|
|
|
(hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
|
|
|
|
VFIO_IRQ_SET_ACTION_TYPE_MASK)))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (data_size)
|
|
|
|
*data_size = 0;
|
|
|
|
|
|
|
|
if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
|
|
|
|
case VFIO_IRQ_SET_DATA_NONE:
|
|
|
|
size = 0;
|
|
|
|
break;
|
|
|
|
case VFIO_IRQ_SET_DATA_BOOL:
|
|
|
|
size = sizeof(uint8_t);
|
|
|
|
break;
|
|
|
|
case VFIO_IRQ_SET_DATA_EVENTFD:
|
|
|
|
size = sizeof(int32_t);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (size) {
|
|
|
|
if (hdr->argsz - minsz < hdr->count * size)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!data_size)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
*data_size = hdr->count * size;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
|
|
|
|
|
vfio: remove all kernel-doc notation
vfio.c abuses (misuses) "/**", which indicates the beginning of
kernel-doc notation in the kernel tree. This causes a bunch of
kernel-doc complaints about this source file, so quieten all of
them by changing all "/**" to "/*".
vfio.c:236: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* IOMMU driver registration
vfio.c:236: warning: missing initial short description on line:
* IOMMU driver registration
vfio.c:295: warning: expecting prototype for Container objects(). Prototype was for vfio_container_get() instead
vfio.c:317: warning: expecting prototype for Group objects(). Prototype was for __vfio_group_get_from_iommu() instead
vfio.c:496: warning: Function parameter or member 'device' not described in 'vfio_device_put'
vfio.c:496: warning: expecting prototype for Device objects(). Prototype was for vfio_device_put() instead
vfio.c:599: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Async device support
vfio.c:599: warning: missing initial short description on line:
* Async device support
vfio.c:693: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO driver API
vfio.c:693: warning: missing initial short description on line:
* VFIO driver API
vfio.c:835: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Get a reference to the vfio_device for a device. Even if the
vfio.c:835: warning: missing initial short description on line:
* Get a reference to the vfio_device for a device. Even if the
vfio.c:969: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO base fd, /dev/vfio/vfio
vfio.c:969: warning: missing initial short description on line:
* VFIO base fd, /dev/vfio/vfio
vfio.c:1187: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1187: warning: missing initial short description on line:
* VFIO Group fd, /dev/vfio/$GROUP
vfio.c:1540: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* VFIO Device fd
vfio.c:1540: warning: missing initial short description on line:
* VFIO Device fd
vfio.c:1615: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1615: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* External user API, exported by symbols to be linked dynamically.
vfio.c:1663: warning: missing initial short description on line:
* External user API, exported by symbols to be linked dynamically.
vfio.c:1742: warning: Function parameter or member 'caps' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'size' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'id' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: Function parameter or member 'version' not described in 'vfio_info_cap_add'
vfio.c:1742: warning: expecting prototype for Sub(). Prototype was for vfio_info_cap_add() instead
vfio.c:2276: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Module/class support
vfio.c:2276: warning: missing initial short description on line:
* Module/class support
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reported-by: kernel test robot <lkp@intel.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Eric Auger <eric.auger@redhat.com>
Cc: Cornelia Huck <cohuck@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lore.kernel.org/r/38a9cb92-a473-40bf-b8f9-85cc5cfc2da4@infradead.org
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
2021-11-10 23:19:40 +00:00
|
|
|
/*
|
2012-07-31 14:16:22 +00:00
|
|
|
* Module/class support
|
|
|
|
*/
|
|
|
|
static char *vfio_devnode(struct device *dev, umode_t *mode)
|
|
|
|
{
|
|
|
|
return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
|
|
|
|
}
|
|
|
|
|
2022-09-22 19:20:23 +00:00
|
|
|
static int __init vfio_init(void)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ida_init(&vfio.group_ida);
|
|
|
|
ida_init(&vfio.device_ida);
|
|
|
|
mutex_init(&vfio.group_lock);
|
|
|
|
INIT_LIST_HEAD(&vfio.group_list);
|
|
|
|
|
|
|
|
ret = vfio_container_init();
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2013-12-19 17:17:13 +00:00
|
|
|
/* /dev/vfio/$GROUP */
|
2012-07-31 14:16:22 +00:00
|
|
|
vfio.class = class_create(THIS_MODULE, "vfio");
|
|
|
|
if (IS_ERR(vfio.class)) {
|
|
|
|
ret = PTR_ERR(vfio.class);
|
2022-09-21 10:44:01 +00:00
|
|
|
goto err_group_class;
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
vfio.class->devnode = vfio_devnode;
|
|
|
|
|
2022-09-21 10:44:01 +00:00
|
|
|
/* /sys/class/vfio-dev/vfioX */
|
|
|
|
vfio.device_class = class_create(THIS_MODULE, "vfio-dev");
|
|
|
|
if (IS_ERR(vfio.device_class)) {
|
|
|
|
ret = PTR_ERR(vfio.device_class);
|
|
|
|
goto err_dev_class;
|
|
|
|
}
|
|
|
|
|
2019-02-12 05:59:29 +00:00
|
|
|
ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio");
|
2012-07-31 14:16:22 +00:00
|
|
|
if (ret)
|
2013-12-19 17:17:13 +00:00
|
|
|
goto err_alloc_chrdev;
|
2012-07-31 14:16:22 +00:00
|
|
|
|
2022-06-22 04:56:51 +00:00
|
|
|
pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
|
2012-07-31 14:16:22 +00:00
|
|
|
return 0;
|
|
|
|
|
2013-12-19 17:17:13 +00:00
|
|
|
err_alloc_chrdev:
|
2022-09-21 10:44:01 +00:00
|
|
|
class_destroy(vfio.device_class);
|
|
|
|
vfio.device_class = NULL;
|
|
|
|
err_dev_class:
|
2012-07-31 14:16:22 +00:00
|
|
|
class_destroy(vfio.class);
|
|
|
|
vfio.class = NULL;
|
2022-09-21 10:44:01 +00:00
|
|
|
err_group_class:
|
2022-09-22 19:20:23 +00:00
|
|
|
vfio_container_cleanup();
|
2012-07-31 14:16:22 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit vfio_cleanup(void)
|
|
|
|
{
|
|
|
|
WARN_ON(!list_empty(&vfio.group_list));
|
|
|
|
|
2022-09-21 10:44:01 +00:00
|
|
|
ida_destroy(&vfio.device_ida);
|
2021-10-15 11:40:54 +00:00
|
|
|
ida_destroy(&vfio.group_ida);
|
2019-02-12 05:59:29 +00:00
|
|
|
unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
|
2022-09-21 10:44:01 +00:00
|
|
|
class_destroy(vfio.device_class);
|
|
|
|
vfio.device_class = NULL;
|
2012-07-31 14:16:22 +00:00
|
|
|
class_destroy(vfio.class);
|
2022-09-22 19:20:23 +00:00
|
|
|
vfio_container_cleanup();
|
2012-07-31 14:16:22 +00:00
|
|
|
vfio.class = NULL;
|
2021-08-06 01:19:00 +00:00
|
|
|
xa_destroy(&vfio_device_set_xa);
|
2012-07-31 14:16:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
module_init(vfio_init);
|
|
|
|
module_exit(vfio_cleanup);
|
|
|
|
|
|
|
|
MODULE_VERSION(DRIVER_VERSION);
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
|
|
MODULE_AUTHOR(DRIVER_AUTHOR);
|
|
|
|
MODULE_DESCRIPTION(DRIVER_DESC);
|
2013-12-19 17:17:13 +00:00
|
|
|
MODULE_ALIAS_MISCDEV(VFIO_MINOR);
|
|
|
|
MODULE_ALIAS("devname:vfio/vfio");
|
2017-02-08 20:13:26 +00:00
|
|
|
MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");
|