mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-05 00:20:32 +00:00
vfio/type1: block on invalid vaddr
Block translation of host virtual address while an iova range has an invalid vaddr. Signed-off-by: Steve Sistare <steven.sistare@oracle.com> Reviewed-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
This commit is contained in:
parent
487ace1340
commit
898b9eaeb3
1 changed files with 90 additions and 5 deletions
|
@ -31,6 +31,7 @@
|
||||||
#include <linux/rbtree.h>
|
#include <linux/rbtree.h>
|
||||||
#include <linux/sched/signal.h>
|
#include <linux/sched/signal.h>
|
||||||
#include <linux/sched/mm.h>
|
#include <linux/sched/mm.h>
|
||||||
|
#include <linux/kthread.h>
|
||||||
#include <linux/slab.h>
|
#include <linux/slab.h>
|
||||||
#include <linux/uaccess.h>
|
#include <linux/uaccess.h>
|
||||||
#include <linux/vfio.h>
|
#include <linux/vfio.h>
|
||||||
|
@ -71,6 +72,7 @@ struct vfio_iommu {
|
||||||
unsigned int dma_avail;
|
unsigned int dma_avail;
|
||||||
unsigned int vaddr_invalid_count;
|
unsigned int vaddr_invalid_count;
|
||||||
uint64_t pgsize_bitmap;
|
uint64_t pgsize_bitmap;
|
||||||
|
wait_queue_head_t vaddr_wait;
|
||||||
bool v2;
|
bool v2;
|
||||||
bool nesting;
|
bool nesting;
|
||||||
bool dirty_page_tracking;
|
bool dirty_page_tracking;
|
||||||
|
@ -146,6 +148,8 @@ struct vfio_regions {
|
||||||
#define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX)
|
#define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX)
|
||||||
#define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
|
#define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
|
||||||
|
|
||||||
|
#define WAITED 1
|
||||||
|
|
||||||
static int put_pfn(unsigned long pfn, int prot);
|
static int put_pfn(unsigned long pfn, int prot);
|
||||||
|
|
||||||
static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
|
static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
|
||||||
|
@ -507,6 +511,61 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int vfio_wait(struct vfio_iommu *iommu)
|
||||||
|
{
|
||||||
|
DEFINE_WAIT(wait);
|
||||||
|
|
||||||
|
prepare_to_wait(&iommu->vaddr_wait, &wait, TASK_KILLABLE);
|
||||||
|
mutex_unlock(&iommu->lock);
|
||||||
|
schedule();
|
||||||
|
mutex_lock(&iommu->lock);
|
||||||
|
finish_wait(&iommu->vaddr_wait, &wait);
|
||||||
|
if (kthread_should_stop() || !iommu->container_open ||
|
||||||
|
fatal_signal_pending(current)) {
|
||||||
|
return -EFAULT;
|
||||||
|
}
|
||||||
|
return WAITED;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Find dma struct and wait for its vaddr to be valid. iommu lock is dropped
|
||||||
|
* if the task waits, but is re-locked on return. Return result in *dma_p.
|
||||||
|
* Return 0 on success with no waiting, WAITED on success if waited, and -errno
|
||||||
|
* on error.
|
||||||
|
*/
|
||||||
|
static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start,
|
||||||
|
size_t size, struct vfio_dma **dma_p)
|
||||||
|
{
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
do {
|
||||||
|
*dma_p = vfio_find_dma(iommu, start, size);
|
||||||
|
if (!*dma_p)
|
||||||
|
ret = -EINVAL;
|
||||||
|
else if (!(*dma_p)->vaddr_invalid)
|
||||||
|
ret = 0;
|
||||||
|
else
|
||||||
|
ret = vfio_wait(iommu);
|
||||||
|
} while (ret > 0);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Wait for all vaddr in the dma_list to become valid. iommu lock is dropped
|
||||||
|
* if the task waits, but is re-locked on return. Return 0 on success with no
|
||||||
|
* waiting, WAITED on success if waited, and -errno on error.
|
||||||
|
*/
|
||||||
|
static int vfio_wait_all_valid(struct vfio_iommu *iommu)
|
||||||
|
{
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
while (iommu->vaddr_invalid_count && ret >= 0)
|
||||||
|
ret = vfio_wait(iommu);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Attempt to pin pages. We really don't want to track all the pfns and
|
* Attempt to pin pages. We really don't want to track all the pfns and
|
||||||
* the iommu can only map chunks of consecutive pfns anyway, so get the
|
* the iommu can only map chunks of consecutive pfns anyway, so get the
|
||||||
|
@ -668,6 +727,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
|
||||||
unsigned long remote_vaddr;
|
unsigned long remote_vaddr;
|
||||||
struct vfio_dma *dma;
|
struct vfio_dma *dma;
|
||||||
bool do_accounting;
|
bool do_accounting;
|
||||||
|
dma_addr_t iova;
|
||||||
|
|
||||||
if (!iommu || !user_pfn || !phys_pfn)
|
if (!iommu || !user_pfn || !phys_pfn)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
@ -678,6 +738,22 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
|
||||||
|
|
||||||
mutex_lock(&iommu->lock);
|
mutex_lock(&iommu->lock);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Wait for all necessary vaddr's to be valid so they can be used in
|
||||||
|
* the main loop without dropping the lock, to avoid racing vs unmap.
|
||||||
|
*/
|
||||||
|
again:
|
||||||
|
if (iommu->vaddr_invalid_count) {
|
||||||
|
for (i = 0; i < npage; i++) {
|
||||||
|
iova = user_pfn[i] << PAGE_SHIFT;
|
||||||
|
ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma);
|
||||||
|
if (ret < 0)
|
||||||
|
goto pin_done;
|
||||||
|
if (ret == WAITED)
|
||||||
|
goto again;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Fail if notifier list is empty */
|
/* Fail if notifier list is empty */
|
||||||
if (!iommu->notifier.head) {
|
if (!iommu->notifier.head) {
|
||||||
ret = -EINVAL;
|
ret = -EINVAL;
|
||||||
|
@ -692,7 +768,6 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
|
||||||
do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
|
do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
|
||||||
|
|
||||||
for (i = 0; i < npage; i++) {
|
for (i = 0; i < npage; i++) {
|
||||||
dma_addr_t iova;
|
|
||||||
struct vfio_pfn *vpfn;
|
struct vfio_pfn *vpfn;
|
||||||
|
|
||||||
iova = user_pfn[i] << PAGE_SHIFT;
|
iova = user_pfn[i] << PAGE_SHIFT;
|
||||||
|
@ -977,8 +1052,10 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
|
||||||
vfio_unlink_dma(iommu, dma);
|
vfio_unlink_dma(iommu, dma);
|
||||||
put_task_struct(dma->task);
|
put_task_struct(dma->task);
|
||||||
vfio_dma_bitmap_free(dma);
|
vfio_dma_bitmap_free(dma);
|
||||||
if (dma->vaddr_invalid)
|
if (dma->vaddr_invalid) {
|
||||||
iommu->vaddr_invalid_count--;
|
iommu->vaddr_invalid_count--;
|
||||||
|
wake_up_all(&iommu->vaddr_wait);
|
||||||
|
}
|
||||||
kfree(dma);
|
kfree(dma);
|
||||||
iommu->dma_avail++;
|
iommu->dma_avail++;
|
||||||
}
|
}
|
||||||
|
@ -1406,6 +1483,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
|
||||||
dma->vaddr = vaddr;
|
dma->vaddr = vaddr;
|
||||||
dma->vaddr_invalid = false;
|
dma->vaddr_invalid = false;
|
||||||
iommu->vaddr_invalid_count--;
|
iommu->vaddr_invalid_count--;
|
||||||
|
wake_up_all(&iommu->vaddr_wait);
|
||||||
}
|
}
|
||||||
goto out_unlock;
|
goto out_unlock;
|
||||||
} else if (dma) {
|
} else if (dma) {
|
||||||
|
@ -1505,6 +1583,10 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
|
||||||
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
ret = vfio_wait_all_valid(iommu);
|
||||||
|
if (ret < 0)
|
||||||
|
return ret;
|
||||||
|
|
||||||
/* Arbitrarily pick the first domain in the list for lookups */
|
/* Arbitrarily pick the first domain in the list for lookups */
|
||||||
if (!list_empty(&iommu->domain_list))
|
if (!list_empty(&iommu->domain_list))
|
||||||
d = list_first_entry(&iommu->domain_list,
|
d = list_first_entry(&iommu->domain_list,
|
||||||
|
@ -2524,6 +2606,7 @@ static void *vfio_iommu_type1_open(unsigned long arg)
|
||||||
iommu->container_open = true;
|
iommu->container_open = true;
|
||||||
mutex_init(&iommu->lock);
|
mutex_init(&iommu->lock);
|
||||||
BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
|
BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
|
||||||
|
init_waitqueue_head(&iommu->vaddr_wait);
|
||||||
|
|
||||||
return iommu;
|
return iommu;
|
||||||
}
|
}
|
||||||
|
@ -2992,12 +3075,13 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,
|
||||||
struct vfio_dma *dma;
|
struct vfio_dma *dma;
|
||||||
bool kthread = current->mm == NULL;
|
bool kthread = current->mm == NULL;
|
||||||
size_t offset;
|
size_t offset;
|
||||||
|
int ret;
|
||||||
|
|
||||||
*copied = 0;
|
*copied = 0;
|
||||||
|
|
||||||
dma = vfio_find_dma(iommu, user_iova, 1);
|
ret = vfio_find_dma_valid(iommu, user_iova, 1, &dma);
|
||||||
if (!dma)
|
if (ret < 0)
|
||||||
return -EINVAL;
|
return ret;
|
||||||
|
|
||||||
if ((write && !(dma->prot & IOMMU_WRITE)) ||
|
if ((write && !(dma->prot & IOMMU_WRITE)) ||
|
||||||
!(dma->prot & IOMMU_READ))
|
!(dma->prot & IOMMU_READ))
|
||||||
|
@ -3099,6 +3183,7 @@ static void vfio_iommu_type1_notify(void *iommu_data,
|
||||||
mutex_lock(&iommu->lock);
|
mutex_lock(&iommu->lock);
|
||||||
iommu->container_open = false;
|
iommu->container_open = false;
|
||||||
mutex_unlock(&iommu->lock);
|
mutex_unlock(&iommu->lock);
|
||||||
|
wake_up_all(&iommu->vaddr_wait);
|
||||||
}
|
}
|
||||||
|
|
||||||
static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
|
static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
|
||||||
|
|
Loading…
Reference in a new issue