mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 19:29:37 +00:00
![Lu Baolu](/assets/img/avatar_default.png)
Tweak the I/O page fault handling framework to route the page faults to the domain and call the page fault handler retrieved from the domain. This makes the I/O page fault handling framework possible to serve more usage scenarios as long as they have an IOMMU domain and install a page fault handler in it. Some unused functions are also removed to avoid dead code. The iommu_get_domain_for_dev_pasid() which retrieves attached domain for a {device, PASID} pair is used. It will be used by the page fault handling framework which knows {device, PASID} reported from the iommu driver. We have a guarantee that the SVA domain doesn't go away during IOPF handling, because unbind() won't free the domain until all the pending page requests have been flushed from the pipeline. The drivers either call iopf_queue_flush_dev() explicitly, or in stall case, the device driver is required to flush all DMAs including stalled transactions before calling unbind(). This also renames iopf_handle_group() to iopf_handler() to avoid confusing. Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Reviewed-by: Jean-Philippe Brucker <jean-philippe@linaro.org> Reviewed-by: Kevin Tian <kevin.tian@intel.com> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Tested-by: Zhangfei Gao <zhangfei.gao@linaro.org> Tested-by: Tony Zhu <tony.zhu@intel.com> Link: https://lore.kernel.org/r/20221031005917.45690-13-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel <jroedel@suse.de>
418 lines
11 KiB
C
418 lines
11 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Handle device page faults
|
|
*
|
|
* Copyright (C) 2020 ARM Ltd.
|
|
*/
|
|
|
|
#include <linux/iommu.h>
|
|
#include <linux/list.h>
|
|
#include <linux/sched/mm.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/workqueue.h>
|
|
|
|
#include "iommu-sva-lib.h"
|
|
|
|
/**
|
|
* struct iopf_queue - IO Page Fault queue
|
|
* @wq: the fault workqueue
|
|
* @devices: devices attached to this queue
|
|
* @lock: protects the device list
|
|
*/
|
|
struct iopf_queue {
|
|
struct workqueue_struct *wq;
|
|
struct list_head devices;
|
|
struct mutex lock;
|
|
};
|
|
|
|
/**
|
|
* struct iopf_device_param - IO Page Fault data attached to a device
|
|
* @dev: the device that owns this param
|
|
* @queue: IOPF queue
|
|
* @queue_list: index into queue->devices
|
|
* @partial: faults that are part of a Page Request Group for which the last
|
|
* request hasn't been submitted yet.
|
|
*/
|
|
struct iopf_device_param {
|
|
struct device *dev;
|
|
struct iopf_queue *queue;
|
|
struct list_head queue_list;
|
|
struct list_head partial;
|
|
};
|
|
|
|
struct iopf_fault {
|
|
struct iommu_fault fault;
|
|
struct list_head list;
|
|
};
|
|
|
|
struct iopf_group {
|
|
struct iopf_fault last_fault;
|
|
struct list_head faults;
|
|
struct work_struct work;
|
|
struct device *dev;
|
|
};
|
|
|
|
static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf,
|
|
enum iommu_page_response_code status)
|
|
{
|
|
struct iommu_page_response resp = {
|
|
.version = IOMMU_PAGE_RESP_VERSION_1,
|
|
.pasid = iopf->fault.prm.pasid,
|
|
.grpid = iopf->fault.prm.grpid,
|
|
.code = status,
|
|
};
|
|
|
|
if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) &&
|
|
(iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID))
|
|
resp.flags = IOMMU_PAGE_RESP_PASID_VALID;
|
|
|
|
return iommu_page_response(dev, &resp);
|
|
}
|
|
|
|
static void iopf_handler(struct work_struct *work)
|
|
{
|
|
struct iopf_group *group;
|
|
struct iommu_domain *domain;
|
|
struct iopf_fault *iopf, *next;
|
|
enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS;
|
|
|
|
group = container_of(work, struct iopf_group, work);
|
|
domain = iommu_get_domain_for_dev_pasid(group->dev,
|
|
group->last_fault.fault.prm.pasid, 0);
|
|
if (!domain || !domain->iopf_handler)
|
|
status = IOMMU_PAGE_RESP_INVALID;
|
|
|
|
list_for_each_entry_safe(iopf, next, &group->faults, list) {
|
|
/*
|
|
* For the moment, errors are sticky: don't handle subsequent
|
|
* faults in the group if there is an error.
|
|
*/
|
|
if (status == IOMMU_PAGE_RESP_SUCCESS)
|
|
status = domain->iopf_handler(&iopf->fault,
|
|
domain->fault_data);
|
|
|
|
if (!(iopf->fault.prm.flags &
|
|
IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE))
|
|
kfree(iopf);
|
|
}
|
|
|
|
iopf_complete_group(group->dev, &group->last_fault, status);
|
|
kfree(group);
|
|
}
|
|
|
|
/**
|
|
* iommu_queue_iopf - IO Page Fault handler
|
|
* @fault: fault event
|
|
* @cookie: struct device, passed to iommu_register_device_fault_handler.
|
|
*
|
|
* Add a fault to the device workqueue, to be handled by mm.
|
|
*
|
|
* This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard
|
|
* them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't
|
|
* expect a response. It may be generated when disabling a PASID (issuing a
|
|
* PASID stop request) by some PCI devices.
|
|
*
|
|
* The PASID stop request is issued by the device driver before unbind(). Once
|
|
* it completes, no page request is generated for this PASID anymore and
|
|
* outstanding ones have been pushed to the IOMMU (as per PCIe 4.0r1.0 - 6.20.1
|
|
* and 10.4.1.2 - Managing PASID TLP Prefix Usage). Some PCI devices will wait
|
|
* for all outstanding page requests to come back with a response before
|
|
* completing the PASID stop request. Others do not wait for page responses, and
|
|
* instead issue this Stop Marker that tells us when the PASID can be
|
|
* reallocated.
|
|
*
|
|
* It is safe to discard the Stop Marker because it is an optimization.
|
|
* a. Page requests, which are posted requests, have been flushed to the IOMMU
|
|
* when the stop request completes.
|
|
* b. The IOMMU driver flushes all fault queues on unbind() before freeing the
|
|
* PASID.
|
|
*
|
|
* So even though the Stop Marker might be issued by the device *after* the stop
|
|
* request completes, outstanding faults will have been dealt with by the time
|
|
* the PASID is freed.
|
|
*
|
|
* Any valid page fault will be eventually routed to an iommu domain and the
|
|
* page fault handler installed there will get called. The users of this
|
|
* handling framework should guarantee that the iommu domain could only be
|
|
* freed after the device has stopped generating page faults (or the iommu
|
|
* hardware has been set to block the page faults) and the pending page faults
|
|
* have been flushed.
|
|
*
|
|
* Return: 0 on success and <0 on error.
|
|
*/
|
|
int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
|
|
{
|
|
int ret;
|
|
struct iopf_group *group;
|
|
struct iopf_fault *iopf, *next;
|
|
struct iopf_device_param *iopf_param;
|
|
|
|
struct device *dev = cookie;
|
|
struct dev_iommu *param = dev->iommu;
|
|
|
|
lockdep_assert_held(¶m->lock);
|
|
|
|
if (fault->type != IOMMU_FAULT_PAGE_REQ)
|
|
/* Not a recoverable page fault */
|
|
return -EOPNOTSUPP;
|
|
|
|
/*
|
|
* As long as we're holding param->lock, the queue can't be unlinked
|
|
* from the device and therefore cannot disappear.
|
|
*/
|
|
iopf_param = param->iopf_param;
|
|
if (!iopf_param)
|
|
return -ENODEV;
|
|
|
|
if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
|
|
iopf = kzalloc(sizeof(*iopf), GFP_KERNEL);
|
|
if (!iopf)
|
|
return -ENOMEM;
|
|
|
|
iopf->fault = *fault;
|
|
|
|
/* Non-last request of a group. Postpone until the last one */
|
|
list_add(&iopf->list, &iopf_param->partial);
|
|
|
|
return 0;
|
|
}
|
|
|
|
group = kzalloc(sizeof(*group), GFP_KERNEL);
|
|
if (!group) {
|
|
/*
|
|
* The caller will send a response to the hardware. But we do
|
|
* need to clean up before leaving, otherwise partial faults
|
|
* will be stuck.
|
|
*/
|
|
ret = -ENOMEM;
|
|
goto cleanup_partial;
|
|
}
|
|
|
|
group->dev = dev;
|
|
group->last_fault.fault = *fault;
|
|
INIT_LIST_HEAD(&group->faults);
|
|
list_add(&group->last_fault.list, &group->faults);
|
|
INIT_WORK(&group->work, iopf_handler);
|
|
|
|
/* See if we have partial faults for this group */
|
|
list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {
|
|
if (iopf->fault.prm.grpid == fault->prm.grpid)
|
|
/* Insert *before* the last fault */
|
|
list_move(&iopf->list, &group->faults);
|
|
}
|
|
|
|
queue_work(iopf_param->queue->wq, &group->work);
|
|
return 0;
|
|
|
|
cleanup_partial:
|
|
list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {
|
|
if (iopf->fault.prm.grpid == fault->prm.grpid) {
|
|
list_del(&iopf->list);
|
|
kfree(iopf);
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(iommu_queue_iopf);
|
|
|
|
/**
|
|
* iopf_queue_flush_dev - Ensure that all queued faults have been processed
|
|
* @dev: the endpoint whose faults need to be flushed.
|
|
*
|
|
* The IOMMU driver calls this before releasing a PASID, to ensure that all
|
|
* pending faults for this PASID have been handled, and won't hit the address
|
|
* space of the next process that uses this PASID. The driver must make sure
|
|
* that no new fault is added to the queue. In particular it must flush its
|
|
* low-level queue before calling this function.
|
|
*
|
|
* Return: 0 on success and <0 on error.
|
|
*/
|
|
int iopf_queue_flush_dev(struct device *dev)
|
|
{
|
|
int ret = 0;
|
|
struct iopf_device_param *iopf_param;
|
|
struct dev_iommu *param = dev->iommu;
|
|
|
|
if (!param)
|
|
return -ENODEV;
|
|
|
|
mutex_lock(¶m->lock);
|
|
iopf_param = param->iopf_param;
|
|
if (iopf_param)
|
|
flush_workqueue(iopf_param->queue->wq);
|
|
else
|
|
ret = -ENODEV;
|
|
mutex_unlock(¶m->lock);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(iopf_queue_flush_dev);
|
|
|
|
/**
|
|
* iopf_queue_discard_partial - Remove all pending partial fault
|
|
* @queue: the queue whose partial faults need to be discarded
|
|
*
|
|
* When the hardware queue overflows, last page faults in a group may have been
|
|
* lost and the IOMMU driver calls this to discard all partial faults. The
|
|
* driver shouldn't be adding new faults to this queue concurrently.
|
|
*
|
|
* Return: 0 on success and <0 on error.
|
|
*/
|
|
int iopf_queue_discard_partial(struct iopf_queue *queue)
|
|
{
|
|
struct iopf_fault *iopf, *next;
|
|
struct iopf_device_param *iopf_param;
|
|
|
|
if (!queue)
|
|
return -EINVAL;
|
|
|
|
mutex_lock(&queue->lock);
|
|
list_for_each_entry(iopf_param, &queue->devices, queue_list) {
|
|
list_for_each_entry_safe(iopf, next, &iopf_param->partial,
|
|
list) {
|
|
list_del(&iopf->list);
|
|
kfree(iopf);
|
|
}
|
|
}
|
|
mutex_unlock(&queue->lock);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(iopf_queue_discard_partial);
|
|
|
|
/**
|
|
* iopf_queue_add_device - Add producer to the fault queue
|
|
* @queue: IOPF queue
|
|
* @dev: device to add
|
|
*
|
|
* Return: 0 on success and <0 on error.
|
|
*/
|
|
int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev)
|
|
{
|
|
int ret = -EBUSY;
|
|
struct iopf_device_param *iopf_param;
|
|
struct dev_iommu *param = dev->iommu;
|
|
|
|
if (!param)
|
|
return -ENODEV;
|
|
|
|
iopf_param = kzalloc(sizeof(*iopf_param), GFP_KERNEL);
|
|
if (!iopf_param)
|
|
return -ENOMEM;
|
|
|
|
INIT_LIST_HEAD(&iopf_param->partial);
|
|
iopf_param->queue = queue;
|
|
iopf_param->dev = dev;
|
|
|
|
mutex_lock(&queue->lock);
|
|
mutex_lock(¶m->lock);
|
|
if (!param->iopf_param) {
|
|
list_add(&iopf_param->queue_list, &queue->devices);
|
|
param->iopf_param = iopf_param;
|
|
ret = 0;
|
|
}
|
|
mutex_unlock(¶m->lock);
|
|
mutex_unlock(&queue->lock);
|
|
|
|
if (ret)
|
|
kfree(iopf_param);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(iopf_queue_add_device);
|
|
|
|
/**
|
|
* iopf_queue_remove_device - Remove producer from fault queue
|
|
* @queue: IOPF queue
|
|
* @dev: device to remove
|
|
*
|
|
* Caller makes sure that no more faults are reported for this device.
|
|
*
|
|
* Return: 0 on success and <0 on error.
|
|
*/
|
|
int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev)
|
|
{
|
|
int ret = -EINVAL;
|
|
struct iopf_fault *iopf, *next;
|
|
struct iopf_device_param *iopf_param;
|
|
struct dev_iommu *param = dev->iommu;
|
|
|
|
if (!param || !queue)
|
|
return -EINVAL;
|
|
|
|
mutex_lock(&queue->lock);
|
|
mutex_lock(¶m->lock);
|
|
iopf_param = param->iopf_param;
|
|
if (iopf_param && iopf_param->queue == queue) {
|
|
list_del(&iopf_param->queue_list);
|
|
param->iopf_param = NULL;
|
|
ret = 0;
|
|
}
|
|
mutex_unlock(¶m->lock);
|
|
mutex_unlock(&queue->lock);
|
|
if (ret)
|
|
return ret;
|
|
|
|
/* Just in case some faults are still stuck */
|
|
list_for_each_entry_safe(iopf, next, &iopf_param->partial, list)
|
|
kfree(iopf);
|
|
|
|
kfree(iopf_param);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(iopf_queue_remove_device);
|
|
|
|
/**
|
|
* iopf_queue_alloc - Allocate and initialize a fault queue
|
|
* @name: a unique string identifying the queue (for workqueue)
|
|
*
|
|
* Return: the queue on success and NULL on error.
|
|
*/
|
|
struct iopf_queue *iopf_queue_alloc(const char *name)
|
|
{
|
|
struct iopf_queue *queue;
|
|
|
|
queue = kzalloc(sizeof(*queue), GFP_KERNEL);
|
|
if (!queue)
|
|
return NULL;
|
|
|
|
/*
|
|
* The WQ is unordered because the low-level handler enqueues faults by
|
|
* group. PRI requests within a group have to be ordered, but once
|
|
* that's dealt with, the high-level function can handle groups out of
|
|
* order.
|
|
*/
|
|
queue->wq = alloc_workqueue("iopf_queue/%s", WQ_UNBOUND, 0, name);
|
|
if (!queue->wq) {
|
|
kfree(queue);
|
|
return NULL;
|
|
}
|
|
|
|
INIT_LIST_HEAD(&queue->devices);
|
|
mutex_init(&queue->lock);
|
|
|
|
return queue;
|
|
}
|
|
EXPORT_SYMBOL_GPL(iopf_queue_alloc);
|
|
|
|
/**
|
|
* iopf_queue_free - Free IOPF queue
|
|
* @queue: queue to free
|
|
*
|
|
* Counterpart to iopf_queue_alloc(). The driver must not be queuing faults or
|
|
* adding/removing devices on this queue anymore.
|
|
*/
|
|
void iopf_queue_free(struct iopf_queue *queue)
|
|
{
|
|
struct iopf_device_param *iopf_param, *next;
|
|
|
|
if (!queue)
|
|
return;
|
|
|
|
list_for_each_entry_safe(iopf_param, next, &queue->devices, queue_list)
|
|
iopf_queue_remove_device(queue, iopf_param->dev);
|
|
|
|
destroy_workqueue(queue->wq);
|
|
kfree(queue);
|
|
}
|
|
EXPORT_SYMBOL_GPL(iopf_queue_free);
|