mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 09:19:51 +00:00
![Fenghua Yu](/assets/img/avatar_default.png)
PASIDs are process-wide. It was attempted to use refcounted PASIDs to free them when the last thread drops the refcount. This turned out to be complex and error prone. Given the fact that the PASID space is 20 bits, which allows up to 1M processes to have a PASID associated concurrently, PASID resource exhaustion is not a realistic concern. Therefore, it was decided to simplify the approach and stick with lazy on demand PASID allocation, but drop the eager free approach and make an allocated PASID's lifetime bound to the lifetime of the process. Get rid of the refcounting mechanisms and replace/rename the interfaces to reflect this new approach. [ bp: Massage commit message. ] Suggested-by: Dave Hansen <dave.hansen@linux.intel.com> Signed-off-by: Fenghua Yu <fenghua.yu@intel.com> Signed-off-by: Borislav Petkov <bp@suse.de> Reviewed-by: Tony Luck <tony.luck@intel.com> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com> Reviewed-by: Jacob Pan <jacob.jun.pan@linux.intel.com> Reviewed-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Joerg Roedel <jroedel@suse.de> Link: https://lore.kernel.org/r/20220207230254.3342514-6-fenghua.yu@intel.com
422 lines
12 KiB
C
422 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* I/O Address Space ID allocator. There is one global IOASID space, split into
|
|
* subsets. Users create a subset with DECLARE_IOASID_SET, then allocate and
|
|
* free IOASIDs with ioasid_alloc() and ioasid_free().
|
|
*/
|
|
#include <linux/ioasid.h>
|
|
#include <linux/module.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/xarray.h>
|
|
|
|
struct ioasid_data {
|
|
ioasid_t id;
|
|
struct ioasid_set *set;
|
|
void *private;
|
|
struct rcu_head rcu;
|
|
};
|
|
|
|
/*
|
|
* struct ioasid_allocator_data - Internal data structure to hold information
|
|
* about an allocator. There are two types of allocators:
|
|
*
|
|
* - Default allocator always has its own XArray to track the IOASIDs allocated.
|
|
* - Custom allocators may share allocation helpers with different private data.
|
|
* Custom allocators that share the same helper functions also share the same
|
|
* XArray.
|
|
* Rules:
|
|
* 1. Default allocator is always available, not dynamically registered. This is
|
|
* to prevent race conditions with early boot code that want to register
|
|
* custom allocators or allocate IOASIDs.
|
|
* 2. Custom allocators take precedence over the default allocator.
|
|
* 3. When all custom allocators sharing the same helper functions are
|
|
* unregistered (e.g. due to hotplug), all outstanding IOASIDs must be
|
|
* freed. Otherwise, outstanding IOASIDs will be lost and orphaned.
|
|
* 4. When switching between custom allocators sharing the same helper
|
|
* functions, outstanding IOASIDs are preserved.
|
|
* 5. When switching between custom allocator and default allocator, all IOASIDs
|
|
* must be freed to ensure unadulterated space for the new allocator.
|
|
*
|
|
* @ops: allocator helper functions and its data
|
|
* @list: registered custom allocators
|
|
* @slist: allocators share the same ops but different data
|
|
* @flags: attributes of the allocator
|
|
* @xa: xarray holds the IOASID space
|
|
* @rcu: used for kfree_rcu when unregistering allocator
|
|
*/
|
|
struct ioasid_allocator_data {
|
|
struct ioasid_allocator_ops *ops;
|
|
struct list_head list;
|
|
struct list_head slist;
|
|
#define IOASID_ALLOCATOR_CUSTOM BIT(0) /* Needs framework to track results */
|
|
unsigned long flags;
|
|
struct xarray xa;
|
|
struct rcu_head rcu;
|
|
};
|
|
|
|
static DEFINE_SPINLOCK(ioasid_allocator_lock);
|
|
static LIST_HEAD(allocators_list);
|
|
|
|
static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque);
|
|
static void default_free(ioasid_t ioasid, void *opaque);
|
|
|
|
static struct ioasid_allocator_ops default_ops = {
|
|
.alloc = default_alloc,
|
|
.free = default_free,
|
|
};
|
|
|
|
static struct ioasid_allocator_data default_allocator = {
|
|
.ops = &default_ops,
|
|
.flags = 0,
|
|
.xa = XARRAY_INIT(ioasid_xa, XA_FLAGS_ALLOC),
|
|
};
|
|
|
|
static struct ioasid_allocator_data *active_allocator = &default_allocator;
|
|
|
|
static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque)
|
|
{
|
|
ioasid_t id;
|
|
|
|
if (xa_alloc(&default_allocator.xa, &id, opaque, XA_LIMIT(min, max), GFP_ATOMIC)) {
|
|
pr_err("Failed to alloc ioasid from %d to %d\n", min, max);
|
|
return INVALID_IOASID;
|
|
}
|
|
|
|
return id;
|
|
}
|
|
|
|
static void default_free(ioasid_t ioasid, void *opaque)
|
|
{
|
|
struct ioasid_data *ioasid_data;
|
|
|
|
ioasid_data = xa_erase(&default_allocator.xa, ioasid);
|
|
kfree_rcu(ioasid_data, rcu);
|
|
}
|
|
|
|
/* Allocate and initialize a new custom allocator with its helper functions */
|
|
static struct ioasid_allocator_data *ioasid_alloc_allocator(struct ioasid_allocator_ops *ops)
|
|
{
|
|
struct ioasid_allocator_data *ia_data;
|
|
|
|
ia_data = kzalloc(sizeof(*ia_data), GFP_ATOMIC);
|
|
if (!ia_data)
|
|
return NULL;
|
|
|
|
xa_init_flags(&ia_data->xa, XA_FLAGS_ALLOC);
|
|
INIT_LIST_HEAD(&ia_data->slist);
|
|
ia_data->flags |= IOASID_ALLOCATOR_CUSTOM;
|
|
ia_data->ops = ops;
|
|
|
|
/* For tracking custom allocators that share the same ops */
|
|
list_add_tail(&ops->list, &ia_data->slist);
|
|
|
|
return ia_data;
|
|
}
|
|
|
|
static bool use_same_ops(struct ioasid_allocator_ops *a, struct ioasid_allocator_ops *b)
|
|
{
|
|
return (a->free == b->free) && (a->alloc == b->alloc);
|
|
}
|
|
|
|
/**
|
|
* ioasid_register_allocator - register a custom allocator
|
|
* @ops: the custom allocator ops to be registered
|
|
*
|
|
* Custom allocators take precedence over the default xarray based allocator.
|
|
* Private data associated with the IOASID allocated by the custom allocators
|
|
* are managed by IOASID framework similar to data stored in xa by default
|
|
* allocator.
|
|
*
|
|
* There can be multiple allocators registered but only one is active. In case
|
|
* of runtime removal of a custom allocator, the next one is activated based
|
|
* on the registration ordering.
|
|
*
|
|
* Multiple allocators can share the same alloc() function, in this case the
|
|
* IOASID space is shared.
|
|
*/
|
|
int ioasid_register_allocator(struct ioasid_allocator_ops *ops)
|
|
{
|
|
struct ioasid_allocator_data *ia_data;
|
|
struct ioasid_allocator_data *pallocator;
|
|
int ret = 0;
|
|
|
|
spin_lock(&ioasid_allocator_lock);
|
|
|
|
ia_data = ioasid_alloc_allocator(ops);
|
|
if (!ia_data) {
|
|
ret = -ENOMEM;
|
|
goto out_unlock;
|
|
}
|
|
|
|
/*
|
|
* No particular preference, we activate the first one and keep
|
|
* the later registered allocators in a list in case the first one gets
|
|
* removed due to hotplug.
|
|
*/
|
|
if (list_empty(&allocators_list)) {
|
|
WARN_ON(active_allocator != &default_allocator);
|
|
/* Use this new allocator if default is not active */
|
|
if (xa_empty(&active_allocator->xa)) {
|
|
rcu_assign_pointer(active_allocator, ia_data);
|
|
list_add_tail(&ia_data->list, &allocators_list);
|
|
goto out_unlock;
|
|
}
|
|
pr_warn("Default allocator active with outstanding IOASID\n");
|
|
ret = -EAGAIN;
|
|
goto out_free;
|
|
}
|
|
|
|
/* Check if the allocator is already registered */
|
|
list_for_each_entry(pallocator, &allocators_list, list) {
|
|
if (pallocator->ops == ops) {
|
|
pr_err("IOASID allocator already registered\n");
|
|
ret = -EEXIST;
|
|
goto out_free;
|
|
} else if (use_same_ops(pallocator->ops, ops)) {
|
|
/*
|
|
* If the new allocator shares the same ops,
|
|
* then they will share the same IOASID space.
|
|
* We should put them under the same xarray.
|
|
*/
|
|
list_add_tail(&ops->list, &pallocator->slist);
|
|
goto out_free;
|
|
}
|
|
}
|
|
list_add_tail(&ia_data->list, &allocators_list);
|
|
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
return 0;
|
|
out_free:
|
|
kfree(ia_data);
|
|
out_unlock:
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(ioasid_register_allocator);
|
|
|
|
/**
|
|
* ioasid_unregister_allocator - Remove a custom IOASID allocator ops
|
|
* @ops: the custom allocator to be removed
|
|
*
|
|
* Remove an allocator from the list, activate the next allocator in
|
|
* the order it was registered. Or revert to default allocator if all
|
|
* custom allocators are unregistered without outstanding IOASIDs.
|
|
*/
|
|
void ioasid_unregister_allocator(struct ioasid_allocator_ops *ops)
|
|
{
|
|
struct ioasid_allocator_data *pallocator;
|
|
struct ioasid_allocator_ops *sops;
|
|
|
|
spin_lock(&ioasid_allocator_lock);
|
|
if (list_empty(&allocators_list)) {
|
|
pr_warn("No custom IOASID allocators active!\n");
|
|
goto exit_unlock;
|
|
}
|
|
|
|
list_for_each_entry(pallocator, &allocators_list, list) {
|
|
if (!use_same_ops(pallocator->ops, ops))
|
|
continue;
|
|
|
|
if (list_is_singular(&pallocator->slist)) {
|
|
/* No shared helper functions */
|
|
list_del(&pallocator->list);
|
|
/*
|
|
* All IOASIDs should have been freed before
|
|
* the last allocator that shares the same ops
|
|
* is unregistered.
|
|
*/
|
|
WARN_ON(!xa_empty(&pallocator->xa));
|
|
if (list_empty(&allocators_list)) {
|
|
pr_info("No custom IOASID allocators, switch to default.\n");
|
|
rcu_assign_pointer(active_allocator, &default_allocator);
|
|
} else if (pallocator == active_allocator) {
|
|
rcu_assign_pointer(active_allocator,
|
|
list_first_entry(&allocators_list,
|
|
struct ioasid_allocator_data, list));
|
|
pr_info("IOASID allocator changed");
|
|
}
|
|
kfree_rcu(pallocator, rcu);
|
|
break;
|
|
}
|
|
/*
|
|
* Find the matching shared ops to delete,
|
|
* but keep outstanding IOASIDs
|
|
*/
|
|
list_for_each_entry(sops, &pallocator->slist, list) {
|
|
if (sops == ops) {
|
|
list_del(&ops->list);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
exit_unlock:
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(ioasid_unregister_allocator);
|
|
|
|
/**
|
|
* ioasid_set_data - Set private data for an allocated ioasid
|
|
* @ioasid: the ID to set data
|
|
* @data: the private data
|
|
*
|
|
* For IOASID that is already allocated, private data can be set
|
|
* via this API. Future lookup can be done via ioasid_find.
|
|
*/
|
|
int ioasid_set_data(ioasid_t ioasid, void *data)
|
|
{
|
|
struct ioasid_data *ioasid_data;
|
|
int ret = 0;
|
|
|
|
spin_lock(&ioasid_allocator_lock);
|
|
ioasid_data = xa_load(&active_allocator->xa, ioasid);
|
|
if (ioasid_data)
|
|
rcu_assign_pointer(ioasid_data->private, data);
|
|
else
|
|
ret = -ENOENT;
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
|
|
/*
|
|
* Wait for readers to stop accessing the old private data, so the
|
|
* caller can free it.
|
|
*/
|
|
if (!ret)
|
|
synchronize_rcu();
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(ioasid_set_data);
|
|
|
|
/**
|
|
* ioasid_alloc - Allocate an IOASID
|
|
* @set: the IOASID set
|
|
* @min: the minimum ID (inclusive)
|
|
* @max: the maximum ID (inclusive)
|
|
* @private: data private to the caller
|
|
*
|
|
* Allocate an ID between @min and @max. The @private pointer is stored
|
|
* internally and can be retrieved with ioasid_find().
|
|
*
|
|
* Return: the allocated ID on success, or %INVALID_IOASID on failure.
|
|
*/
|
|
ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, ioasid_t max,
|
|
void *private)
|
|
{
|
|
struct ioasid_data *data;
|
|
void *adata;
|
|
ioasid_t id;
|
|
|
|
data = kzalloc(sizeof(*data), GFP_ATOMIC);
|
|
if (!data)
|
|
return INVALID_IOASID;
|
|
|
|
data->set = set;
|
|
data->private = private;
|
|
|
|
/*
|
|
* Custom allocator needs allocator data to perform platform specific
|
|
* operations.
|
|
*/
|
|
spin_lock(&ioasid_allocator_lock);
|
|
adata = active_allocator->flags & IOASID_ALLOCATOR_CUSTOM ? active_allocator->ops->pdata : data;
|
|
id = active_allocator->ops->alloc(min, max, adata);
|
|
if (id == INVALID_IOASID) {
|
|
pr_err("Failed ASID allocation %lu\n", active_allocator->flags);
|
|
goto exit_free;
|
|
}
|
|
|
|
if ((active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) &&
|
|
xa_alloc(&active_allocator->xa, &id, data, XA_LIMIT(id, id), GFP_ATOMIC)) {
|
|
/* Custom allocator needs framework to store and track allocation results */
|
|
pr_err("Failed to alloc ioasid from %d\n", id);
|
|
active_allocator->ops->free(id, active_allocator->ops->pdata);
|
|
goto exit_free;
|
|
}
|
|
data->id = id;
|
|
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
return id;
|
|
exit_free:
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
kfree(data);
|
|
return INVALID_IOASID;
|
|
}
|
|
EXPORT_SYMBOL_GPL(ioasid_alloc);
|
|
|
|
/**
|
|
* ioasid_free - Free an ioasid
|
|
* @ioasid: the ID to remove
|
|
*/
|
|
void ioasid_free(ioasid_t ioasid)
|
|
{
|
|
struct ioasid_data *ioasid_data;
|
|
|
|
spin_lock(&ioasid_allocator_lock);
|
|
ioasid_data = xa_load(&active_allocator->xa, ioasid);
|
|
if (!ioasid_data) {
|
|
pr_err("Trying to free unknown IOASID %u\n", ioasid);
|
|
goto exit_unlock;
|
|
}
|
|
|
|
active_allocator->ops->free(ioasid, active_allocator->ops->pdata);
|
|
/* Custom allocator needs additional steps to free the xa element */
|
|
if (active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) {
|
|
ioasid_data = xa_erase(&active_allocator->xa, ioasid);
|
|
kfree_rcu(ioasid_data, rcu);
|
|
}
|
|
|
|
exit_unlock:
|
|
spin_unlock(&ioasid_allocator_lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(ioasid_free);
|
|
|
|
/**
|
|
* ioasid_find - Find IOASID data
|
|
* @set: the IOASID set
|
|
* @ioasid: the IOASID to find
|
|
* @getter: function to call on the found object
|
|
*
|
|
* The optional getter function allows to take a reference to the found object
|
|
* under the rcu lock. The function can also check if the object is still valid:
|
|
* if @getter returns false, then the object is invalid and NULL is returned.
|
|
*
|
|
* If the IOASID exists, return the private pointer passed to ioasid_alloc.
|
|
* Private data can be NULL if not set. Return an error if the IOASID is not
|
|
* found, or if @set is not NULL and the IOASID does not belong to the set.
|
|
*/
|
|
void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid,
|
|
bool (*getter)(void *))
|
|
{
|
|
void *priv;
|
|
struct ioasid_data *ioasid_data;
|
|
struct ioasid_allocator_data *idata;
|
|
|
|
rcu_read_lock();
|
|
idata = rcu_dereference(active_allocator);
|
|
ioasid_data = xa_load(&idata->xa, ioasid);
|
|
if (!ioasid_data) {
|
|
priv = ERR_PTR(-ENOENT);
|
|
goto unlock;
|
|
}
|
|
if (set && ioasid_data->set != set) {
|
|
/* data found but does not belong to the set */
|
|
priv = ERR_PTR(-EACCES);
|
|
goto unlock;
|
|
}
|
|
/* Now IOASID and its set is verified, we can return the private data */
|
|
priv = rcu_dereference(ioasid_data->private);
|
|
if (getter && !getter(priv))
|
|
priv = NULL;
|
|
unlock:
|
|
rcu_read_unlock();
|
|
|
|
return priv;
|
|
}
|
|
EXPORT_SYMBOL_GPL(ioasid_find);
|
|
|
|
MODULE_AUTHOR("Jean-Philippe Brucker <jean-philippe.brucker@arm.com>");
|
|
MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
|
|
MODULE_DESCRIPTION("IO Address Space ID (IOASID) allocator");
|
|
MODULE_LICENSE("GPL");
|