// SPDX-License-Identifier: GPL-2.0
/*
* I/O Address Space ID allocator. There is one global IOASID space, split into
* subsets. Users create a subset with DECLARE_IOASID_SET, then allocate and
* free IOASIDs with ioasid_alloc() and ioasid_free().
*/
#include <linux/ioasid.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/xarray.h>
struct ioasid_data {
ioasid_t id;
struct ioasid_set *set;
void *private;
struct rcu_head rcu;
};
/*
* struct ioasid_allocator_data - Internal data structure to hold information
* about an allocator. There are two types of allocators:
*
* - Default allocator always has its own XArray to track the IOASIDs allocated.
* - Custom allocators may share allocation helpers with different private data.
* Custom allocators that share the same helper functions also share the same
* XArray.
* Rules:
* 1. Default allocator is always available, not dynamically registered. This is
* to prevent race conditions with early boot code that want to register
* custom allocators or allocate IOASIDs.
* 2. Custom allocators take precedence over the default allocator.
* 3. When all custom allocators sharing the same helper functions are
* unregistered (e.g. due to hotplug), all outstanding IOASIDs must be
* freed. Otherwise, outstanding IOASIDs will be lost and orphaned.
* 4. When switching between custom allocators sharing the same helper
* functions, outstanding IOASIDs are preserved.
* 5. When switching between custom allocator and default allocator, all IOASIDs
* must be freed to ensure unadulterated space for the new allocator.
*
* @ops: allocator helper functions and its data
* @list: registered custom allocators
* @slist: allocators share the same ops but different data
* @flags: attributes of the allocator
* @xa: xarray holds the IOASID space
* @rcu: used for kfree_rcu when unregistering allocator
*/
struct ioasid_allocator_data {
struct ioasid_allocator_ops *ops;
struct list_head list;
struct list_head slist;
#define IOASID_ALLOCATOR_CUSTOM BIT(0) /* Needs framework to track results */
unsigned long flags;
struct xarray xa;
struct rcu_head rcu;
};
static DEFINE_SPINLOCK(ioasid_allocator_lock);
static LIST_HEAD(allocators_list);
static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque);
static void default_free(ioasid_t ioasid, void *opaque);
static struct ioasid_allocator_ops default_ops = {
.alloc = default_alloc,
.free = default_free,
};
static struct ioasid_allocator_data default_allocator = {
.ops = &default_ops,
.flags = 0,
.xa = XARRAY_INIT(ioasid_xa, XA_FLAGS_ALLOC),
};
static struct ioasid_allocator_data *active_allocator = &default_allocator;
static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque)
{
ioasid_t id;
if (xa_alloc(&default_allocator.xa, &id, opaque, XA_LIMIT(min, max), GFP_ATOMIC)) {
pr_err("Failed to alloc ioasid from %d to %d\n", min, max);
return INVALID_IOASID;
}
return id;
}
static void default_free(ioasid_t ioasid, void *opaque)
{
struct ioasid_data *ioasid_data;
ioasid_data = xa_erase(&default_allocator.xa, ioasid);
kfree_rcu(ioasid_data, rcu);
}
/* Allocate and initialize a new custom allocator with its helper functions */
static struct ioasid_allocator_data *ioasid_alloc_allocator(struct ioasid_allocator_ops *ops)
{
struct ioasid_allocator_data *ia_data;
ia_data = kzalloc(sizeof(*ia_data), GFP_ATOMIC);
if (!ia_data)
return NULL;
xa_init_flags(&ia_data->xa, XA_FLAGS_ALLOC);
INIT_LIST_HEAD(&ia_data->slist);
ia_data->flags |= IOASID_ALLOCATOR_CUSTOM;
ia_data->ops = ops;
/* For tracking custom allocators that share the same ops */
list_add_tail(&ops->list, &ia_data->slist);
return ia_data;
}
static bool use_same_ops(struct ioasid_allocator_ops *a, struct ioasid_allocator_ops *b)
{
return (a->free == b->free) && (a->alloc == b->alloc);
}
/**
* ioasid_register_allocator - register a custom allocator
* @ops: the custom allocator ops to be registered
*
* Custom allocators take precedence over the default xarray based allocator.
* Private data associated with the IOASID allocated by the custom allocators
* are managed by IOASID framework similar to data stored in xa by default
* allocator.
*
* There can be multiple allocators registered but only one is active. In case
* of runtime removal of a custom allocator, the next one is activated based
* on the registration ordering.
*
* Multiple allocators can share the same alloc() function, in this case the
* IOASID space is shared.
*/
int ioasid_register_allocator(struct ioasid_allocator_ops *ops)
{
struct ioasid_allocator_data *ia_data;
struct ioasid_allocator_data *pallocator;
int ret = 0;
spin_lock(&ioasid_allocator_lock);
ia_data = ioasid_alloc_allocator(ops);
if (!ia_data) {
ret = -ENOMEM;
goto out_unlock;
}
/*
* No particular preference, we activate the first one and keep
* the later registered allocators in a list in case the first one gets
* removed due to hotplug.
*/
if (list_empty(&allocators_list)) {
WARN_ON(active_allocator != &default_allocator);
/* Use this new allocator if default is not active */
if (xa_empty(&active_allocator->xa)) {
rcu_assign_pointer(active_allocator, ia_data);
list_add_tail(&ia_data->list, &allocators_list);
goto out_unlock;
}
pr_warn("Default allocator active with outstanding IOASID\n");
ret = -EAGAIN;
goto out_free;
}
/* Check if the allocator is already registered */
list_for_each_entry(pallocator, &allocators_list, list) {
if (pallocator->ops == ops) {
pr_err("IOASID allocator already registered\n");
ret = -EEXIST;
goto out_free;
} else if (use_same_ops(pallocator->ops, ops)) {
/*
* If the new allocator shares the same ops,
* then they will share the same IOASID space.
* We should put them under the same xarray.
*/
list_add_tail(&ops->list, &pallocator->slist);
goto out_free;
}
}
list_add_tail(&ia_data->list, &allocators_list);
spin_unlock(&ioasid_allocator_lock);
return 0;
out_free:
kfree(ia_data);
out_unlock:
spin_unlock(&ioasid_allocator_lock);
return ret;
}
EXPORT_SYMBOL_GPL(ioasid_register_allocator);
/**
* ioasid_unregister_allocator - Remove a custom IOASID allocator ops
* @ops: the custom allocator to be removed
*
* Remove an allocator from the list, activate the next allocator in
* the order it was registered. Or revert to default allocator if all
* custom allocators are unregistered without outstanding IOASIDs.
*/
void ioasid_unregister_allocator(struct ioasid_allocator_ops *ops)
{
struct ioasid_allocator_data *pallocator;
struct ioasid_allocator_ops *sops;
spin_lock(&ioasid_allocator_lock);
if (list_empty(&allocators_list)) {
pr_warn("No custom IOASID allocators active!\n");
goto exit_unlock;
}
list_for_each_entry(pallocator, &allocators_list, list) {
if (!use_same_ops(pallocator->ops, ops))
continue;
if (list_is_singular(&pallocator->slist)) {
/* No shared helper functions */
list_del(&pallocator->list);
/*
* All IOASIDs should have been freed before
* the last allocator that shares the same ops
* is unregistered.
*/
WARN_ON(!xa_empty(&pallocator->xa));
if (list_empty(&allocators_list)) {
pr_info("No custom IOASID allocators, switch to default.\n");
rcu_assign_pointer(active_allocator, &default_allocator);
} else if (pallocator == active_allocator) {
rcu_assign_pointer(active_allocator,
list_first_entry(&allocators_list,
struct ioasid_allocator_data, list));
pr_info("IOASID allocator changed");
}
kfree_rcu(pallocator, rcu);
break;
}
/*
* Find the matching shared ops to delete,
* but keep outstanding IOASIDs
*/
list_for_each_entry(sops, &pallocator->slist, list) {
if (sops == ops) {
list_del(&ops->list);
break;
}
}
break;
}
exit_unlock:
spin_unlock(&ioasid_allocator_lock);
}
EXPORT_SYMBOL_GPL(ioasid_unregister_allocator);
/**
* ioasid_set_data - Set private data for an allocated ioasid
* @ioasid: the ID to set data
* @data: the private data
*
* For IOASID that is already allocated, private data can be set
* via this API. Future lookup can be done via ioasid_find.
*/
int ioasid_set_data(ioasid_t ioasid, void *data)
{
struct ioasid_data *ioasid_data;
int ret = 0;
spin_lock(&ioasid_allocator_lock);
ioasid_data = xa_load(&active_allocator->xa, ioasid);
if (ioasid_data)
rcu_assign_pointer(ioasid_data->private, data);
else
ret = -ENOENT;
spin_unlock(&ioasid_allocator_lock);
/*
* Wait for readers to stop accessing the old private data, so the
* caller can free it.
*/
if (!ret)
synchronize_rcu();
return ret;
}
EXPORT_SYMBOL_GPL(ioasid_set_data);
/**
* ioasid_alloc - Allocate an IOASID
* @set: the IOASID set
* @min: the minimum ID (inclusive)
* @max: the maximum ID (inclusive)
* @private: data private to the caller
*
* Allocate an ID between @min and @max. The @private pointer is stored
* internally and can be retrieved with ioasid_find().
*
* Return: the allocated ID on success, or %INVALID_IOASID on failure.
*/
ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, ioasid_t max,
void *private)
{
struct ioasid_data *data;
void *adata;
ioasid_t id;
data = kzalloc(sizeof(*data), GFP_ATOMIC);
if (!data)
return INVALID_IOASID;
data->set = set;
data->private = private;
/*
* Custom allocator needs allocator data to perform platform specific
* operations.
*/
spin_lock(&ioasid_allocator_lock);
adata = active_allocator->flags & IOASID_ALLOCATOR_CUSTOM ? active_allocator->ops->pdata : data;
id = active_allocator->ops->alloc(min, max, adata);
if (id == INVALID_IOASID) {
pr_err("Failed ASID allocation %lu\n", active_allocator->flags);
goto exit_free;
}
if ((active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) &&
xa_alloc(&active_allocator->xa, &id, data, XA_LIMIT(id, id), GFP_ATOMIC)) {
/* Custom allocator needs framework to store and track allocation results */
pr_err("Failed to alloc ioasid from %d\n", id);
active_allocator->ops->free(id, active_allocator->ops->pdata);
goto exit_free;
}
data->id = id;
spin_unlock(&ioasid_allocator_lock);
return id;
exit_free:
spin_unlock(&ioasid_allocator_lock);
kfree(data);
return INVALID_IOASID;
}
EXPORT_SYMBOL_GPL(ioasid_alloc);
/**
* ioasid_free - Free an ioasid
* @ioasid: the ID to remove
*/
void ioasid_free(ioasid_t ioasid)
{
struct ioasid_data *ioasid_data;
spin_lock(&ioasid_allocator_lock);
ioasid_data = xa_load(&active_allocator->xa, ioasid);
if (!ioasid_data) {
pr_err("Trying to free unknown IOASID %u\n", ioasid);
goto exit_unlock;
}
active_allocator->ops->free(ioasid, active_allocator->ops->pdata);
/* Custom allocator needs additional steps to free the xa element */
if (active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) {
ioasid_data = xa_erase(&active_allocator->xa, ioasid);
kfree_rcu(ioasid_data, rcu);
}
exit_unlock:
spin_unlock(&ioasid_allocator_lock);
}
EXPORT_SYMBOL_GPL(ioasid_free);
/**
* ioasid_find - Find IOASID data
* @set: the IOASID set
* @ioasid: the IOASID to find
* @getter: function to call on the found object
*
* The optional getter function allows to take a reference to the found object
* under the rcu lock. The function can also check if the object is still valid:
* if @getter returns false, then the object is invalid and NULL is returned.
*
* If the IOASID exists, return the private pointer passed to ioasid_alloc.
* Private data can be NULL if not set. Return an error if the IOASID is not
* found, or if @set is not NULL and the IOASID does not belong to the set.
*/
void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid,
bool (*getter)(void *))
{
void *priv;
struct ioasid_data *ioasid_data;
struct ioasid_allocator_data *idata;
rcu_read_lock();
idata = rcu_dereference(active_allocator);
ioasid_data = xa_load(&idata->xa, ioasid);
if (!ioasid_data) {
priv = ERR_PTR(-ENOENT);
goto unlock;
}
if (set && ioasid_data->set != set) {
/* data found but does not belong to the set */
priv = ERR_PTR(-EACCES);
goto unlock;
}
/* Now IOASID and its set is verified, we can return the private data */
priv = rcu_dereference(ioasid_data->private);
if (getter && !getter(priv))
priv = NULL;
unlock:
rcu_read_unlock();
return priv;
}
EXPORT_SYMBOL_GPL(ioasid_find);
MODULE_AUTHOR("Jean-Philippe Brucker <jean-philippe.brucker@arm.com>");
MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
MODULE_DESCRIPTION("IO Address Space ID (IOASID) allocator");
MODULE_LICENSE("GPL");
