*/

DEFINE_IDA(pdom_ids);

static int amd_iommu_attach_device(struct iommu_domain *dom,

struct device *dev);

static void set_dte_entry(struct amd_iommu *iommu,

struct iommu_dev_data *dev_data);

/****************************************************************************

*

* Helper functions

*

****************************************************************************/

static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom)

{

return (pdom && (pdom->pd_mode == PD_MODE_V2));

return NULL;

mutex_init(&dev_data->mutex);

dev_data->devid = devid;

ratelimit_default_init(&dev_data->rs);

return dev_data;

}

{

struct iommu_dev_data *dev_data;

struct llist_node *node;

static int clone_alias(struct pci_dev *pdev, u16 alias, void *data)

{

struct amd_iommu *iommu;

u16 devid = pci_dev_id(pdev);

if (devid == alias)

return 0;

if (!iommu)

return 0;

}

static void clone_aliases(struct amd_iommu *iommu, struct device *dev)

return -ENOMEM;

dev_data->dev = dev;

setup_aliases(iommu, dev);

/*

dev_data->flags = pdev_get_caps(to_pci_dev(dev));

}

return 0;

}

static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)

{

int i;

for (i = 0; i < 4; ++i)

}

static void dump_command(unsigned long phys_addr)

int type, devid, flags, tag;

volatile u32 *event = __evt;

int count = 0;

u32 pasid;

retry:

(event[1] & EVENT_DOMID_MASK_LO);

flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;

address = (u64)(((u64)event[3]) << 32) | event[2];

if (type == 0) {

/* Did we hit the erratum? */

dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",

iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

pasid, address, flags);

dump_dte_entry(iommu, devid);

break;

case EVENT_TYPE_DEV_TAB_ERR:

return iommu_queue_command(iommu, &cmd);

}

static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)

{

u32 devid;

domain_flush_complete(domain);

}

int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)

{

struct iommu_dev_data *dev_data;

return ret;

}

static void set_dte_entry(struct amd_iommu *iommu,

struct iommu_dev_data *dev_data)

{

u16 domid;

struct protection_domain *domain = dev_data->domain;

struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;

if (gcr3_info && gcr3_info->gcr3_tbl)

domid = dev_data->gcr3_info.domid;

else

domid = domain->id;

if (domain->iop.mode != PAGE_MODE_NONE)

<< DEV_ENTRY_MODE_SHIFT;

/*

*/

if (dev_data->ppr)

if (domain->dirty_tracking)

}

/*

* A kdump kernel might be replacing a domain ID that was copied from

}

}

{

}

/* Update and flush DTE for the given device */

if (set)

set_dte_entry(iommu, dev_data);

else

clone_aliases(iommu, dev_data->dev);

device_flush_dte(dev_data);

struct protection_domain *pdom)

{

struct pdom_iommu_info *pdom_iommu_info, *curr;

unsigned long flags;

int ret = 0;

goto out_unlock;

}

out_unlock:

spin_unlock_irqrestore(&pdom->lock, flags);

return ret;

}

out_err:

iommu_completion_wait(iommu);

if (dev_is_pci(dev))

pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);

*

*****************************************************************************/

{

spin_lock_init(&domain->lock);

INIT_LIST_HEAD(&domain->dev_list);

INIT_LIST_HEAD(&domain->dev_data_list);

xa_init(&domain->iommu_array);

}

{

struct protection_domain *domain;

int domid;

}

domain->id = domid;

return domain;

}

static int pdom_setup_pgtable(struct protection_domain *domain,

{

struct io_pgtable_ops *pgtbl_ops;

break;

break;

}

if (!pgtbl_ops)

return -ENOMEM;

return 0;

}

{

return ~0ULL;

/* V2 with 4/5 level page table */

return iommu && (iommu->features & FEATURE_HDSUP);

}

{

bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;

struct protection_domain *domain;

int ret;

if (!domain)

return ERR_PTR(-ENOMEM);

if (ret) {

pdom_id_free(domain->id);

kfree(domain);

domain->domain.geometry.force_aperture = true;

domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap;

return &domain->domain;

}

static struct iommu_domain *

amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,

const struct iommu_user_data *user_data)

{

const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |

IOMMU_HWPT_ALLOC_PASID;

if ((flags & ~supported_flags) || user_data)

return ERR_PTR(-EOPNOTSUPP);

if (!amd_iommu_pasid_supported())

}

}

void amd_iommu_domain_free(struct iommu_domain *dom)

{

struct protection_domain *domain = to_pdomain(dom);

}

static int blocked_domain_attach_device(struct iommu_domain *domain,

return 0;

}

static struct iommu_domain blocked_domain = {

.type = IOMMU_DOMAIN_BLOCKED,

.ops = &(const struct iommu_domain_ops) {

.attach_dev = blocked_domain_attach_device,

}

};

identity_domain.id = pdom_id_alloc();

}

/* Same as blocked domain except it supports only ops->attach_dev() */

bool enable)

{

struct protection_domain *pdomain = to_pdomain(domain);

struct iommu_dev_data *dev_data;

bool domain_flush = false;

struct amd_iommu *iommu;

unsigned long flags;

spin_lock_irqsave(&pdomain->lock, flags);

if (!(pdomain->dirty_tracking ^ enable)) {

}

list_for_each_entry(dev_data, &pdomain->dev_list, list) {

iommu = get_amd_iommu_from_dev_data(dev_data);

/* Flush device DTE */

device_flush_dte(dev_data);

domain_flush = true;

}

.blocked_domain = &blocked_domain,

.release_domain = &release_domain,

.identity_domain = &identity_domain.domain,

.domain_alloc_paging_flags = amd_iommu_domain_alloc_paging_flags,

.domain_alloc_sva = amd_iommu_domain_alloc_sva,

.probe_device = amd_iommu_probe_device,

.def_domain_type = amd_iommu_def_domain_type,

.dev_enable_feat = amd_iommu_dev_enable_feature,

.dev_disable_feat = amd_iommu_dev_disable_feature,

.page_response = amd_iommu_page_response,

.default_domain_ops = &(const struct iommu_domain_ops) {

.attach_dev = amd_iommu_attach_device,

raw_spin_unlock_irqrestore(&iommu->lock, flags);

}

static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,

struct irq_remap_table *table)

{

}

static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)

return table;

}

{

struct irq_remap_table *table;

if (!table)

return NULL;

if (!table->table) {

kfree(table);

return NULL;

}

raw_spin_lock_init(&table->lock);

return table;

}

return 0;

}

static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,

{

struct irq_remap_table *table = NULL;

struct irq_remap_table *new_table = NULL;

struct amd_iommu_pci_seg *pci_seg;

unsigned long flags;

u16 alias;

spin_lock_irqsave(&iommu_table_lock, flags);

spin_unlock_irqrestore(&iommu_table_lock, flags);

/* Nothing there yet, allocate new irq remapping table */

if (!new_table)

return NULL;

spin_unlock_irqrestore(&iommu_table_lock, flags);

if (new_table) {

kfree(new_table);

}

return table;

}

static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,

{

struct irq_remap_table *table;

int index, c, alignment = 1;

unsigned long flags;

if (!table)

return -ENODEV;

/* Scan table for free entries */

for (index = ALIGN(table->min_index, alignment), c = 0;

if (!iommu->irte_ops->is_allocated(table, index)) {

c += 1;

} else {

msg->data = index;

msg->address_lo = 0;

msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;

msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;

}

struct amd_ir_data *data = NULL;

struct amd_iommu *iommu;

struct irq_cfg *cfg;

int i, ret, devid, seg, sbdf;

int index;

if (!iommu)

return -EINVAL;

ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);

if (ret < 0)

return ret;

if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {

struct irq_remap_table *table;

if (table) {

if (!table->min_index) {

/*

bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);

index = alloc_irq_index(iommu, devid, nr_irqs, align,

} else {

}

if (index < 0) {

irq_data->chip_data = data;

irq_data->chip = &amd_ir_chip;

irq_remapping_prepare_irte(data, cfg, info, devid, index, i);

}

return 0;