#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
#define MAX_AGAW_WIDTH 64
+ #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
static inline int agaw_to_width(int agaw)
{
- return 30 + agaw * LEVEL_STRIDE;
+ return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
}
static inline int width_to_agaw(int width)
{
- return (width - 30) / LEVEL_STRIDE;
+ return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
}
static inline unsigned int level_to_offset_bits(int level)
static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
{
- return 1 << ((lvl - 1) * LEVEL_STRIDE);
+ return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
}
/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
pte->val = 0;
}
- static inline void dma_set_pte_readable(struct dma_pte *pte)
- {
- pte->val |= DMA_PTE_READ;
- }
-
- static inline void dma_set_pte_writable(struct dma_pte *pte)
- {
- pte->val |= DMA_PTE_WRITE;
- }
-
- static inline void dma_set_pte_snp(struct dma_pte *pte)
- {
- pte->val |= DMA_PTE_SNP;
- }
-
- static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
- {
- pte->val = (pte->val & ~3) | (prot & 3);
- }
-
static inline u64 dma_pte_addr(struct dma_pte *pte)
{
#ifdef CONFIG_64BIT
#endif
}
- static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
- {
- pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
- }
-
static inline bool dma_pte_present(struct dma_pte *pte)
{
return (pte->val & 3) != 0;
static void flush_unmaps_timeout(unsigned long data);
- DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
+ static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
#define HIGH_WATER_MARK 250
struct deferred_flush_tables {
struct dmar_drhd_unit *drhd = NULL;
int i;
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
+ for_each_active_drhd_unit(drhd) {
if (segment != drhd->segment)
continue;
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
unsigned int large_page = 1;
struct dma_pte *first_pte, *pte;
- int order;
BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
} while (start_pfn && start_pfn <= last_pfn);
- order = (large_page - 1) * 9;
- return order;
+ return min_t(int, (large_page - 1) * 9, MAX_AGAW_PFN_WIDTH);
}
static void dma_pte_free_level(struct dmar_domain *domain, int level,
/* If range covers entire pagetable, free it */
if (!(start_pfn > level_pfn ||
- last_pfn < level_pfn + level_size(level))) {
+ last_pfn < level_pfn + level_size(level) - 1)) {
dma_clear_pte(pte);
domain_flush_cache(domain, pte, sizeof(*pte));
free_pgtable_page(level_pte);
unsigned long nlongs;
ndomains = cap_ndoms(iommu->cap);
- pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
- ndomains);
+ pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
+ iommu->seq_id, ndomains);
nlongs = BITS_TO_LONGS(ndomains);
spin_lock_init(&iommu->lock);
*/
iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
if (!iommu->domain_ids) {
- printk(KERN_ERR "Allocating domain id array failed\n");
+ pr_err("IOMMU%d: allocating domain id array failed\n",
+ iommu->seq_id);
return -ENOMEM;
}
iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
GFP_KERNEL);
if (!iommu->domains) {
- printk(KERN_ERR "Allocating domain array failed\n");
+ pr_err("IOMMU%d: allocating domain array failed\n",
+ iommu->seq_id);
+ kfree(iommu->domain_ids);
+ iommu->domain_ids = NULL;
return -ENOMEM;
}
static void domain_exit(struct dmar_domain *domain);
static void vm_domain_exit(struct dmar_domain *domain);
- void free_dmar_iommu(struct intel_iommu *iommu)
+ static void free_dmar_iommu(struct intel_iommu *iommu)
{
struct dmar_domain *domain;
- int i;
+ int i, count;
unsigned long flags;
if ((iommu->domains) && (iommu->domain_ids)) {
clear_bit(i, iommu->domain_ids);
spin_lock_irqsave(&domain->iommu_lock, flags);
- if (--domain->iommu_count == 0) {
+ count = --domain->iommu_count;
+ spin_unlock_irqrestore(&domain->iommu_lock, flags);
+ if (count == 0) {
if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
vm_domain_exit(domain);
else
domain_exit(domain);
}
- spin_unlock_irqrestore(&domain->iommu_lock, flags);
}
}
if (iommu->gcmd & DMA_GCMD_TE)
iommu_disable_translation(iommu);
- if (iommu->irq) {
- irq_set_handler_data(iommu->irq, NULL);
- /* This will mask the irq */
- free_irq(iommu->irq, iommu);
- destroy_irq(iommu->irq);
- }
-
kfree(iommu->domains);
kfree(iommu->domain_ids);
+ iommu->domains = NULL;
+ iommu->domain_ids = NULL;
g_iommus[iommu->seq_id] = NULL;
if (!si_domain)
return -EFAULT;
- pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
-
for_each_active_iommu(iommu, drhd) {
ret = iommu_attach_domain(si_domain, iommu);
if (ret) {
}
si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
+ pr_debug("IOMMU: identity mapping domain is domain %d\n",
+ si_domain->id);
if (hw)
return 0;
goto error;
}
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
+ for_each_active_iommu(iommu, drhd) {
g_iommus[iommu->seq_id] = iommu;
ret = iommu_init_domains(iommu);
/*
* Start from the sane iommu hardware state.
*/
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
-
+ for_each_active_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized by us
* (for example, while enabling interrupt-remapping) then
dmar_disable_qi(iommu);
}
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
-
+ for_each_active_iommu(iommu, drhd) {
if (dmar_enable_qi(iommu)) {
/*
* Queued Invalidate not enabled, use Register Based
* global invalidate iotlb
* enable translation
*/
- for_each_drhd_unit(drhd) {
+ for_each_iommu(iommu, drhd) {
if (drhd->ignored) {
/*
* we always have to disable PMRs or DMA may fail on
* this device
*/
if (force_on)
- iommu_disable_protect_mem_regions(drhd->iommu);
+ iommu_disable_protect_mem_regions(iommu);
continue;
}
- iommu = drhd->iommu;
iommu_flush_write_buffer(iommu);
return 0;
error:
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
- iommu = drhd->iommu;
- free_iommu(iommu);
- }
+ for_each_active_iommu(iommu, drhd)
+ free_dmar_iommu(iommu);
+ kfree(deferred_flush);
kfree(g_iommus);
return ret;
}
struct pci_dev *pdev;
int found;
- if (unlikely(dev->bus != &pci_bus_type))
+ if (unlikely(!dev_is_pci(dev)))
return 1;
pdev = to_pci_dev(dev);
}
}
- for_each_drhd_unit(drhd) {
+ for_each_active_drhd_unit(drhd) {
int i;
- if (drhd->ignored || drhd->include_all)
+ if (drhd->include_all)
continue;
for (i = 0; i < drhd->devices_cnt; i++)
rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
{
struct acpi_dmar_reserved_memory *rmrr;
- int ret;
rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
- ret = dmar_parse_dev_scope((void *)(rmrr + 1),
- ((void *)rmrr) + rmrr->header.length,
- &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
-
- if (ret || (rmrru->devices_cnt == 0)) {
- list_del(&rmrru->list);
- kfree(rmrru);
- }
- return ret;
+ return dmar_parse_dev_scope((void *)(rmrr + 1),
+ ((void *)rmrr) + rmrr->header.length,
+ &rmrru->devices_cnt, &rmrru->devices,
+ rmrr->segment);
}
static LIST_HEAD(dmar_atsr_units);
static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
{
- int rc;
struct acpi_dmar_atsr *atsr;
if (atsru->include_all)
return 0;
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
- rc = dmar_parse_dev_scope((void *)(atsr + 1),
- (void *)atsr + atsr->header.length,
- &atsru->devices_cnt, &atsru->devices,
- atsr->segment);
- if (rc || !atsru->devices_cnt) {
- list_del(&atsru->list);
- kfree(atsru);
+ return dmar_parse_dev_scope((void *)(atsr + 1),
+ (void *)atsr + atsr->header.length,
+ &atsru->devices_cnt, &atsru->devices,
+ atsr->segment);
+ }
+
+ static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
+ {
+ dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
+ kfree(atsru);
+ }
+
+ static void intel_iommu_free_dmars(void)
+ {
+ struct dmar_rmrr_unit *rmrru, *rmrr_n;
+ struct dmar_atsr_unit *atsru, *atsr_n;
+
+ list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
+ list_del(&rmrru->list);
+ dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
+ kfree(rmrru);
}
- return rc;
+ list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
+ list_del(&atsru->list);
+ intel_iommu_free_atsr(atsru);
+ }
}
int dmar_find_matched_atsr_unit(struct pci_dev *dev)
int __init dmar_parse_rmrr_atsr_dev(void)
{
- struct dmar_rmrr_unit *rmrr, *rmrr_n;
- struct dmar_atsr_unit *atsr, *atsr_n;
+ struct dmar_rmrr_unit *rmrr;
+ struct dmar_atsr_unit *atsr;
int ret = 0;
- list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
+ list_for_each_entry(rmrr, &dmar_rmrr_units, list) {
ret = rmrr_parse_dev(rmrr);
if (ret)
return ret;
}
- list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
+ list_for_each_entry(atsr, &dmar_atsr_units, list) {
ret = atsr_parse_dev(atsr);
if (ret)
return ret;
int __init intel_iommu_init(void)
{
- int ret = 0;
+ int ret = -ENODEV;
struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
/* VT-d is required for a TXT/tboot launch, so enforce that */
force_on = tboot_force_iommu();
if (dmar_table_init()) {
if (force_on)
panic("tboot: Failed to initialize DMAR table\n");
- return -ENODEV;
+ goto out_free_dmar;
}
/*
* Disable translation if already enabled prior to OS handover.
*/
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu;
-
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
+ for_each_active_iommu(iommu, drhd)
if (iommu->gcmd & DMA_GCMD_TE)
iommu_disable_translation(iommu);
- }
if (dmar_dev_scope_init() < 0) {
if (force_on)
panic("tboot: Failed to initialize DMAR device scope\n");
- return -ENODEV;
+ goto out_free_dmar;
}
if (no_iommu || dmar_disabled)
- return -ENODEV;
+ goto out_free_dmar;
if (iommu_init_mempool()) {
if (force_on)
panic("tboot: Failed to initialize iommu memory\n");
- return -ENODEV;
+ goto out_free_dmar;
}
if (list_empty(&dmar_rmrr_units))
if (dmar_init_reserved_ranges()) {
if (force_on)
panic("tboot: Failed to reserve iommu ranges\n");
- return -ENODEV;
+ goto out_free_mempool;
}
init_no_remapping_devices();
if (force_on)
panic("tboot: Failed to initialize DMARs\n");
printk(KERN_ERR "IOMMU: dmar init failed\n");
- put_iova_domain(&reserved_iova_list);
- iommu_exit_mempool();
- return ret;
+ goto out_free_reserved_range;
}
printk(KERN_INFO
"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
intel_iommu_enabled = 1;
return 0;
+
+ out_free_reserved_range:
+ put_iova_domain(&reserved_iova_list);
+ out_free_mempool:
+ iommu_exit_mempool();
+ out_free_dmar:
+ intel_iommu_free_dmars();
+ return ret;
}
static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
}
/* domain id for virtual machine, it won't be set in context */
- static unsigned long vm_domid;
+ static atomic_t vm_domid = ATOMIC_INIT(0);
static struct dmar_domain *iommu_alloc_vm_domain(void)
{
if (!domain)
return NULL;
- domain->id = vm_domid++;
+ domain->id = atomic_inc_return(&vm_domid);
domain->nid = -1;
memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
unsigned long i;
unsigned long ndomains;
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
- iommu = drhd->iommu;
-
+ for_each_active_iommu(iommu, drhd) {
ndomains = cap_ndoms(iommu->cap);
for_each_set_bit(i, iommu->domain_ids, ndomains) {
if (iommu->domains[i] == domain) {
#include <linux/hpet.h>
#include <linux/pci.h>
#include <linux/irq.h>
+#include <linux/intel-iommu.h>
+#include <linux/acpi.h>
#include <asm/io_apic.h>
#include <asm/smp.h>
#include <asm/cpu.h>
-#include <linux/intel-iommu.h>
-#include <acpi/acpi.h>
#include <asm/irq_remapping.h>
#include <asm/pci-direct.h>
#include <asm/msidef.h>
static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
+ static int __init parse_ioapics_under_ir(void);
+
static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
{
struct irq_cfg *cfg = irq_get_chip_data(irq);
return cfg ? &cfg->irq_2_iommu : NULL;
}
- int get_irte(int irq, struct irte *entry)
+ static int get_irte(int irq, struct irte *entry)
{
struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
unsigned long flags;
struct ir_table *table = iommu->ir_table;
struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
struct irq_cfg *cfg = irq_get_chip_data(irq);
- u16 index, start_index;
unsigned int mask = 0;
unsigned long flags;
- int i;
+ int index;
if (!count || !irq_iommu)
return -1;
- /*
- * start the IRTE search from index 0.
- */
- index = start_index = 0;
-
if (count > 1) {
count = __roundup_pow_of_two(count);
mask = ilog2(count);
}
raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
- do {
- for (i = index; i < index + count; i++)
- if (table->base[i].present)
- break;
- /* empty index found */
- if (i == index + count)
- break;
-
- index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
-
- if (index == start_index) {
- raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
- printk(KERN_ERR "can't allocate an IRTE\n");
- return -1;
- }
- } while (1);
-
- for (i = index; i < index + count; i++)
- table->base[i].present = 1;
-
- cfg->remapped = 1;
- irq_iommu->iommu = iommu;
- irq_iommu->irte_index = index;
- irq_iommu->sub_handle = 0;
- irq_iommu->irte_mask = mask;
-
+ index = bitmap_find_free_region(table->bitmap,
+ INTR_REMAP_TABLE_ENTRIES, mask);
+ if (index < 0) {
+ pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);
+ } else {
+ cfg->remapped = 1;
+ irq_iommu->iommu = iommu;
+ irq_iommu->irte_index = index;
+ irq_iommu->sub_handle = 0;
+ irq_iommu->irte_mask = mask;
+ }
raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
return index;
set_64bit(&entry->low, 0);
set_64bit(&entry->high, 0);
}
+ bitmap_release_region(iommu->ir_table->bitmap, index,
+ irq_iommu->irte_mask);
return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
}
return -1;
}
- set_irte_sid(irte, 1, 0, sid);
+ set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid);
return 0;
}
{
struct ir_table *ir_table;
struct page *pages;
+ unsigned long *bitmap;
ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
GFP_ATOMIC);
INTR_REMAP_PAGE_ORDER);
if (!pages) {
- printk(KERN_ERR "failed to allocate pages of order %d\n",
- INTR_REMAP_PAGE_ORDER);
+ pr_err("IR%d: failed to allocate pages of order %d\n",
+ iommu->seq_id, INTR_REMAP_PAGE_ORDER);
kfree(iommu->ir_table);
return -ENOMEM;
}
+ bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES),
+ sizeof(long), GFP_ATOMIC);
+ if (bitmap == NULL) {
+ pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);
+ __free_pages(pages, INTR_REMAP_PAGE_ORDER);
+ kfree(ir_table);
+ return -ENOMEM;
+ }
+
ir_table->base = page_address(pages);
+ ir_table->bitmap = bitmap;
iommu_set_irq_remapping(iommu, mode);
return 0;
static int __init intel_irq_remapping_supported(void)
{
struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
if (disable_irq_remap)
return 0;
if (!dmar_ir_support())
return 0;
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu = drhd->iommu;
-
+ for_each_iommu(iommu, drhd)
if (!ecap_ir_support(iommu->ecap))
return 0;
- }
return 1;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
bool x2apic_present;
int setup = 0;
int eim = 0;
}
if (x2apic_present) {
+ pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
+
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu = drhd->iommu;
-
+ for_each_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
/*
* check for the Interrupt-remapping support
*/
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu = drhd->iommu;
-
+ for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
/*
* Enable queued invalidation for all the DRHD's.
*/
- for_each_drhd_unit(drhd) {
- int ret;
- struct intel_iommu *iommu = drhd->iommu;
- ret = dmar_enable_qi(iommu);
+ for_each_iommu(iommu, drhd) {
+ int ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu = drhd->iommu;
-
+ for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
* Finds the assocaition between IOAPIC's and its Interrupt-remapping
* hardware unit.
*/
- int __init parse_ioapics_under_ir(void)
+ static int __init parse_ioapics_under_ir(void)
{
struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
int ir_supported = 0;
int ioapic_idx;
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu = drhd->iommu;
-
+ for_each_iommu(iommu, drhd)
if (ecap_ir_support(iommu->ecap)) {
if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
return -1;
ir_supported = 1;
}
- }
if (!ir_supported)
return 0;
return 1;
}
- int __init ir_dev_scope_init(void)
+ static int __init ir_dev_scope_init(void)
{
if (!irq_remapping_enabled)
return 0;
projects / ~andy / linux / commitdiff