#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
#define MAX_AGAW_WIDTH 64
+#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
static inline int agaw_to_width(int agaw)
{
- return 30 + agaw * LEVEL_STRIDE;
+ return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
}
static inline int width_to_agaw(int width)
{
- return (width - 30) / LEVEL_STRIDE;
+ return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
}
static inline unsigned int level_to_offset_bits(int level)
static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
{
- return 1 << ((lvl - 1) * LEVEL_STRIDE);
+ return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
}
/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
pte->val = 0;
}
-static inline void dma_set_pte_readable(struct dma_pte *pte)
-{
- pte->val |= DMA_PTE_READ;
-}
-
-static inline void dma_set_pte_writable(struct dma_pte *pte)
-{
- pte->val |= DMA_PTE_WRITE;
-}
-
-static inline void dma_set_pte_snp(struct dma_pte *pte)
-{
- pte->val |= DMA_PTE_SNP;
-}
-
-static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
-{
- pte->val = (pte->val & ~3) | (prot & 3);
-}
-
static inline u64 dma_pte_addr(struct dma_pte *pte)
{
#ifdef CONFIG_64BIT
#endif
}
-static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
-{
- pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
-}
-
static inline bool dma_pte_present(struct dma_pte *pte)
{
return (pte->val & 3) != 0;
static void flush_unmaps_timeout(unsigned long data);
-DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
+static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
#define HIGH_WATER_MARK 250
struct deferred_flush_tables {
struct dmar_drhd_unit *drhd = NULL;
int i;
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
+ for_each_active_drhd_unit(drhd) {
if (segment != drhd->segment)
continue;
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
unsigned int large_page = 1;
struct dma_pte *first_pte, *pte;
- int order;
BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
} while (start_pfn && start_pfn <= last_pfn);
- order = (large_page - 1) * 9;
- return order;
+ return min_t(int, (large_page - 1) * 9, MAX_AGAW_PFN_WIDTH);
}
static void dma_pte_free_level(struct dmar_domain *domain, int level,
unsigned long nlongs;
ndomains = cap_ndoms(iommu->cap);
- pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
- ndomains);
+ pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
+ iommu->seq_id, ndomains);
nlongs = BITS_TO_LONGS(ndomains);
spin_lock_init(&iommu->lock);
*/
iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
if (!iommu->domain_ids) {
- printk(KERN_ERR "Allocating domain id array failed\n");
+ pr_err("IOMMU%d: allocating domain id array failed\n",
+ iommu->seq_id);
return -ENOMEM;
}
iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
GFP_KERNEL);
if (!iommu->domains) {
- printk(KERN_ERR "Allocating domain array failed\n");
+ pr_err("IOMMU%d: allocating domain array failed\n",
+ iommu->seq_id);
+ kfree(iommu->domain_ids);
+ iommu->domain_ids = NULL;
return -ENOMEM;
}
static void domain_exit(struct dmar_domain *domain);
static void vm_domain_exit(struct dmar_domain *domain);
-void free_dmar_iommu(struct intel_iommu *iommu)
+static void free_dmar_iommu(struct intel_iommu *iommu)
{
struct dmar_domain *domain;
- int i;
+ int i, count;
unsigned long flags;
if ((iommu->domains) && (iommu->domain_ids)) {
clear_bit(i, iommu->domain_ids);
spin_lock_irqsave(&domain->iommu_lock, flags);
- if (--domain->iommu_count == 0) {
+ count = --domain->iommu_count;
+ spin_unlock_irqrestore(&domain->iommu_lock, flags);
+ if (count == 0) {
if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
vm_domain_exit(domain);
else
domain_exit(domain);
}
- spin_unlock_irqrestore(&domain->iommu_lock, flags);
}
}
if (iommu->gcmd & DMA_GCMD_TE)
iommu_disable_translation(iommu);
- if (iommu->irq) {
- irq_set_handler_data(iommu->irq, NULL);
- /* This will mask the irq */
- free_irq(iommu->irq, iommu);
- destroy_irq(iommu->irq);
- }
-
kfree(iommu->domains);
kfree(iommu->domain_ids);
+ iommu->domains = NULL;
+ iommu->domain_ids = NULL;
g_iommus[iommu->seq_id] = NULL;
if (!si_domain)
return -EFAULT;
- pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
-
for_each_active_iommu(iommu, drhd) {
ret = iommu_attach_domain(si_domain, iommu);
if (ret) {
}
si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
+ pr_debug("IOMMU: identity mapping domain is domain %d\n",
+ si_domain->id);
if (hw)
return 0;
goto error;
}
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
+ for_each_active_iommu(iommu, drhd) {
g_iommus[iommu->seq_id] = iommu;
ret = iommu_init_domains(iommu);
/*
* Start from the sane iommu hardware state.
*/
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
-
+ for_each_active_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized by us
* (for example, while enabling interrupt-remapping) then
dmar_disable_qi(iommu);
}
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
-
+ for_each_active_iommu(iommu, drhd) {
if (dmar_enable_qi(iommu)) {
/*
* Queued Invalidate not enabled, use Register Based
* global invalidate iotlb
* enable translation
*/
- for_each_drhd_unit(drhd) {
+ for_each_iommu(iommu, drhd) {
if (drhd->ignored) {
/*
* we always have to disable PMRs or DMA may fail on
* this device
*/
if (force_on)
- iommu_disable_protect_mem_regions(drhd->iommu);
+ iommu_disable_protect_mem_regions(iommu);
continue;
}
- iommu = drhd->iommu;
iommu_flush_write_buffer(iommu);
return 0;
error:
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
- iommu = drhd->iommu;
- free_iommu(iommu);
- }
+ for_each_active_iommu(iommu, drhd)
+ free_dmar_iommu(iommu);
+ kfree(deferred_flush);
kfree(g_iommus);
return ret;
}
struct pci_dev *pdev;
int found;
- if (unlikely(dev->bus != &pci_bus_type))
+ if (unlikely(!dev_is_pci(dev)))
return 1;
pdev = to_pci_dev(dev);
}
}
- for_each_drhd_unit(drhd) {
+ for_each_active_drhd_unit(drhd) {
int i;
- if (drhd->ignored || drhd->include_all)
+ if (drhd->include_all)
continue;
for (i = 0; i < drhd->devices_cnt; i++)
rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
{
struct acpi_dmar_reserved_memory *rmrr;
- int ret;
rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
- ret = dmar_parse_dev_scope((void *)(rmrr + 1),
- ((void *)rmrr) + rmrr->header.length,
- &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
-
- if (ret || (rmrru->devices_cnt == 0)) {
- list_del(&rmrru->list);
- kfree(rmrru);
- }
- return ret;
+ return dmar_parse_dev_scope((void *)(rmrr + 1),
+ ((void *)rmrr) + rmrr->header.length,
+ &rmrru->devices_cnt, &rmrru->devices,
+ rmrr->segment);
}
static LIST_HEAD(dmar_atsr_units);
static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
{
- int rc;
struct acpi_dmar_atsr *atsr;
if (atsru->include_all)
return 0;
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
- rc = dmar_parse_dev_scope((void *)(atsr + 1),
- (void *)atsr + atsr->header.length,
- &atsru->devices_cnt, &atsru->devices,
- atsr->segment);
- if (rc || !atsru->devices_cnt) {
- list_del(&atsru->list);
- kfree(atsru);
+ return dmar_parse_dev_scope((void *)(atsr + 1),
+ (void *)atsr + atsr->header.length,
+ &atsru->devices_cnt, &atsru->devices,
+ atsr->segment);
+}
+
+static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
+{
+ dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
+ kfree(atsru);
+}
+
+static void intel_iommu_free_dmars(void)
+{
+ struct dmar_rmrr_unit *rmrru, *rmrr_n;
+ struct dmar_atsr_unit *atsru, *atsr_n;
+
+ list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
+ list_del(&rmrru->list);
+ dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
+ kfree(rmrru);
}
- return rc;
+ list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
+ list_del(&atsru->list);
+ intel_iommu_free_atsr(atsru);
+ }
}
int dmar_find_matched_atsr_unit(struct pci_dev *dev)
int __init dmar_parse_rmrr_atsr_dev(void)
{
- struct dmar_rmrr_unit *rmrr, *rmrr_n;
- struct dmar_atsr_unit *atsr, *atsr_n;
+ struct dmar_rmrr_unit *rmrr;
+ struct dmar_atsr_unit *atsr;
int ret = 0;
- list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
+ list_for_each_entry(rmrr, &dmar_rmrr_units, list) {
ret = rmrr_parse_dev(rmrr);
if (ret)
return ret;
}
- list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
+ list_for_each_entry(atsr, &dmar_atsr_units, list) {
ret = atsr_parse_dev(atsr);
if (ret)
return ret;
int __init intel_iommu_init(void)
{
- int ret = 0;
+ int ret = -ENODEV;
struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
/* VT-d is required for a TXT/tboot launch, so enforce that */
force_on = tboot_force_iommu();
if (dmar_table_init()) {
if (force_on)
panic("tboot: Failed to initialize DMAR table\n");
- return -ENODEV;
+ goto out_free_dmar;
}
/*
* Disable translation if already enabled prior to OS handover.
*/
- for_each_drhd_unit(drhd) {
- struct intel_iommu *iommu;
-
- if (drhd->ignored)
- continue;
-
- iommu = drhd->iommu;
+ for_each_active_iommu(iommu, drhd)
if (iommu->gcmd & DMA_GCMD_TE)
iommu_disable_translation(iommu);
- }
if (dmar_dev_scope_init() < 0) {
if (force_on)
panic("tboot: Failed to initialize DMAR device scope\n");
- return -ENODEV;
+ goto out_free_dmar;
}
if (no_iommu || dmar_disabled)
- return -ENODEV;
+ goto out_free_dmar;
if (iommu_init_mempool()) {
if (force_on)
panic("tboot: Failed to initialize iommu memory\n");
- return -ENODEV;
+ goto out_free_dmar;
}
if (list_empty(&dmar_rmrr_units))
if (dmar_init_reserved_ranges()) {
if (force_on)
panic("tboot: Failed to reserve iommu ranges\n");
- return -ENODEV;
+ goto out_free_mempool;
}
init_no_remapping_devices();
if (force_on)
panic("tboot: Failed to initialize DMARs\n");
printk(KERN_ERR "IOMMU: dmar init failed\n");
- put_iova_domain(&reserved_iova_list);
- iommu_exit_mempool();
- return ret;
+ goto out_free_reserved_range;
}
printk(KERN_INFO
"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
intel_iommu_enabled = 1;
return 0;
+
+out_free_reserved_range:
+ put_iova_domain(&reserved_iova_list);
+out_free_mempool:
+ iommu_exit_mempool();
+out_free_dmar:
+ intel_iommu_free_dmars();
+ return ret;
}
static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
}
/* domain id for virtual machine, it won't be set in context */
-static unsigned long vm_domid;
+static atomic_t vm_domid = ATOMIC_INIT(0);
static struct dmar_domain *iommu_alloc_vm_domain(void)
{
if (!domain)
return NULL;
- domain->id = vm_domid++;
+ domain->id = atomic_inc_return(&vm_domid);
domain->nid = -1;
memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
unsigned long i;
unsigned long ndomains;
- for_each_drhd_unit(drhd) {
- if (drhd->ignored)
- continue;
- iommu = drhd->iommu;
-
+ for_each_active_iommu(iommu, drhd) {
ndomains = cap_ndoms(iommu->cap);
for_each_set_bit(i, iommu->domain_ids, ndomains) {
if (iommu->domains[i] == domain) {
projects / ~andy / linux / blobdiff