#include <linux/msi.h>
#include <linux/amd-iommu.h>
#include <linux/export.h>
-#include <linux/acpi.h>
#include <acpi/acpi.h>
#include <asm/pci-direct.h>
#include <asm/iommu.h>
#include <asm/gart.h>
#include <asm/x86_init.h>
#include <asm/iommu_table.h>
+#include <asm/io_apic.h>
+#include <asm/irq_remapping.h>
#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
+#include "irq_remapping.h"
/*
* definitions for the ACPI scanning code
#define IVHD_DEV_ALIAS_RANGE 0x43
#define IVHD_DEV_EXT_SELECT 0x46
#define IVHD_DEV_EXT_SELECT_RANGE 0x47
+#define IVHD_DEV_SPECIAL 0x48
+
+#define IVHD_SPECIAL_IOAPIC 1
+#define IVHD_SPECIAL_HPET 2
#define IVHD_FLAG_HT_TUN_EN_MASK 0x01
#define IVHD_FLAG_PASSPW_EN_MASK 0x02
} __attribute__((packed));
bool amd_iommu_dump;
+bool amd_iommu_irq_remap __read_mostly;
static bool amd_iommu_detected;
static bool __initdata amd_iommu_disabled;
struct amd_iommu **amd_iommu_rlookup_table;
/*
- * AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap
+ * This table is used to find the irq remapping table for a given device id
+ * quickly.
+ */
+struct irq_remap_table **irq_lookup_table;
+
+/*
+ * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
* to know which ones are already in use.
*/
unsigned long *amd_iommu_pd_alloc_bitmap;
/****************************************************************************
*
- * The following functions belong the the code path which parses the ACPI table
+ * The following functions belong to the code path which parses the ACPI table
* the second time. In this ACPI parsing iteration we allocate IOMMU specific
* data structures, initialize the device/alias/rlookup table and also
* basically initialize the hardware.
set_iommu_for_device(iommu, devid);
}
+static int add_special_device(u8 type, u8 id, u16 devid)
+{
+ struct devid_map *entry;
+ struct list_head *list;
+
+ if (type != IVHD_SPECIAL_IOAPIC && type != IVHD_SPECIAL_HPET)
+ return -EINVAL;
+
+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+ if (!entry)
+ return -ENOMEM;
+
+ entry->id = id;
+ entry->devid = devid;
+
+ if (type == IVHD_SPECIAL_IOAPIC)
+ list = &ioapic_map;
+ else
+ list = &hpet_map;
+
+ list_add_tail(&entry->list, list);
+
+ return 0;
+}
+
/*
- * Reads the device exclusion range from ACPI and initialize IOMMU with
+ * Reads the device exclusion range from ACPI and initializes the IOMMU with
* it
*/
static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
* Takes a pointer to an AMD IOMMU entry in the ACPI table and
* initializes the hardware and our data structures with it.
*/
-static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
+static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
struct ivhd_header *h)
{
u8 *p = (u8 *)h;
flags, ext_flags);
}
break;
+ case IVHD_DEV_SPECIAL: {
+ u8 handle, type;
+ const char *var;
+ u16 devid;
+ int ret;
+
+ handle = e->ext & 0xff;
+ devid = (e->ext >> 8) & 0xffff;
+ type = (e->ext >> 24) & 0xff;
+
+ if (type == IVHD_SPECIAL_IOAPIC)
+ var = "IOAPIC";
+ else if (type == IVHD_SPECIAL_HPET)
+ var = "HPET";
+ else
+ var = "UNKNOWN";
+
+ DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",
+ var, (int)handle,
+ PCI_BUS(devid),
+ PCI_SLOT(devid),
+ PCI_FUNC(devid));
+
+ set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
+ ret = add_special_device(type, handle, devid);
+ if (ret)
+ return ret;
+ break;
+ }
default:
break;
}
p += ivhd_entry_length(p);
}
+
+ return 0;
}
/* Initializes the device->iommu mapping for the driver */
*/
static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
{
+ int ret;
+
spin_lock_init(&iommu->lock);
/* Add IOMMU to internal data structures */
iommu->int_enabled = false;
- init_iommu_from_acpi(iommu, h);
+ ret = init_iommu_from_acpi(iommu, h);
+ if (ret)
+ return ret;
+
+ /*
+ * Make sure IOMMU is not considered to translate itself. The IVRS
+ * table tells us so, but this is a lie!
+ */
+ amd_iommu_rlookup_table[iommu->devid] = NULL;
+
init_iommu_devices(iommu);
return 0;
if (iommu_feature(iommu, (1ULL << i)))
pr_cont(" %s", feat_str[i]);
}
- }
pr_cont("\n");
+ }
}
+ if (irq_remapping_enabled)
+ pr_info("AMD-Vi: Interrupt remapping enabled\n");
}
static int __init amd_iommu_init_pci(void)
/****************************************************************************
*
* The following functions initialize the MSI interrupts for all IOMMUs
- * in the system. Its a bit challenging because there could be multiple
+ * in the system. It's a bit challenging because there could be multiple
* IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
* pci_dev.
*
*
* The next functions belong to the third pass of parsing the ACPI
* table. In this last pass the memory mapping requirements are
- * gathered (like exclusion and unity mapping reanges).
+ * gathered (like exclusion and unity mapping ranges).
*
****************************************************************************/
* Init the device table to not allow DMA access for devices and
* suppress all page faults
*/
-static void init_device_table(void)
+static void init_device_table_dma(void)
{
u32 devid;
}
}
+static void __init uninit_device_table_dma(void)
+{
+ u32 devid;
+
+ for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
+ amd_iommu_dev_table[devid].data[0] = 0ULL;
+ amd_iommu_dev_table[devid].data[1] = 0ULL;
+ }
+}
+
+static void init_device_table(void)
+{
+ u32 devid;
+
+ if (!amd_iommu_irq_remap)
+ return;
+
+ for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
+ set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN);
+}
+
static void iommu_init_flags(struct amd_iommu *iommu)
{
iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
static void __init free_on_init_error(void)
{
- amd_iommu_uninit_devices();
+ free_pages((unsigned long)irq_lookup_table,
+ get_order(rlookup_table_size));
- free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
- get_order(MAX_DOMAIN_ID/8));
+ if (amd_iommu_irq_cache) {
+ kmem_cache_destroy(amd_iommu_irq_cache);
+ amd_iommu_irq_cache = NULL;
+
+ }
free_pages((unsigned long)amd_iommu_rlookup_table,
get_order(rlookup_table_size));
free_iommu_all();
- free_unity_maps();
-
#ifdef CONFIG_GART_IOMMU
/*
* We failed to initialize the AMD IOMMU - try fallback to GART
#endif
}
+static bool __init check_ioapic_information(void)
+{
+ int idx;
+
+ for (idx = 0; idx < nr_ioapics; idx++) {
+ int id = mpc_ioapic_id(idx);
+
+ if (get_ioapic_devid(id) < 0) {
+ pr_err(FW_BUG "AMD-Vi: IO-APIC[%d] not in IVRS table\n", id);
+ pr_err("AMD-Vi: Disabling interrupt remapping due to BIOS Bug\n");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static void __init free_dma_resources(void)
+{
+ amd_iommu_uninit_devices();
+
+ free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
+ get_order(MAX_DOMAIN_ID/8));
+
+ free_unity_maps();
+}
+
/*
* This is the hardware init function for AMD IOMMU in the system.
* This function is called either from amd_iommu_init or from the interrupt
if (amd_iommu_pd_alloc_bitmap == NULL)
goto out;
- /* init the device table */
- init_device_table();
-
/*
* let all alias entries point to itself
*/
if (ret)
goto out;
+ if (amd_iommu_irq_remap)
+ amd_iommu_irq_remap = check_ioapic_information();
+
+ if (amd_iommu_irq_remap) {
+ /*
+ * Interrupt remapping enabled, create kmem_cache for the
+ * remapping tables.
+ */
+ amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
+ MAX_IRQS_PER_TABLE * sizeof(u32),
+ IRQ_TABLE_ALIGNMENT,
+ 0, NULL);
+ if (!amd_iommu_irq_cache)
+ goto out;
+
+ irq_lookup_table = (void *)__get_free_pages(
+ GFP_KERNEL | __GFP_ZERO,
+ get_order(rlookup_table_size));
+ if (!irq_lookup_table)
+ goto out;
+ }
+
ret = init_memory_definitions(ivrs_base);
if (ret)
goto out;
+ /* init the device table */
+ init_device_table();
+
out:
/* Don't leak any ACPI memory */
early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
/* Make sure ACS will be enabled during PCI probe */
pci_request_acs();
+ if (!disable_irq_remap)
+ amd_iommu_irq_remap = true;
+
return true;
}
static int amd_iommu_init_dma(void)
{
+ struct amd_iommu *iommu;
int ret;
+ init_device_table_dma();
+
+ for_each_iommu(iommu)
+ iommu_flush_all_caches(iommu);
+
if (iommu_pass_through)
ret = amd_iommu_init_passthrough();
else
return ret;
}
+#ifdef CONFIG_IRQ_REMAP
+int __init amd_iommu_prepare(void)
+{
+ return iommu_go_to_state(IOMMU_ACPI_FINISHED);
+}
+
+int __init amd_iommu_supported(void)
+{
+ return amd_iommu_irq_remap ? 1 : 0;
+}
+
+int __init amd_iommu_enable(void)
+{
+ int ret;
+
+ ret = iommu_go_to_state(IOMMU_ENABLED);
+ if (ret)
+ return ret;
+
+ irq_remapping_enabled = 1;
+
+ return 0;
+}
+
+void amd_iommu_disable(void)
+{
+ amd_iommu_suspend();
+}
+
+int amd_iommu_reenable(int mode)
+{
+ amd_iommu_resume();
+
+ return 0;
+}
+int __init amd_iommu_enable_faulting(void)
+{
+ /* We enable MSI later when PCI is initialized */
+ return 0;
+}
+#endif
/*
* This is the core init function for AMD IOMMU hardware in the system.
ret = iommu_go_to_state(IOMMU_INITIALIZED);
if (ret) {
- disable_iommus();
- free_on_init_error();
+ free_dma_resources();
+ if (!irq_remapping_enabled) {
+ disable_iommus();
+ free_on_init_error();
+ } else {
+ struct amd_iommu *iommu;
+
+ uninit_device_table_dma();
+ for_each_iommu(iommu)
+ iommu_flush_all_caches(iommu);
+ }
}
return ret;
projects / ~andy / linux / blobdiff