From eef93fdb7cd41ae36794db0e765059dc1039e940 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:46:59 -0700 Subject: iommu: Rename intr_remapping files to intel_intr_remapping The files contain code mostly relevant for the Intel implementation of interrupt remapping. Make that visible in the file names. Also inline intr_remapping.h into intr_remapping.c because it is only included there and the content is very small. So there is no reason for a seperate header file. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/Makefile | 2 +- drivers/iommu/intel_intr_remapping.c | 849 +++++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.c | 834 ---------------------------------- drivers/iommu/intr_remapping.h | 17 - 4 files changed, 850 insertions(+), 852 deletions(-) create mode 100644 drivers/iommu/intel_intr_remapping.c delete mode 100644 drivers/iommu/intr_remapping.c delete mode 100644 drivers/iommu/intr_remapping.h (limited to 'drivers') diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 7ad7a3bc1242..1533ebf1d68e 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o -obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o +obj-$(CONFIG_IRQ_REMAP) += intel_intr_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c new file mode 100644 index 000000000000..212fff0c24b5 --- /dev/null +++ b/drivers/iommu/intel_intr_remapping.c @@ -0,0 +1,849 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ioapic_scope { + struct intel_iommu *iommu; + unsigned int id; + unsigned int bus; /* PCI bus number */ + unsigned int devfn; /* PCI devfn number */ +}; + +struct hpet_scope { + struct intel_iommu *iommu; + u8 id; + unsigned int bus; + unsigned int devfn; +}; + +#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) + +static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; +static struct hpet_scope ir_hpet[MAX_HPET_TBS]; +static int ir_ioapic_num, ir_hpet_num; +int intr_remapping_enabled; + +static int disable_intremap; +static int disable_sourceid_checking; +static int no_x2apic_optout; + +static __init int setup_nointremap(char *str) +{ + disable_intremap = 1; + return 0; +} +early_param("nointremap", setup_nointremap); + +static __init int setup_intremap(char *str) +{ + if (!str) + return -EINVAL; + + while (*str) { + if (!strncmp(str, "on", 2)) + disable_intremap = 0; + else if (!strncmp(str, "off", 3)) + disable_intremap = 1; + else if (!strncmp(str, "nosid", 5)) + disable_sourceid_checking = 1; + else if (!strncmp(str, "no_x2apic_optout", 16)) + no_x2apic_optout = 1; + + str += strcspn(str, ","); + while (*str == ',') + str++; + } + + return 0; +} +early_param("intremap", setup_intremap); + +static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); + +static struct irq_2_iommu *irq_2_iommu(unsigned int irq) +{ + struct irq_cfg *cfg = irq_get_chip_data(irq); + return cfg ? &cfg->irq_2_iommu : NULL; +} + +int get_irte(int irq, struct irte *entry) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int index; + + if (!entry || !irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + index = irq_iommu->irte_index + irq_iommu->sub_handle; + *entry = *(irq_iommu->iommu->ir_table->base + index); + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + return 0; +} + +int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) +{ + struct ir_table *table = iommu->ir_table; + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + u16 index, start_index; + unsigned int mask = 0; + unsigned long flags; + int i; + + if (!count || !irq_iommu) + return -1; + + /* + * start the IRTE search from index 0. + */ + index = start_index = 0; + + if (count > 1) { + count = __roundup_pow_of_two(count); + mask = ilog2(count); + } + + if (mask > ecap_max_handle_mask(iommu->ecap)) { + printk(KERN_ERR + "Requested mask %x exceeds the max invalidation handle" + " mask value %Lx\n", mask, + ecap_max_handle_mask(iommu->ecap)); + return -1; + } + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + do { + for (i = index; i < index + count; i++) + if (table->base[i].present) + break; + /* empty index found */ + if (i == index + count) + break; + + index = (index + count) % INTR_REMAP_TABLE_ENTRIES; + + if (index == start_index) { + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + printk(KERN_ERR "can't allocate an IRTE\n"); + return -1; + } + } while (1); + + for (i = index; i < index + count; i++) + table->base[i].present = 1; + + irq_iommu->iommu = iommu; + irq_iommu->irte_index = index; + irq_iommu->sub_handle = 0; + irq_iommu->irte_mask = mask; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return index; +} + +static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) +{ + struct qi_desc desc; + + desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) + | QI_IEC_SELECTIVE; + desc.high = 0; + + return qi_submit_sync(&desc, iommu); +} + +int map_irq_to_irte_handle(int irq, u16 *sub_handle) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int index; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + *sub_handle = irq_iommu->sub_handle; + index = irq_iommu->irte_index; + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + return index; +} + +int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + irq_iommu->iommu = iommu; + irq_iommu->irte_index = index; + irq_iommu->sub_handle = subhandle; + irq_iommu->irte_mask = 0; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return 0; +} + +int modify_irte(int irq, struct irte *irte_modified) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + struct intel_iommu *iommu; + unsigned long flags; + struct irte *irte; + int rc, index; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + iommu = irq_iommu->iommu; + + index = irq_iommu->irte_index + irq_iommu->sub_handle; + irte = &iommu->ir_table->base[index]; + + set_64bit(&irte->low, irte_modified->low); + set_64bit(&irte->high, irte_modified->high); + __iommu_flush_cache(iommu, irte, sizeof(*irte)); + + rc = qi_flush_iec(iommu, index, 0); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return rc; +} + +struct intel_iommu *map_hpet_to_ir(u8 hpet_id) +{ + int i; + + for (i = 0; i < MAX_HPET_TBS; i++) + if (ir_hpet[i].id == hpet_id) + return ir_hpet[i].iommu; + return NULL; +} + +struct intel_iommu *map_ioapic_to_ir(int apic) +{ + int i; + + for (i = 0; i < MAX_IO_APICS; i++) + if (ir_ioapic[i].id == apic) + return ir_ioapic[i].iommu; + return NULL; +} + +struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) +{ + struct dmar_drhd_unit *drhd; + + drhd = dmar_find_matched_drhd_unit(dev); + if (!drhd) + return NULL; + + return drhd->iommu; +} + +static int clear_entries(struct irq_2_iommu *irq_iommu) +{ + struct irte *start, *entry, *end; + struct intel_iommu *iommu; + int index; + + if (irq_iommu->sub_handle) + return 0; + + iommu = irq_iommu->iommu; + index = irq_iommu->irte_index + irq_iommu->sub_handle; + + start = iommu->ir_table->base + index; + end = start + (1 << irq_iommu->irte_mask); + + for (entry = start; entry < end; entry++) { + set_64bit(&entry->low, 0); + set_64bit(&entry->high, 0); + } + + return qi_flush_iec(iommu, index, irq_iommu->irte_mask); +} + +int free_irte(int irq) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int rc; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + rc = clear_entries(irq_iommu); + + irq_iommu->iommu = NULL; + irq_iommu->irte_index = 0; + irq_iommu->sub_handle = 0; + irq_iommu->irte_mask = 0; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return rc; +} + +/* + * source validation type + */ +#define SVT_NO_VERIFY 0x0 /* no verification is required */ +#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ +#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ + +/* + * source-id qualifier + */ +#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ +#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore + * the third least significant bit + */ +#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore + * the second and third least significant bits + */ +#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore + * the least three significant bits + */ + +/* + * set SVT, SQ and SID fields of irte to verify + * source ids of interrupt requests + */ +static void set_irte_sid(struct irte *irte, unsigned int svt, + unsigned int sq, unsigned int sid) +{ + if (disable_sourceid_checking) + svt = SVT_NO_VERIFY; + irte->svt = svt; + irte->sq = sq; + irte->sid = sid; +} + +int set_ioapic_sid(struct irte *irte, int apic) +{ + int i; + u16 sid = 0; + + if (!irte) + return -1; + + for (i = 0; i < MAX_IO_APICS; i++) { + if (ir_ioapic[i].id == apic) { + sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; + break; + } + } + + if (sid == 0) { + pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); + return -1; + } + + set_irte_sid(irte, 1, 0, sid); + + return 0; +} + +int set_hpet_sid(struct irte *irte, u8 id) +{ + int i; + u16 sid = 0; + + if (!irte) + return -1; + + for (i = 0; i < MAX_HPET_TBS; i++) { + if (ir_hpet[i].id == id) { + sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; + break; + } + } + + if (sid == 0) { + pr_warning("Failed to set source-id of HPET block (%d)\n", id); + return -1; + } + + /* + * Should really use SQ_ALL_16. Some platforms are broken. + * While we figure out the right quirks for these broken platforms, use + * SQ_13_IGNORE_3 for now. + */ + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); + + return 0; +} + +int set_msi_sid(struct irte *irte, struct pci_dev *dev) +{ + struct pci_dev *bridge; + + if (!irte || !dev) + return -1; + + /* PCIe device or Root Complex integrated PCI device */ + if (pci_is_pcie(dev) || !dev->bus->parent) { + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, + (dev->bus->number << 8) | dev->devfn); + return 0; + } + + bridge = pci_find_upstream_pcie_bridge(dev); + if (bridge) { + if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */ + set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, + (bridge->bus->number << 8) | dev->bus->number); + else /* this is a legacy PCI bridge */ + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, + (bridge->bus->number << 8) | bridge->devfn); + } + + return 0; +} + +static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) +{ + u64 addr; + u32 sts; + unsigned long flags; + + addr = virt_to_phys((void *)iommu->ir_table->base); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + dmar_writeq(iommu->reg + DMAR_IRTA_REG, + (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); + + /* Set interrupt-remapping table pointer */ + iommu->gcmd |= DMA_GCMD_SIRTP; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_IRTPS), sts); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); + + /* + * global invalidation of interrupt entry cache before enabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + /* Enable interrupt-remapping */ + iommu->gcmd |= DMA_GCMD_IRE; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_IRES), sts); + + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + + +static int setup_intr_remapping(struct intel_iommu *iommu, int mode) +{ + struct ir_table *ir_table; + struct page *pages; + + ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), + GFP_ATOMIC); + + if (!iommu->ir_table) + return -ENOMEM; + + pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, + INTR_REMAP_PAGE_ORDER); + + if (!pages) { + printk(KERN_ERR "failed to allocate pages of order %d\n", + INTR_REMAP_PAGE_ORDER); + kfree(iommu->ir_table); + return -ENOMEM; + } + + ir_table->base = page_address(pages); + + iommu_set_intr_remapping(iommu, mode); + return 0; +} + +/* + * Disable Interrupt Remapping. + */ +static void iommu_disable_intr_remapping(struct intel_iommu *iommu) +{ + unsigned long flags; + u32 sts; + + if (!ecap_ir_support(iommu->ecap)) + return; + + /* + * global invalidation of interrupt entry cache before disabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); + if (!(sts & DMA_GSTS_IRES)) + goto end; + + iommu->gcmd &= ~DMA_GCMD_IRE; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, !(sts & DMA_GSTS_IRES), sts); + +end: + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + +static int __init dmar_x2apic_optout(void) +{ + struct acpi_table_dmar *dmar; + dmar = (struct acpi_table_dmar *)dmar_tbl; + if (!dmar || no_x2apic_optout) + return 0; + return dmar->flags & DMAR_X2APIC_OPT_OUT; +} + +int __init intr_remapping_supported(void) +{ + struct dmar_drhd_unit *drhd; + + if (disable_intremap) + return 0; + + if (!dmar_ir_support()) + return 0; + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + return 0; + } + + return 1; +} + +int __init enable_intr_remapping(void) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + int eim = 0; + + if (parse_ioapics_under_ir() != 1) { + printk(KERN_INFO "Not enable interrupt remapping\n"); + return -1; + } + + if (x2apic_supported()) { + eim = !dmar_x2apic_optout(); + WARN(!eim, KERN_WARNING + "Your BIOS is broken and requested that x2apic be disabled\n" + "This will leave your machine vulnerable to irq-injection attacks\n" + "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); + } + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + /* + * If the queued invalidation is already initialized, + * shouldn't disable it. + */ + if (iommu->qi) + continue; + + /* + * Clear previous faults. + */ + dmar_fault(-1, iommu); + + /* + * Disable intr remapping and queued invalidation, if already + * enabled prior to OS handover. + */ + iommu_disable_intr_remapping(iommu); + + dmar_disable_qi(iommu); + } + + /* + * check for the Interrupt-remapping support + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + continue; + + if (eim && !ecap_eim_support(iommu->ecap)) { + printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " + " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); + return -1; + } + } + + /* + * Enable queued invalidation for all the DRHD's. + */ + for_each_drhd_unit(drhd) { + int ret; + struct intel_iommu *iommu = drhd->iommu; + ret = dmar_enable_qi(iommu); + + if (ret) { + printk(KERN_ERR "DRHD %Lx: failed to enable queued, " + " invalidation, ecap %Lx, ret %d\n", + drhd->reg_base_addr, iommu->ecap, ret); + return -1; + } + } + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + continue; + + if (setup_intr_remapping(iommu, eim)) + goto error; + + setup = 1; + } + + if (!setup) + goto error; + + intr_remapping_enabled = 1; + pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); + + return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + +static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, + struct intel_iommu *iommu) +{ + struct acpi_dmar_pci_path *path; + u8 bus; + int count; + + bus = scope->bus; + path = (struct acpi_dmar_pci_path *)(scope + 1); + count = (scope->length - sizeof(struct acpi_dmar_device_scope)) + / sizeof(struct acpi_dmar_pci_path); + + while (--count > 0) { + /* + * Access PCI directly due to the PCI + * subsystem isn't initialized yet. + */ + bus = read_pci_config_byte(bus, path->dev, path->fn, + PCI_SECONDARY_BUS); + path++; + } + ir_hpet[ir_hpet_num].bus = bus; + ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn); + ir_hpet[ir_hpet_num].iommu = iommu; + ir_hpet[ir_hpet_num].id = scope->enumeration_id; + ir_hpet_num++; +} + +static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, + struct intel_iommu *iommu) +{ + struct acpi_dmar_pci_path *path; + u8 bus; + int count; + + bus = scope->bus; + path = (struct acpi_dmar_pci_path *)(scope + 1); + count = (scope->length - sizeof(struct acpi_dmar_device_scope)) + / sizeof(struct acpi_dmar_pci_path); + + while (--count > 0) { + /* + * Access PCI directly due to the PCI + * subsystem isn't initialized yet. + */ + bus = read_pci_config_byte(bus, path->dev, path->fn, + PCI_SECONDARY_BUS); + path++; + } + + ir_ioapic[ir_ioapic_num].bus = bus; + ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn); + ir_ioapic[ir_ioapic_num].iommu = iommu; + ir_ioapic[ir_ioapic_num].id = scope->enumeration_id; + ir_ioapic_num++; +} + +static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, + struct intel_iommu *iommu) +{ + struct acpi_dmar_hardware_unit *drhd; + struct acpi_dmar_device_scope *scope; + void *start, *end; + + drhd = (struct acpi_dmar_hardware_unit *)header; + + start = (void *)(drhd + 1); + end = ((void *)drhd) + header->length; + + while (start < end) { + scope = start; + if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { + if (ir_ioapic_num == MAX_IO_APICS) { + printk(KERN_WARNING "Exceeded Max IO APICS\n"); + return -1; + } + + printk(KERN_INFO "IOAPIC id %d under DRHD base " + " 0x%Lx IOMMU %d\n", scope->enumeration_id, + drhd->address, iommu->seq_id); + + ir_parse_one_ioapic_scope(scope, iommu); + } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) { + if (ir_hpet_num == MAX_HPET_TBS) { + printk(KERN_WARNING "Exceeded Max HPET blocks\n"); + return -1; + } + + printk(KERN_INFO "HPET id %d under DRHD base" + " 0x%Lx\n", scope->enumeration_id, + drhd->address); + + ir_parse_one_hpet_scope(scope, iommu); + } + start += scope->length; + } + + return 0; +} + +/* + * Finds the assocaition between IOAPIC's and its Interrupt-remapping + * hardware unit. + */ +int __init parse_ioapics_under_ir(void) +{ + struct dmar_drhd_unit *drhd; + int ir_supported = 0; + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (ecap_ir_support(iommu->ecap)) { + if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) + return -1; + + ir_supported = 1; + } + } + + if (ir_supported && ir_ioapic_num != nr_ioapics) { + printk(KERN_WARNING + "Not all IO-APIC's listed under remapping hardware\n"); + return -1; + } + + return ir_supported; +} + +int __init ir_dev_scope_init(void) +{ + if (!intr_remapping_enabled) + return 0; + + return dmar_dev_scope_init(); +} +rootfs_initcall(ir_dev_scope_init); + +void disable_intr_remapping(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + + /* + * Disable Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + iommu_disable_intr_remapping(iommu); + } +} + +int reenable_intr_remapping(int eim) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + struct intel_iommu *iommu = NULL; + + for_each_iommu(iommu, drhd) + if (iommu->qi) + dmar_reenable_qi(iommu); + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + /* Set up interrupt remapping for iommu.*/ + iommu_set_intr_remapping(iommu, eim); + setup = 1; + } + + if (!setup) + goto error; + + return 0; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c deleted file mode 100644 index 6777ca049471..000000000000 --- a/drivers/iommu/intr_remapping.c +++ /dev/null @@ -1,834 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "intr_remapping.h" -#include -#include - -static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; -static struct hpet_scope ir_hpet[MAX_HPET_TBS]; -static int ir_ioapic_num, ir_hpet_num; -int intr_remapping_enabled; - -static int disable_intremap; -static int disable_sourceid_checking; -static int no_x2apic_optout; - -static __init int setup_nointremap(char *str) -{ - disable_intremap = 1; - return 0; -} -early_param("nointremap", setup_nointremap); - -static __init int setup_intremap(char *str) -{ - if (!str) - return -EINVAL; - - while (*str) { - if (!strncmp(str, "on", 2)) - disable_intremap = 0; - else if (!strncmp(str, "off", 3)) - disable_intremap = 1; - else if (!strncmp(str, "nosid", 5)) - disable_sourceid_checking = 1; - else if (!strncmp(str, "no_x2apic_optout", 16)) - no_x2apic_optout = 1; - - str += strcspn(str, ","); - while (*str == ',') - str++; - } - - return 0; -} -early_param("intremap", setup_intremap); - -static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); - -static struct irq_2_iommu *irq_2_iommu(unsigned int irq) -{ - struct irq_cfg *cfg = irq_get_chip_data(irq); - return cfg ? &cfg->irq_2_iommu : NULL; -} - -int get_irte(int irq, struct irte *entry) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int index; - - if (!entry || !irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - index = irq_iommu->irte_index + irq_iommu->sub_handle; - *entry = *(irq_iommu->iommu->ir_table->base + index); - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - return 0; -} - -int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) -{ - struct ir_table *table = iommu->ir_table; - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - u16 index, start_index; - unsigned int mask = 0; - unsigned long flags; - int i; - - if (!count || !irq_iommu) - return -1; - - /* - * start the IRTE search from index 0. - */ - index = start_index = 0; - - if (count > 1) { - count = __roundup_pow_of_two(count); - mask = ilog2(count); - } - - if (mask > ecap_max_handle_mask(iommu->ecap)) { - printk(KERN_ERR - "Requested mask %x exceeds the max invalidation handle" - " mask value %Lx\n", mask, - ecap_max_handle_mask(iommu->ecap)); - return -1; - } - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - do { - for (i = index; i < index + count; i++) - if (table->base[i].present) - break; - /* empty index found */ - if (i == index + count) - break; - - index = (index + count) % INTR_REMAP_TABLE_ENTRIES; - - if (index == start_index) { - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - printk(KERN_ERR "can't allocate an IRTE\n"); - return -1; - } - } while (1); - - for (i = index; i < index + count; i++) - table->base[i].present = 1; - - irq_iommu->iommu = iommu; - irq_iommu->irte_index = index; - irq_iommu->sub_handle = 0; - irq_iommu->irte_mask = mask; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return index; -} - -static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) -{ - struct qi_desc desc; - - desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) - | QI_IEC_SELECTIVE; - desc.high = 0; - - return qi_submit_sync(&desc, iommu); -} - -int map_irq_to_irte_handle(int irq, u16 *sub_handle) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int index; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - *sub_handle = irq_iommu->sub_handle; - index = irq_iommu->irte_index; - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - return index; -} - -int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - irq_iommu->iommu = iommu; - irq_iommu->irte_index = index; - irq_iommu->sub_handle = subhandle; - irq_iommu->irte_mask = 0; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return 0; -} - -int modify_irte(int irq, struct irte *irte_modified) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - struct intel_iommu *iommu; - unsigned long flags; - struct irte *irte; - int rc, index; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - iommu = irq_iommu->iommu; - - index = irq_iommu->irte_index + irq_iommu->sub_handle; - irte = &iommu->ir_table->base[index]; - - set_64bit(&irte->low, irte_modified->low); - set_64bit(&irte->high, irte_modified->high); - __iommu_flush_cache(iommu, irte, sizeof(*irte)); - - rc = qi_flush_iec(iommu, index, 0); - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return rc; -} - -struct intel_iommu *map_hpet_to_ir(u8 hpet_id) -{ - int i; - - for (i = 0; i < MAX_HPET_TBS; i++) - if (ir_hpet[i].id == hpet_id) - return ir_hpet[i].iommu; - return NULL; -} - -struct intel_iommu *map_ioapic_to_ir(int apic) -{ - int i; - - for (i = 0; i < MAX_IO_APICS; i++) - if (ir_ioapic[i].id == apic) - return ir_ioapic[i].iommu; - return NULL; -} - -struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) -{ - struct dmar_drhd_unit *drhd; - - drhd = dmar_find_matched_drhd_unit(dev); - if (!drhd) - return NULL; - - return drhd->iommu; -} - -static int clear_entries(struct irq_2_iommu *irq_iommu) -{ - struct irte *start, *entry, *end; - struct intel_iommu *iommu; - int index; - - if (irq_iommu->sub_handle) - return 0; - - iommu = irq_iommu->iommu; - index = irq_iommu->irte_index + irq_iommu->sub_handle; - - start = iommu->ir_table->base + index; - end = start + (1 << irq_iommu->irte_mask); - - for (entry = start; entry < end; entry++) { - set_64bit(&entry->low, 0); - set_64bit(&entry->high, 0); - } - - return qi_flush_iec(iommu, index, irq_iommu->irte_mask); -} - -int free_irte(int irq) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int rc; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - rc = clear_entries(irq_iommu); - - irq_iommu->iommu = NULL; - irq_iommu->irte_index = 0; - irq_iommu->sub_handle = 0; - irq_iommu->irte_mask = 0; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return rc; -} - -/* - * source validation type - */ -#define SVT_NO_VERIFY 0x0 /* no verification is required */ -#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ -#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ - -/* - * source-id qualifier - */ -#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ -#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore - * the third least significant bit - */ -#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore - * the second and third least significant bits - */ -#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore - * the least three significant bits - */ - -/* - * set SVT, SQ and SID fields of irte to verify - * source ids of interrupt requests - */ -static void set_irte_sid(struct irte *irte, unsigned int svt, - unsigned int sq, unsigned int sid) -{ - if (disable_sourceid_checking) - svt = SVT_NO_VERIFY; - irte->svt = svt; - irte->sq = sq; - irte->sid = sid; -} - -int set_ioapic_sid(struct irte *irte, int apic) -{ - int i; - u16 sid = 0; - - if (!irte) - return -1; - - for (i = 0; i < MAX_IO_APICS; i++) { - if (ir_ioapic[i].id == apic) { - sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; - break; - } - } - - if (sid == 0) { - pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); - return -1; - } - - set_irte_sid(irte, 1, 0, sid); - - return 0; -} - -int set_hpet_sid(struct irte *irte, u8 id) -{ - int i; - u16 sid = 0; - - if (!irte) - return -1; - - for (i = 0; i < MAX_HPET_TBS; i++) { - if (ir_hpet[i].id == id) { - sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; - break; - } - } - - if (sid == 0) { - pr_warning("Failed to set source-id of HPET block (%d)\n", id); - return -1; - } - - /* - * Should really use SQ_ALL_16. Some platforms are broken. - * While we figure out the right quirks for these broken platforms, use - * SQ_13_IGNORE_3 for now. - */ - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); - - return 0; -} - -int set_msi_sid(struct irte *irte, struct pci_dev *dev) -{ - struct pci_dev *bridge; - - if (!irte || !dev) - return -1; - - /* PCIe device or Root Complex integrated PCI device */ - if (pci_is_pcie(dev) || !dev->bus->parent) { - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, - (dev->bus->number << 8) | dev->devfn); - return 0; - } - - bridge = pci_find_upstream_pcie_bridge(dev); - if (bridge) { - if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */ - set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, - (bridge->bus->number << 8) | dev->bus->number); - else /* this is a legacy PCI bridge */ - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, - (bridge->bus->number << 8) | bridge->devfn); - } - - return 0; -} - -static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) -{ - u64 addr; - u32 sts; - unsigned long flags; - - addr = virt_to_phys((void *)iommu->ir_table->base); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - dmar_writeq(iommu->reg + DMAR_IRTA_REG, - (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); - - /* Set interrupt-remapping table pointer */ - iommu->gcmd |= DMA_GCMD_SIRTP; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, (sts & DMA_GSTS_IRTPS), sts); - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); - - /* - * global invalidation of interrupt entry cache before enabling - * interrupt-remapping. - */ - qi_global_iec(iommu); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - /* Enable interrupt-remapping */ - iommu->gcmd |= DMA_GCMD_IRE; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, (sts & DMA_GSTS_IRES), sts); - - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); -} - - -static int setup_intr_remapping(struct intel_iommu *iommu, int mode) -{ - struct ir_table *ir_table; - struct page *pages; - - ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), - GFP_ATOMIC); - - if (!iommu->ir_table) - return -ENOMEM; - - pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, - INTR_REMAP_PAGE_ORDER); - - if (!pages) { - printk(KERN_ERR "failed to allocate pages of order %d\n", - INTR_REMAP_PAGE_ORDER); - kfree(iommu->ir_table); - return -ENOMEM; - } - - ir_table->base = page_address(pages); - - iommu_set_intr_remapping(iommu, mode); - return 0; -} - -/* - * Disable Interrupt Remapping. - */ -static void iommu_disable_intr_remapping(struct intel_iommu *iommu) -{ - unsigned long flags; - u32 sts; - - if (!ecap_ir_support(iommu->ecap)) - return; - - /* - * global invalidation of interrupt entry cache before disabling - * interrupt-remapping. - */ - qi_global_iec(iommu); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); - if (!(sts & DMA_GSTS_IRES)) - goto end; - - iommu->gcmd &= ~DMA_GCMD_IRE; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, !(sts & DMA_GSTS_IRES), sts); - -end: - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); -} - -static int __init dmar_x2apic_optout(void) -{ - struct acpi_table_dmar *dmar; - dmar = (struct acpi_table_dmar *)dmar_tbl; - if (!dmar || no_x2apic_optout) - return 0; - return dmar->flags & DMAR_X2APIC_OPT_OUT; -} - -int __init intr_remapping_supported(void) -{ - struct dmar_drhd_unit *drhd; - - if (disable_intremap) - return 0; - - if (!dmar_ir_support()) - return 0; - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - return 0; - } - - return 1; -} - -int __init enable_intr_remapping(void) -{ - struct dmar_drhd_unit *drhd; - int setup = 0; - int eim = 0; - - if (parse_ioapics_under_ir() != 1) { - printk(KERN_INFO "Not enable interrupt remapping\n"); - return -1; - } - - if (x2apic_supported()) { - eim = !dmar_x2apic_optout(); - WARN(!eim, KERN_WARNING - "Your BIOS is broken and requested that x2apic be disabled\n" - "This will leave your machine vulnerable to irq-injection attacks\n" - "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); - } - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - /* - * If the queued invalidation is already initialized, - * shouldn't disable it. - */ - if (iommu->qi) - continue; - - /* - * Clear previous faults. - */ - dmar_fault(-1, iommu); - - /* - * Disable intr remapping and queued invalidation, if already - * enabled prior to OS handover. - */ - iommu_disable_intr_remapping(iommu); - - dmar_disable_qi(iommu); - } - - /* - * check for the Interrupt-remapping support - */ - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - continue; - - if (eim && !ecap_eim_support(iommu->ecap)) { - printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " - " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); - return -1; - } - } - - /* - * Enable queued invalidation for all the DRHD's. - */ - for_each_drhd_unit(drhd) { - int ret; - struct intel_iommu *iommu = drhd->iommu; - ret = dmar_enable_qi(iommu); - - if (ret) { - printk(KERN_ERR "DRHD %Lx: failed to enable queued, " - " invalidation, ecap %Lx, ret %d\n", - drhd->reg_base_addr, iommu->ecap, ret); - return -1; - } - } - - /* - * Setup Interrupt-remapping for all the DRHD's now. - */ - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - continue; - - if (setup_intr_remapping(iommu, eim)) - goto error; - - setup = 1; - } - - if (!setup) - goto error; - - intr_remapping_enabled = 1; - pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); - - return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; - -error: - /* - * handle error condition gracefully here! - */ - return -1; -} - -static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, - struct intel_iommu *iommu) -{ - struct acpi_dmar_pci_path *path; - u8 bus; - int count; - - bus = scope->bus; - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (--count > 0) { - /* - * Access PCI directly due to the PCI - * subsystem isn't initialized yet. - */ - bus = read_pci_config_byte(bus, path->dev, path->fn, - PCI_SECONDARY_BUS); - path++; - } - ir_hpet[ir_hpet_num].bus = bus; - ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn); - ir_hpet[ir_hpet_num].iommu = iommu; - ir_hpet[ir_hpet_num].id = scope->enumeration_id; - ir_hpet_num++; -} - -static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, - struct intel_iommu *iommu) -{ - struct acpi_dmar_pci_path *path; - u8 bus; - int count; - - bus = scope->bus; - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (--count > 0) { - /* - * Access PCI directly due to the PCI - * subsystem isn't initialized yet. - */ - bus = read_pci_config_byte(bus, path->dev, path->fn, - PCI_SECONDARY_BUS); - path++; - } - - ir_ioapic[ir_ioapic_num].bus = bus; - ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn); - ir_ioapic[ir_ioapic_num].iommu = iommu; - ir_ioapic[ir_ioapic_num].id = scope->enumeration_id; - ir_ioapic_num++; -} - -static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, - struct intel_iommu *iommu) -{ - struct acpi_dmar_hardware_unit *drhd; - struct acpi_dmar_device_scope *scope; - void *start, *end; - - drhd = (struct acpi_dmar_hardware_unit *)header; - - start = (void *)(drhd + 1); - end = ((void *)drhd) + header->length; - - while (start < end) { - scope = start; - if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { - if (ir_ioapic_num == MAX_IO_APICS) { - printk(KERN_WARNING "Exceeded Max IO APICS\n"); - return -1; - } - - printk(KERN_INFO "IOAPIC id %d under DRHD base " - " 0x%Lx IOMMU %d\n", scope->enumeration_id, - drhd->address, iommu->seq_id); - - ir_parse_one_ioapic_scope(scope, iommu); - } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) { - if (ir_hpet_num == MAX_HPET_TBS) { - printk(KERN_WARNING "Exceeded Max HPET blocks\n"); - return -1; - } - - printk(KERN_INFO "HPET id %d under DRHD base" - " 0x%Lx\n", scope->enumeration_id, - drhd->address); - - ir_parse_one_hpet_scope(scope, iommu); - } - start += scope->length; - } - - return 0; -} - -/* - * Finds the assocaition between IOAPIC's and its Interrupt-remapping - * hardware unit. - */ -int __init parse_ioapics_under_ir(void) -{ - struct dmar_drhd_unit *drhd; - int ir_supported = 0; - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (ecap_ir_support(iommu->ecap)) { - if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) - return -1; - - ir_supported = 1; - } - } - - if (ir_supported && ir_ioapic_num != nr_ioapics) { - printk(KERN_WARNING - "Not all IO-APIC's listed under remapping hardware\n"); - return -1; - } - - return ir_supported; -} - -int __init ir_dev_scope_init(void) -{ - if (!intr_remapping_enabled) - return 0; - - return dmar_dev_scope_init(); -} -rootfs_initcall(ir_dev_scope_init); - -void disable_intr_remapping(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu = NULL; - - /* - * Disable Interrupt-remapping for all the DRHD's now. - */ - for_each_iommu(iommu, drhd) { - if (!ecap_ir_support(iommu->ecap)) - continue; - - iommu_disable_intr_remapping(iommu); - } -} - -int reenable_intr_remapping(int eim) -{ - struct dmar_drhd_unit *drhd; - int setup = 0; - struct intel_iommu *iommu = NULL; - - for_each_iommu(iommu, drhd) - if (iommu->qi) - dmar_reenable_qi(iommu); - - /* - * Setup Interrupt-remapping for all the DRHD's now. - */ - for_each_iommu(iommu, drhd) { - if (!ecap_ir_support(iommu->ecap)) - continue; - - /* Set up interrupt remapping for iommu.*/ - iommu_set_intr_remapping(iommu, eim); - setup = 1; - } - - if (!setup) - goto error; - - return 0; - -error: - /* - * handle error condition gracefully here! - */ - return -1; -} - diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h deleted file mode 100644 index 5662fecfee60..000000000000 --- a/drivers/iommu/intr_remapping.h +++ /dev/null @@ -1,17 +0,0 @@ -#include - -struct ioapic_scope { - struct intel_iommu *iommu; - unsigned int id; - unsigned int bus; /* PCI bus number */ - unsigned int devfn; /* PCI devfn number */ -}; - -struct hpet_scope { - struct intel_iommu *iommu; - u8 id; - unsigned int bus; - unsigned int devfn; -}; - -#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) -- cgit v1.2.3 From 736baef4472d00574089f295bc759ac002b9558c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:00 -0700 Subject: iommu/vt-d: Make intr-remapping initialization generic This patch introduces irq_remap_ops to hold implementation specific function pointer to handle interrupt remapping. As the first part the initialization functions for VT-d are converted to these ops. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/Makefile | 2 +- drivers/iommu/dmar.c | 1 + drivers/iommu/intel-iommu.c | 1 + drivers/iommu/intel_intr_remapping.c | 52 ++++++------------------ drivers/iommu/intr_remapping.c | 76 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.h | 46 ++++++++++++++++++++++ 6 files changed, 137 insertions(+), 41 deletions(-) create mode 100644 drivers/iommu/intr_remapping.c create mode 100644 drivers/iommu/intr_remapping.h (limited to 'drivers') diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 1533ebf1d68e..823e1cf8708f 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o -obj-$(CONFIG_IRQ_REMAP) += intel_intr_remapping.o +obj-$(CONFIG_IRQ_REMAP) += intel_intr_remapping.o intr_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 35c1e17fce1d..647e366403dc 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #define PREFIX "DMAR: " diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index f93d5ac8f81c..e1439808192c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index 212fff0c24b5..9c742fb111b6 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -11,8 +11,11 @@ #include #include #include +#include #include +#include "intr_remapping.h" + struct ioapic_scope { struct intel_iommu *iommu; unsigned int id; @@ -32,42 +35,6 @@ struct hpet_scope { static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; static struct hpet_scope ir_hpet[MAX_HPET_TBS]; static int ir_ioapic_num, ir_hpet_num; -int intr_remapping_enabled; - -static int disable_intremap; -static int disable_sourceid_checking; -static int no_x2apic_optout; - -static __init int setup_nointremap(char *str) -{ - disable_intremap = 1; - return 0; -} -early_param("nointremap", setup_nointremap); - -static __init int setup_intremap(char *str) -{ - if (!str) - return -EINVAL; - - while (*str) { - if (!strncmp(str, "on", 2)) - disable_intremap = 0; - else if (!strncmp(str, "off", 3)) - disable_intremap = 1; - else if (!strncmp(str, "nosid", 5)) - disable_sourceid_checking = 1; - else if (!strncmp(str, "no_x2apic_optout", 16)) - no_x2apic_optout = 1; - - str += strcspn(str, ","); - while (*str == ',') - str++; - } - - return 0; -} -early_param("intremap", setup_intremap); static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); @@ -465,7 +432,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) } -static int setup_intr_remapping(struct intel_iommu *iommu, int mode) +static int intel_setup_intr_remapping(struct intel_iommu *iommu, int mode) { struct ir_table *ir_table; struct page *pages; @@ -534,7 +501,7 @@ static int __init dmar_x2apic_optout(void) return dmar->flags & DMAR_X2APIC_OPT_OUT; } -int __init intr_remapping_supported(void) +static int __init intel_intr_remapping_supported(void) { struct dmar_drhd_unit *drhd; @@ -554,7 +521,7 @@ int __init intr_remapping_supported(void) return 1; } -int __init enable_intr_remapping(void) +static int __init intel_enable_intr_remapping(void) { struct dmar_drhd_unit *drhd; int setup = 0; @@ -638,7 +605,7 @@ int __init enable_intr_remapping(void) if (!ecap_ir_support(iommu->ecap)) continue; - if (setup_intr_remapping(iommu, eim)) + if (intel_setup_intr_remapping(iommu, eim)) goto error; setup = 1; @@ -847,3 +814,8 @@ error: return -1; } +struct irq_remap_ops intel_irq_remap_ops = { + .supported = intel_intr_remapping_supported, + .hardware_init = dmar_table_init, + .hardware_enable = intel_enable_intr_remapping, +}; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c new file mode 100644 index 000000000000..670c69a80afd --- /dev/null +++ b/drivers/iommu/intr_remapping.c @@ -0,0 +1,76 @@ +#include +#include +#include + +#include "intr_remapping.h" + +int intr_remapping_enabled; + +int disable_intremap; +int disable_sourceid_checking; +int no_x2apic_optout; + +static struct irq_remap_ops *remap_ops; + +static __init int setup_nointremap(char *str) +{ + disable_intremap = 1; + return 0; +} +early_param("nointremap", setup_nointremap); + +static __init int setup_intremap(char *str) +{ + if (!str) + return -EINVAL; + + while (*str) { + if (!strncmp(str, "on", 2)) + disable_intremap = 0; + else if (!strncmp(str, "off", 3)) + disable_intremap = 1; + else if (!strncmp(str, "nosid", 5)) + disable_sourceid_checking = 1; + else if (!strncmp(str, "no_x2apic_optout", 16)) + no_x2apic_optout = 1; + + str += strcspn(str, ","); + while (*str == ',') + str++; + } + + return 0; +} +early_param("intremap", setup_intremap); + +void __init setup_intr_remapping(void) +{ + remap_ops = &intel_irq_remap_ops; +} + +int intr_remapping_supported(void) +{ + if (disable_intremap) + return 0; + + if (!remap_ops || !remap_ops->supported) + return 0; + + return remap_ops->supported(); +} + +int __init intr_hardware_init(void) +{ + if (!remap_ops || !remap_ops->hardware_init) + return -ENODEV; + + return remap_ops->hardware_init(); +} + +int __init intr_hardware_enable(void) +{ + if (!remap_ops || !remap_ops->hardware_enable) + return -ENODEV; + + return remap_ops->hardware_enable(); +} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h new file mode 100644 index 000000000000..d6df732e001f --- /dev/null +++ b/drivers/iommu/intr_remapping.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2012 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * This header file contains stuff that is shared between different interrupt + * remapping drivers but with no need to be visible outside of the IOMMU layer. + */ + +#ifndef __INTR_REMAPPING_H +#define __INTR_REMAPPING_H + +#ifdef CONFIG_IRQ_REMAP + +extern int disable_intremap; +extern int disable_sourceid_checking; +extern int no_x2apic_optout; + +struct irq_remap_ops { + /* Check whether Interrupt Remapping is supported */ + int (*supported)(void); + + /* Initializes hardware and makes it ready for remapping interrupts */ + int (*hardware_init)(void); + + /* Enables the remapping hardware */ + int (*hardware_enable)(void); +}; + +extern struct irq_remap_ops intel_irq_remap_ops; + +#endif /* CONFIG_IRQ_REMAP */ + +#endif /* __INTR_REMAPPING_H */ -- cgit v1.2.3 From 4f3d8b67ad3090f9fb72f8235d21cde53cd24b79 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:01 -0700 Subject: iommu/vt-d: Convert missing apic.c intr-remapping call to remap_ops Convert these calls too: * Disable of remapping hardware * Reenable of remapping hardware * Enable fault handling With that all of arch/x86/kernel/apic/apic.c is converted to use the generic intr-remapping interface. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/intel_intr_remapping.c | 7 +++++-- drivers/iommu/intr_remapping.c | 24 ++++++++++++++++++++++++ drivers/iommu/intr_remapping.h | 9 +++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index 9c742fb111b6..610b75b66c07 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -764,7 +764,7 @@ int __init ir_dev_scope_init(void) } rootfs_initcall(ir_dev_scope_init); -void disable_intr_remapping(void) +static void disable_intr_remapping(void) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu = NULL; @@ -780,7 +780,7 @@ void disable_intr_remapping(void) } } -int reenable_intr_remapping(int eim) +static int reenable_intr_remapping(int eim) { struct dmar_drhd_unit *drhd; int setup = 0; @@ -818,4 +818,7 @@ struct irq_remap_ops intel_irq_remap_ops = { .supported = intel_intr_remapping_supported, .hardware_init = dmar_table_init, .hardware_enable = intel_enable_intr_remapping, + .hardware_disable = disable_intr_remapping, + .hardware_reenable = reenable_intr_remapping, + .enable_faulting = enable_drhd_fault_handling, }; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 670c69a80afd..9aabed7c0320 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -74,3 +74,27 @@ int __init intr_hardware_enable(void) return remap_ops->hardware_enable(); } + +void intr_hardware_disable(void) +{ + if (!remap_ops || !remap_ops->hardware_disable) + return; + + remap_ops->hardware_disable(); +} + +int intr_hardware_reenable(int mode) +{ + if (!remap_ops || !remap_ops->hardware_reenable) + return 0; + + return remap_ops->hardware_reenable(mode); +} + +int __init intr_enable_fault_handling(void) +{ + if (!remap_ops || !remap_ops->enable_faulting) + return -ENODEV; + + return remap_ops->enable_faulting(); +} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h index d6df732e001f..2744c9ae4aec 100644 --- a/drivers/iommu/intr_remapping.h +++ b/drivers/iommu/intr_remapping.h @@ -37,6 +37,15 @@ struct irq_remap_ops { /* Enables the remapping hardware */ int (*hardware_enable)(void); + + /* Disables the remapping hardware */ + void (*hardware_disable)(void); + + /* Reenables the remapping hardware */ + int (*hardware_reenable)(int); + + /* Enable fault handling */ + int (*enable_faulting)(void); }; extern struct irq_remap_ops intel_irq_remap_ops; -- cgit v1.2.3 From 0c3f173a88c4ae3e4253427cf574a59ad5352918 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:02 -0700 Subject: iommu/vt-d: Convert IR ioapic-setup to use remap_ops The IOAPIC setup routine for interrupt remapping is VT-d specific. Move it to the irq_remap_ops and add a call helper function. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/intel_intr_remapping.c | 89 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.c | 12 +++++ drivers/iommu/intr_remapping.h | 8 ++++ 3 files changed, 109 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index 610b75b66c07..f495eba4b6ab 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -31,6 +31,7 @@ struct hpet_scope { }; #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) +#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8) static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; static struct hpet_scope ir_hpet[MAX_HPET_TBS]; @@ -814,6 +815,93 @@ error: return -1; } +static void prepare_irte(struct irte *irte, int vector, + unsigned int dest) +{ + memset(irte, 0, sizeof(*irte)); + + irte->present = 1; + irte->dst_mode = apic->irq_dest_mode; + /* + * Trigger mode in the IRTE will always be edge, and for IO-APIC, the + * actual level or edge trigger will be setup in the IO-APIC + * RTE. This will help simplify level triggered irq migration. + * For more details, see the comments (in io_apic.c) explainig IO-APIC + * irq migration in the presence of interrupt-remapping. + */ + irte->trigger_mode = 0; + irte->dlvry_mode = apic->irq_delivery_mode; + irte->vector = vector; + irte->dest_id = IRTE_DEST(dest); + irte->redir_hint = 1; +} + +static int intel_setup_ioapic_entry(int irq, + struct IO_APIC_route_entry *route_entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) +{ + int ioapic_id = mpc_ioapic_id(attr->ioapic); + struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id); + struct IR_IO_APIC_route_entry *entry; + struct irte irte; + int index; + + if (!iommu) { + pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); + return -ENODEV; + } + + entry = (struct IR_IO_APIC_route_entry *)route_entry; + + index = alloc_irte(iommu, irq, 1); + if (index < 0) { + pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id); + return -ENOMEM; + } + + prepare_irte(&irte, vector, destination); + + /* Set source-id of interrupt request */ + set_ioapic_sid(&irte, ioapic_id); + + modify_irte(irq, &irte); + + apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " + "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " + "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " + "Avail:%X Vector:%02X Dest:%08X " + "SID:%04X SQ:%X SVT:%X)\n", + attr->ioapic, irte.present, irte.fpd, irte.dst_mode, + irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, + irte.avail, irte.vector, irte.dest_id, + irte.sid, irte.sq, irte.svt); + + memset(entry, 0, sizeof(*entry)); + + entry->index2 = (index >> 15) & 0x1; + entry->zero = 0; + entry->format = 1; + entry->index = (index & 0x7fff); + /* + * IO-APIC RTE will be configured with virtual vector. + * irq handler will do the explicit EOI to the io-apic. + */ + entry->vector = attr->ioapic_pin; + entry->mask = 0; /* enable IRQ */ + entry->trigger = attr->trigger; + entry->polarity = attr->polarity; + + /* Mask level triggered irqs. + * Use IRQ_DELAYED_DISABLE for edge triggered irqs. + */ + if (attr->trigger) + entry->mask = 1; + + return 0; +} + + struct irq_remap_ops intel_irq_remap_ops = { .supported = intel_intr_remapping_supported, .hardware_init = dmar_table_init, @@ -821,4 +909,5 @@ struct irq_remap_ops intel_irq_remap_ops = { .hardware_disable = disable_intr_remapping, .hardware_reenable = reenable_intr_remapping, .enable_faulting = enable_drhd_fault_handling, + .setup_ioapic_entry = intel_setup_ioapic_entry, }; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 9aabed7c0320..739148ab2538 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -98,3 +98,15 @@ int __init intr_enable_fault_handling(void) return remap_ops->enable_faulting(); } + +int intr_setup_ioapic_entry(int irq, + struct IO_APIC_route_entry *entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) +{ + if (!remap_ops || !remap_ops->setup_ioapic_entry) + return -ENODEV; + + return remap_ops->setup_ioapic_entry(irq, entry, destination, + vector, attr); +} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h index 2744c9ae4aec..e8994f2b3bbe 100644 --- a/drivers/iommu/intr_remapping.h +++ b/drivers/iommu/intr_remapping.h @@ -24,6 +24,9 @@ #ifdef CONFIG_IRQ_REMAP +struct IO_APIC_route_entry; +struct io_apic_irq_attr; + extern int disable_intremap; extern int disable_sourceid_checking; extern int no_x2apic_optout; @@ -46,6 +49,11 @@ struct irq_remap_ops { /* Enable fault handling */ int (*enable_faulting)(void); + + /* IO-APIC setup routine */ + int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *, + unsigned int, int, + struct io_apic_irq_attr *); }; extern struct irq_remap_ops intel_irq_remap_ops; -- cgit v1.2.3 From 4c1bad6a0af1e297c8d05365e65af89d8c7bf9d1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:03 -0700 Subject: iommu/vt-d: Convert IR set_affinity function to remap_ops The function to set interrupt affinity with interrupt remapping enabled is Intel specific too. So move it to the irq_remap_ops too. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/intel_intr_remapping.c | 54 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.c | 9 ++++++ drivers/iommu/intr_remapping.h | 6 ++++ 3 files changed, 69 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index f495eba4b6ab..25372c1f3c8c 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -901,6 +901,59 @@ static int intel_setup_ioapic_entry(int irq, return 0; } +/* + * Migrate the IO-APIC irq in the presence of intr-remapping. + * + * For both level and edge triggered, irq migration is a simple atomic + * update(of vector and cpu destination) of IRTE and flush the hardware cache. + * + * For level triggered, we eliminate the io-apic RTE modification (with the + * updated vector information), by using a virtual vector (io-apic pin number). + * Real vector that is used for interrupting cpu will be coming from + * the interrupt-remapping table entry. + * + * As the migration is a simple atomic update of IRTE, the same mechanism + * is used to migrate MSI irq's in the presence of interrupt-remapping. + */ +static int +intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) +{ + struct irq_cfg *cfg = data->chip_data; + unsigned int dest, irq = data->irq; + struct irte irte; + + if (!cpumask_intersects(mask, cpu_online_mask)) + return -EINVAL; + + if (get_irte(irq, &irte)) + return -EBUSY; + + if (assign_irq_vector(irq, cfg, mask)) + return -EBUSY; + + dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); + + irte.vector = cfg->vector; + irte.dest_id = IRTE_DEST(dest); + + /* + * Atomically updates the IRTE with the new destination, vector + * and flushes the interrupt entry cache. + */ + modify_irte(irq, &irte); + + /* + * After this point, all the interrupts will start arriving + * at the new destination. So, time to cleanup the previous + * vector allocation. + */ + if (cfg->move_in_progress) + send_cleanup_vector(cfg); + + cpumask_copy(data->affinity, mask); + return 0; +} struct irq_remap_ops intel_irq_remap_ops = { .supported = intel_intr_remapping_supported, @@ -910,4 +963,5 @@ struct irq_remap_ops intel_irq_remap_ops = { .hardware_reenable = reenable_intr_remapping, .enable_faulting = enable_drhd_fault_handling, .setup_ioapic_entry = intel_setup_ioapic_entry, + .set_affinity = intel_ioapic_set_affinity, }; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 739148ab2538..2f4f27ffb861 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -110,3 +110,12 @@ int intr_setup_ioapic_entry(int irq, return remap_ops->setup_ioapic_entry(irq, entry, destination, vector, attr); } + +int intr_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) +{ + if (!remap_ops || !remap_ops->set_affinity) + return 0; + + return remap_ops->set_affinity(data, mask, force); +} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h index e8994f2b3bbe..e0bc6e0ba1fb 100644 --- a/drivers/iommu/intr_remapping.h +++ b/drivers/iommu/intr_remapping.h @@ -26,6 +26,8 @@ struct IO_APIC_route_entry; struct io_apic_irq_attr; +struct irq_data; +struct cpumask; extern int disable_intremap; extern int disable_sourceid_checking; @@ -54,6 +56,10 @@ struct irq_remap_ops { int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *, unsigned int, int, struct io_apic_irq_attr *); + + /* Set the CPU affinity of a remapped interrupt */ + int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, + bool force); }; extern struct irq_remap_ops intel_irq_remap_ops; -- cgit v1.2.3 From 9d619f65722236e0e0c35467d1528caed206e439 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:04 -0700 Subject: iommu/vt-d: Convert free_irte into a remap_ops callback The operation for releasing a remapping entry is iommu specific too. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/intel_intr_remapping.c | 3 ++- drivers/iommu/intr_remapping.c | 8 ++++++++ drivers/iommu/intr_remapping.h | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index 25372c1f3c8c..44a6e04a070b 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -253,7 +253,7 @@ static int clear_entries(struct irq_2_iommu *irq_iommu) return qi_flush_iec(iommu, index, irq_iommu->irte_mask); } -int free_irte(int irq) +static int free_irte(int irq) { struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); unsigned long flags; @@ -964,4 +964,5 @@ struct irq_remap_ops intel_irq_remap_ops = { .enable_faulting = enable_drhd_fault_handling, .setup_ioapic_entry = intel_setup_ioapic_entry, .set_affinity = intel_ioapic_set_affinity, + .free_irq = free_irte, }; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 2f4f27ffb861..a68d304f9729 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -119,3 +119,11 @@ int intr_set_affinity(struct irq_data *data, const struct cpumask *mask, return remap_ops->set_affinity(data, mask, force); } + +void intr_free_irq(int irq) +{ + if (!remap_ops || !remap_ops->free_irq) + return; + + remap_ops->free_irq(irq); +} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h index e0bc6e0ba1fb..57485539383d 100644 --- a/drivers/iommu/intr_remapping.h +++ b/drivers/iommu/intr_remapping.h @@ -60,6 +60,9 @@ struct irq_remap_ops { /* Set the CPU affinity of a remapped interrupt */ int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, bool force); + + /* Free an IRQ */ + int (*free_irq)(int); }; extern struct irq_remap_ops intel_irq_remap_ops; -- cgit v1.2.3 From 5e2b930b0784a30c98dee8e9d79c1f84c31f7209 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:05 -0700 Subject: iommu/vt-d: Convert MSI remapping setup to remap_ops This patch introduces remapping-ops for setting ups MSI interrupts. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/intel_intr_remapping.c | 97 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.c | 35 +++++++++++++ drivers/iommu/intr_remapping.h | 16 ++++++ 3 files changed, 148 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index 44a6e04a070b..a3bae67ec43c 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "intr_remapping.h" @@ -955,6 +956,98 @@ intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, return 0; } +static void intel_compose_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ + struct irq_cfg *cfg; + struct irte irte; + u16 sub_handle; + int ir_index; + + cfg = irq_get_chip_data(irq); + + ir_index = map_irq_to_irte_handle(irq, &sub_handle); + BUG_ON(ir_index == -1); + + prepare_irte(&irte, cfg->vector, dest); + + /* Set source-id of interrupt request */ + if (pdev) + set_msi_sid(&irte, pdev); + else + set_hpet_sid(&irte, hpet_id); + + modify_irte(irq, &irte); + + msg->address_hi = MSI_ADDR_BASE_HI; + msg->data = sub_handle; + msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | + MSI_ADDR_IR_SHV | + MSI_ADDR_IR_INDEX1(ir_index) | + MSI_ADDR_IR_INDEX2(ir_index); +} + +/* + * Map the PCI dev to the corresponding remapping hardware unit + * and allocate 'nvec' consecutive interrupt-remapping table entries + * in it. + */ +static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) +{ + struct intel_iommu *iommu; + int index; + + iommu = map_dev_to_ir(dev); + if (!iommu) { + printk(KERN_ERR + "Unable to map PCI %s to iommu\n", pci_name(dev)); + return -ENOENT; + } + + index = alloc_irte(iommu, irq, nvec); + if (index < 0) { + printk(KERN_ERR + "Unable to allocate %d IRTE for PCI %s\n", nvec, + pci_name(dev)); + return -ENOSPC; + } + return index; +} + +static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + struct intel_iommu *iommu; + + iommu = map_dev_to_ir(pdev); + if (!iommu) + return -ENOENT; + /* + * setup the mapping between the irq and the IRTE + * base index, the sub_handle pointing to the + * appropriate interrupt remap table entry. + */ + set_irte_irq(irq, iommu, index, sub_handle); + + return 0; +} + +static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) +{ + struct intel_iommu *iommu = map_hpet_to_ir(id); + int index; + + if (!iommu) + return -1; + + index = alloc_irte(iommu, irq, 1); + if (index < 0) + return -1; + + return 0; +} + struct irq_remap_ops intel_irq_remap_ops = { .supported = intel_intr_remapping_supported, .hardware_init = dmar_table_init, @@ -965,4 +1058,8 @@ struct irq_remap_ops intel_irq_remap_ops = { .setup_ioapic_entry = intel_setup_ioapic_entry, .set_affinity = intel_ioapic_set_affinity, .free_irq = free_irte, + .compose_msi_msg = intel_compose_msi_msg, + .msi_alloc_irq = intel_msi_alloc_irq, + .msi_setup_irq = intel_msi_setup_irq, + .setup_hpet_msi = intel_setup_hpet_msi, }; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index a68d304f9729..9dc179316ba1 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -127,3 +127,38 @@ void intr_free_irq(int irq) remap_ops->free_irq(irq); } + +void intr_compose_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ + if (!remap_ops || !remap_ops->compose_msi_msg) + return; + + remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); +} + +int intr_msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec) +{ + if (!remap_ops || !remap_ops->msi_alloc_irq) + return -ENODEV; + + return remap_ops->msi_alloc_irq(pdev, irq, nvec); +} + +int intr_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + if (!remap_ops || !remap_ops->msi_setup_irq) + return -ENODEV; + + return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle); +} + +int intr_setup_hpet_msi(unsigned int irq, unsigned int id) +{ + if (!remap_ops || !remap_ops->setup_hpet_msi) + return -ENODEV; + + return remap_ops->setup_hpet_msi(irq, id); +} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h index 57485539383d..6f4ea0a387b1 100644 --- a/drivers/iommu/intr_remapping.h +++ b/drivers/iommu/intr_remapping.h @@ -28,6 +28,8 @@ struct IO_APIC_route_entry; struct io_apic_irq_attr; struct irq_data; struct cpumask; +struct pci_dev; +struct msi_msg; extern int disable_intremap; extern int disable_sourceid_checking; @@ -63,6 +65,20 @@ struct irq_remap_ops { /* Free an IRQ */ int (*free_irq)(int); + + /* Create MSI msg to use for interrupt remapping */ + void (*compose_msi_msg)(struct pci_dev *, + unsigned int, unsigned int, + struct msi_msg *, u8); + + /* Allocate remapping resources for MSI */ + int (*msi_alloc_irq)(struct pci_dev *, int, int); + + /* Setup the remapped MSI irq */ + int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int); + + /* Setup interrupt remapping for an HPET MSI */ + int (*setup_hpet_msi)(unsigned int, unsigned int); }; extern struct irq_remap_ops intel_irq_remap_ops; -- cgit v1.2.3 From 263b5e8629c9ce21c9cd4c0e29c097afb1c10ef3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Mar 2012 11:47:06 -0700 Subject: x86, iommu/vt-d: Clean up interfaces for interrupt remapping Remove the Intel specific interfaces from dmar.h and remove asm/irq_remapping.h which is only used for io_apic.c anyway. Signed-off-by: Joerg Roedel Acked-by: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Suresh Siddha Signed-off-by: Joerg Roedel --- drivers/iommu/intel_intr_remapping.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index a3bae67ec43c..7472634df350 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -64,7 +64,7 @@ int get_irte(int irq, struct irte *entry) return 0; } -int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) +static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) { struct ir_table *table = iommu->ir_table; struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); @@ -136,7 +136,7 @@ static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) return qi_submit_sync(&desc, iommu); } -int map_irq_to_irte_handle(int irq, u16 *sub_handle) +static int map_irq_to_irte_handle(int irq, u16 *sub_handle) { struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); unsigned long flags; @@ -152,7 +152,7 @@ int map_irq_to_irte_handle(int irq, u16 *sub_handle) return index; } -int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) +static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) { struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); unsigned long flags; @@ -172,7 +172,7 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) return 0; } -int modify_irte(int irq, struct irte *irte_modified) +static int modify_irte(int irq, struct irte *irte_modified) { struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); struct intel_iommu *iommu; @@ -200,7 +200,7 @@ int modify_irte(int irq, struct irte *irte_modified) return rc; } -struct intel_iommu *map_hpet_to_ir(u8 hpet_id) +static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) { int i; @@ -210,7 +210,7 @@ struct intel_iommu *map_hpet_to_ir(u8 hpet_id) return NULL; } -struct intel_iommu *map_ioapic_to_ir(int apic) +static struct intel_iommu *map_ioapic_to_ir(int apic) { int i; @@ -220,7 +220,7 @@ struct intel_iommu *map_ioapic_to_ir(int apic) return NULL; } -struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) +static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) { struct dmar_drhd_unit *drhd; @@ -312,7 +312,7 @@ static void set_irte_sid(struct irte *irte, unsigned int svt, irte->sid = sid; } -int set_ioapic_sid(struct irte *irte, int apic) +static int set_ioapic_sid(struct irte *irte, int apic) { int i; u16 sid = 0; @@ -337,7 +337,7 @@ int set_ioapic_sid(struct irte *irte, int apic) return 0; } -int set_hpet_sid(struct irte *irte, u8 id) +static int set_hpet_sid(struct irte *irte, u8 id) { int i; u16 sid = 0; @@ -367,7 +367,7 @@ int set_hpet_sid(struct irte *irte, u8 id) return 0; } -int set_msi_sid(struct irte *irte, struct pci_dev *dev) +static int set_msi_sid(struct irte *irte, struct pci_dev *dev) { struct pci_dev *bridge; -- cgit v1.2.3 From 95a02e976c39d63716b8c7c226bc530a2041536f Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Fri, 30 Mar 2012 11:47:07 -0700 Subject: iommu: rename intr_remapping references to irq_remapping Make the code consistent with the naming conventions of irq subsystem. Signed-off-by: Suresh Siddha Cc: Joerg Roedel Cc: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 8 ++-- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/intel_intr_remapping.c | 40 +++++++++---------- drivers/iommu/intr_remapping.c | 74 ++++++++++++++++++------------------ drivers/iommu/intr_remapping.h | 10 ++--- 5 files changed, 67 insertions(+), 67 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 647e366403dc..ee74f698eef8 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -556,7 +556,7 @@ int __init detect_intel_iommu(void) dmar = (struct acpi_table_dmar *) dmar_tbl; - if (ret && intr_remapping_enabled && cpu_has_x2apic && + if (ret && irq_remapping_enabled && cpu_has_x2apic && dmar->flags & 0x1) printk(KERN_INFO "Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); @@ -1042,7 +1042,7 @@ static const char *dma_remap_fault_reasons[] = "non-zero reserved fields in PTE", }; -static const char *intr_remap_fault_reasons[] = +static const char *irq_remap_fault_reasons[] = { "Detected reserved fields in the decoded interrupt-remapped request", "Interrupt index exceeded the interrupt-remapping table size", @@ -1058,9 +1058,9 @@ static const char *intr_remap_fault_reasons[] = const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type) { if (fault_reason >= 0x20 && (fault_reason <= 0x20 + - ARRAY_SIZE(intr_remap_fault_reasons))) { + ARRAY_SIZE(irq_remap_fault_reasons))) { *fault_type = INTR_REMAP; - return intr_remap_fault_reasons[fault_reason - 0x20]; + return irq_remap_fault_reasons[fault_reason - 0x20]; } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) { *fault_type = DMA_REMAP; return dma_remap_fault_reasons[fault_reason]; diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index e1439808192c..cef5b8226f3d 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4083,7 +4083,7 @@ static int intel_iommu_domain_has_cap(struct iommu_domain *domain, if (cap == IOMMU_CAP_CACHE_COHERENCY) return dmar_domain->iommu_snooping; if (cap == IOMMU_CAP_INTR_REMAP) - return intr_remapping_enabled; + return irq_remapping_enabled; return 0; } diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c index 7472634df350..efeb601c782f 100644 --- a/drivers/iommu/intel_intr_remapping.c +++ b/drivers/iommu/intel_intr_remapping.c @@ -394,7 +394,7 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev) return 0; } -static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) +static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) { u64 addr; u32 sts; @@ -434,7 +434,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) } -static int intel_setup_intr_remapping(struct intel_iommu *iommu, int mode) +static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode) { struct ir_table *ir_table; struct page *pages; @@ -457,14 +457,14 @@ static int intel_setup_intr_remapping(struct intel_iommu *iommu, int mode) ir_table->base = page_address(pages); - iommu_set_intr_remapping(iommu, mode); + iommu_set_irq_remapping(iommu, mode); return 0; } /* * Disable Interrupt Remapping. */ -static void iommu_disable_intr_remapping(struct intel_iommu *iommu) +static void iommu_disable_irq_remapping(struct intel_iommu *iommu) { unsigned long flags; u32 sts; @@ -503,11 +503,11 @@ static int __init dmar_x2apic_optout(void) return dmar->flags & DMAR_X2APIC_OPT_OUT; } -static int __init intel_intr_remapping_supported(void) +static int __init intel_irq_remapping_supported(void) { struct dmar_drhd_unit *drhd; - if (disable_intremap) + if (disable_irq_remap) return 0; if (!dmar_ir_support()) @@ -523,7 +523,7 @@ static int __init intel_intr_remapping_supported(void) return 1; } -static int __init intel_enable_intr_remapping(void) +static int __init intel_enable_irq_remapping(void) { struct dmar_drhd_unit *drhd; int setup = 0; @@ -561,7 +561,7 @@ static int __init intel_enable_intr_remapping(void) * Disable intr remapping and queued invalidation, if already * enabled prior to OS handover. */ - iommu_disable_intr_remapping(iommu); + iommu_disable_irq_remapping(iommu); dmar_disable_qi(iommu); } @@ -607,7 +607,7 @@ static int __init intel_enable_intr_remapping(void) if (!ecap_ir_support(iommu->ecap)) continue; - if (intel_setup_intr_remapping(iommu, eim)) + if (intel_setup_irq_remapping(iommu, eim)) goto error; setup = 1; @@ -616,7 +616,7 @@ static int __init intel_enable_intr_remapping(void) if (!setup) goto error; - intr_remapping_enabled = 1; + irq_remapping_enabled = 1; pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; @@ -759,14 +759,14 @@ int __init parse_ioapics_under_ir(void) int __init ir_dev_scope_init(void) { - if (!intr_remapping_enabled) + if (!irq_remapping_enabled) return 0; return dmar_dev_scope_init(); } rootfs_initcall(ir_dev_scope_init); -static void disable_intr_remapping(void) +static void disable_irq_remapping(void) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu = NULL; @@ -778,11 +778,11 @@ static void disable_intr_remapping(void) if (!ecap_ir_support(iommu->ecap)) continue; - iommu_disable_intr_remapping(iommu); + iommu_disable_irq_remapping(iommu); } } -static int reenable_intr_remapping(int eim) +static int reenable_irq_remapping(int eim) { struct dmar_drhd_unit *drhd; int setup = 0; @@ -800,7 +800,7 @@ static int reenable_intr_remapping(int eim) continue; /* Set up interrupt remapping for iommu.*/ - iommu_set_intr_remapping(iommu, eim); + iommu_set_irq_remapping(iommu, eim); setup = 1; } @@ -1049,11 +1049,11 @@ static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) } struct irq_remap_ops intel_irq_remap_ops = { - .supported = intel_intr_remapping_supported, - .hardware_init = dmar_table_init, - .hardware_enable = intel_enable_intr_remapping, - .hardware_disable = disable_intr_remapping, - .hardware_reenable = reenable_intr_remapping, + .supported = intel_irq_remapping_supported, + .prepare = dmar_table_init, + .enable = intel_enable_irq_remapping, + .disable = disable_irq_remapping, + .reenable = reenable_irq_remapping, .enable_faulting = enable_drhd_fault_handling, .setup_ioapic_entry = intel_setup_ioapic_entry, .set_affinity = intel_ioapic_set_affinity, diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 9dc179316ba1..523a7b3a1205 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -4,9 +4,9 @@ #include "intr_remapping.h" -int intr_remapping_enabled; +int irq_remapping_enabled; -int disable_intremap; +int disable_irq_remap; int disable_sourceid_checking; int no_x2apic_optout; @@ -14,21 +14,21 @@ static struct irq_remap_ops *remap_ops; static __init int setup_nointremap(char *str) { - disable_intremap = 1; + disable_irq_remap = 1; return 0; } early_param("nointremap", setup_nointremap); -static __init int setup_intremap(char *str) +static __init int setup_irqremap(char *str) { if (!str) return -EINVAL; while (*str) { if (!strncmp(str, "on", 2)) - disable_intremap = 0; + disable_irq_remap = 0; else if (!strncmp(str, "off", 3)) - disable_intremap = 1; + disable_irq_remap = 1; else if (!strncmp(str, "nosid", 5)) disable_sourceid_checking = 1; else if (!strncmp(str, "no_x2apic_optout", 16)) @@ -41,16 +41,16 @@ static __init int setup_intremap(char *str) return 0; } -early_param("intremap", setup_intremap); +early_param("intremap", setup_irqremap); -void __init setup_intr_remapping(void) +void __init setup_irq_remapping_ops(void) { remap_ops = &intel_irq_remap_ops; } -int intr_remapping_supported(void) +int irq_remapping_supported(void) { - if (disable_intremap) + if (disable_irq_remap) return 0; if (!remap_ops || !remap_ops->supported) @@ -59,39 +59,39 @@ int intr_remapping_supported(void) return remap_ops->supported(); } -int __init intr_hardware_init(void) +int __init irq_remapping_prepare(void) { - if (!remap_ops || !remap_ops->hardware_init) + if (!remap_ops || !remap_ops->prepare) return -ENODEV; - return remap_ops->hardware_init(); + return remap_ops->prepare(); } -int __init intr_hardware_enable(void) +int __init irq_remapping_enable(void) { - if (!remap_ops || !remap_ops->hardware_enable) + if (!remap_ops || !remap_ops->enable) return -ENODEV; - return remap_ops->hardware_enable(); + return remap_ops->enable(); } -void intr_hardware_disable(void) +void irq_remapping_disable(void) { - if (!remap_ops || !remap_ops->hardware_disable) + if (!remap_ops || !remap_ops->disable) return; - remap_ops->hardware_disable(); + remap_ops->disable(); } -int intr_hardware_reenable(int mode) +int irq_remapping_reenable(int mode) { - if (!remap_ops || !remap_ops->hardware_reenable) + if (!remap_ops || !remap_ops->reenable) return 0; - return remap_ops->hardware_reenable(mode); + return remap_ops->reenable(mode); } -int __init intr_enable_fault_handling(void) +int __init irq_remap_enable_fault_handling(void) { if (!remap_ops || !remap_ops->enable_faulting) return -ENODEV; @@ -99,10 +99,10 @@ int __init intr_enable_fault_handling(void) return remap_ops->enable_faulting(); } -int intr_setup_ioapic_entry(int irq, - struct IO_APIC_route_entry *entry, - unsigned int destination, int vector, - struct io_apic_irq_attr *attr) +int setup_ioapic_remapped_entry(int irq, + struct IO_APIC_route_entry *entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) { if (!remap_ops || !remap_ops->setup_ioapic_entry) return -ENODEV; @@ -111,8 +111,8 @@ int intr_setup_ioapic_entry(int irq, vector, attr); } -int intr_set_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) +int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) { if (!remap_ops || !remap_ops->set_affinity) return 0; @@ -120,7 +120,7 @@ int intr_set_affinity(struct irq_data *data, const struct cpumask *mask, return remap_ops->set_affinity(data, mask, force); } -void intr_free_irq(int irq) +void free_remapped_irq(int irq) { if (!remap_ops || !remap_ops->free_irq) return; @@ -128,9 +128,9 @@ void intr_free_irq(int irq) remap_ops->free_irq(irq); } -void intr_compose_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id) +void compose_remapped_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) { if (!remap_ops || !remap_ops->compose_msi_msg) return; @@ -138,7 +138,7 @@ void intr_compose_msi_msg(struct pci_dev *pdev, remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); } -int intr_msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec) +int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) { if (!remap_ops || !remap_ops->msi_alloc_irq) return -ENODEV; @@ -146,8 +146,8 @@ int intr_msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec) return remap_ops->msi_alloc_irq(pdev, irq, nvec); } -int intr_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) +int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) { if (!remap_ops || !remap_ops->msi_setup_irq) return -ENODEV; @@ -155,7 +155,7 @@ int intr_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle); } -int intr_setup_hpet_msi(unsigned int irq, unsigned int id) +int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) { if (!remap_ops || !remap_ops->setup_hpet_msi) return -ENODEV; diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h index 6f4ea0a387b1..bd5d98fec148 100644 --- a/drivers/iommu/intr_remapping.h +++ b/drivers/iommu/intr_remapping.h @@ -31,7 +31,7 @@ struct cpumask; struct pci_dev; struct msi_msg; -extern int disable_intremap; +extern int disable_irq_remap; extern int disable_sourceid_checking; extern int no_x2apic_optout; @@ -40,16 +40,16 @@ struct irq_remap_ops { int (*supported)(void); /* Initializes hardware and makes it ready for remapping interrupts */ - int (*hardware_init)(void); + int (*prepare)(void); /* Enables the remapping hardware */ - int (*hardware_enable)(void); + int (*enable)(void); /* Disables the remapping hardware */ - void (*hardware_disable)(void); + void (*disable)(void); /* Reenables the remapping hardware */ - int (*hardware_reenable)(int); + int (*reenable)(int); /* Enable fault handling */ int (*enable_faulting)(void); -- cgit v1.2.3 From 8a8f422d3b4f2cde8e0e1d31638279a26a886a82 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Fri, 30 Mar 2012 11:47:08 -0700 Subject: iommu: rename intr_remapping.[ch] to irq_remapping.[ch] Make the file names consistent with the naming conventions of irq subsystem. Signed-off-by: Suresh Siddha Cc: Joerg Roedel Cc: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Joerg Roedel --- drivers/iommu/Makefile | 2 +- drivers/iommu/dmar.c | 2 +- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/intel_intr_remapping.c | 1065 ---------------------------------- drivers/iommu/intel_irq_remapping.c | 1065 ++++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.c | 164 ------ drivers/iommu/intr_remapping.h | 88 --- drivers/iommu/irq_remapping.c | 164 ++++++ drivers/iommu/irq_remapping.h | 88 +++ 9 files changed, 1320 insertions(+), 1320 deletions(-) delete mode 100644 drivers/iommu/intel_intr_remapping.c create mode 100644 drivers/iommu/intel_irq_remapping.c delete mode 100644 drivers/iommu/intr_remapping.c delete mode 100644 drivers/iommu/intr_remapping.h create mode 100644 drivers/iommu/irq_remapping.c create mode 100644 drivers/iommu/irq_remapping.h (limited to 'drivers') diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 823e1cf8708f..3e5e82ae9f0d 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o -obj-$(CONFIG_IRQ_REMAP) += intel_intr_remapping.o intr_remapping.o +obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index ee74f698eef8..5ef65cf66152 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #define PREFIX "DMAR: " diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index cef5b8226f3d..bf2fbaad5e22 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c deleted file mode 100644 index efeb601c782f..000000000000 --- a/drivers/iommu/intel_intr_remapping.c +++ /dev/null @@ -1,1065 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "intr_remapping.h" - -struct ioapic_scope { - struct intel_iommu *iommu; - unsigned int id; - unsigned int bus; /* PCI bus number */ - unsigned int devfn; /* PCI devfn number */ -}; - -struct hpet_scope { - struct intel_iommu *iommu; - u8 id; - unsigned int bus; - unsigned int devfn; -}; - -#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) -#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8) - -static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; -static struct hpet_scope ir_hpet[MAX_HPET_TBS]; -static int ir_ioapic_num, ir_hpet_num; - -static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); - -static struct irq_2_iommu *irq_2_iommu(unsigned int irq) -{ - struct irq_cfg *cfg = irq_get_chip_data(irq); - return cfg ? &cfg->irq_2_iommu : NULL; -} - -int get_irte(int irq, struct irte *entry) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int index; - - if (!entry || !irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - index = irq_iommu->irte_index + irq_iommu->sub_handle; - *entry = *(irq_iommu->iommu->ir_table->base + index); - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - return 0; -} - -static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) -{ - struct ir_table *table = iommu->ir_table; - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - u16 index, start_index; - unsigned int mask = 0; - unsigned long flags; - int i; - - if (!count || !irq_iommu) - return -1; - - /* - * start the IRTE search from index 0. - */ - index = start_index = 0; - - if (count > 1) { - count = __roundup_pow_of_two(count); - mask = ilog2(count); - } - - if (mask > ecap_max_handle_mask(iommu->ecap)) { - printk(KERN_ERR - "Requested mask %x exceeds the max invalidation handle" - " mask value %Lx\n", mask, - ecap_max_handle_mask(iommu->ecap)); - return -1; - } - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - do { - for (i = index; i < index + count; i++) - if (table->base[i].present) - break; - /* empty index found */ - if (i == index + count) - break; - - index = (index + count) % INTR_REMAP_TABLE_ENTRIES; - - if (index == start_index) { - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - printk(KERN_ERR "can't allocate an IRTE\n"); - return -1; - } - } while (1); - - for (i = index; i < index + count; i++) - table->base[i].present = 1; - - irq_iommu->iommu = iommu; - irq_iommu->irte_index = index; - irq_iommu->sub_handle = 0; - irq_iommu->irte_mask = mask; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return index; -} - -static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) -{ - struct qi_desc desc; - - desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) - | QI_IEC_SELECTIVE; - desc.high = 0; - - return qi_submit_sync(&desc, iommu); -} - -static int map_irq_to_irte_handle(int irq, u16 *sub_handle) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int index; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - *sub_handle = irq_iommu->sub_handle; - index = irq_iommu->irte_index; - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - return index; -} - -static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - irq_iommu->iommu = iommu; - irq_iommu->irte_index = index; - irq_iommu->sub_handle = subhandle; - irq_iommu->irte_mask = 0; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return 0; -} - -static int modify_irte(int irq, struct irte *irte_modified) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - struct intel_iommu *iommu; - unsigned long flags; - struct irte *irte; - int rc, index; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - iommu = irq_iommu->iommu; - - index = irq_iommu->irte_index + irq_iommu->sub_handle; - irte = &iommu->ir_table->base[index]; - - set_64bit(&irte->low, irte_modified->low); - set_64bit(&irte->high, irte_modified->high); - __iommu_flush_cache(iommu, irte, sizeof(*irte)); - - rc = qi_flush_iec(iommu, index, 0); - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return rc; -} - -static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) -{ - int i; - - for (i = 0; i < MAX_HPET_TBS; i++) - if (ir_hpet[i].id == hpet_id) - return ir_hpet[i].iommu; - return NULL; -} - -static struct intel_iommu *map_ioapic_to_ir(int apic) -{ - int i; - - for (i = 0; i < MAX_IO_APICS; i++) - if (ir_ioapic[i].id == apic) - return ir_ioapic[i].iommu; - return NULL; -} - -static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) -{ - struct dmar_drhd_unit *drhd; - - drhd = dmar_find_matched_drhd_unit(dev); - if (!drhd) - return NULL; - - return drhd->iommu; -} - -static int clear_entries(struct irq_2_iommu *irq_iommu) -{ - struct irte *start, *entry, *end; - struct intel_iommu *iommu; - int index; - - if (irq_iommu->sub_handle) - return 0; - - iommu = irq_iommu->iommu; - index = irq_iommu->irte_index + irq_iommu->sub_handle; - - start = iommu->ir_table->base + index; - end = start + (1 << irq_iommu->irte_mask); - - for (entry = start; entry < end; entry++) { - set_64bit(&entry->low, 0); - set_64bit(&entry->high, 0); - } - - return qi_flush_iec(iommu, index, irq_iommu->irte_mask); -} - -static int free_irte(int irq) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int rc; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - rc = clear_entries(irq_iommu); - - irq_iommu->iommu = NULL; - irq_iommu->irte_index = 0; - irq_iommu->sub_handle = 0; - irq_iommu->irte_mask = 0; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return rc; -} - -/* - * source validation type - */ -#define SVT_NO_VERIFY 0x0 /* no verification is required */ -#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ -#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ - -/* - * source-id qualifier - */ -#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ -#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore - * the third least significant bit - */ -#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore - * the second and third least significant bits - */ -#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore - * the least three significant bits - */ - -/* - * set SVT, SQ and SID fields of irte to verify - * source ids of interrupt requests - */ -static void set_irte_sid(struct irte *irte, unsigned int svt, - unsigned int sq, unsigned int sid) -{ - if (disable_sourceid_checking) - svt = SVT_NO_VERIFY; - irte->svt = svt; - irte->sq = sq; - irte->sid = sid; -} - -static int set_ioapic_sid(struct irte *irte, int apic) -{ - int i; - u16 sid = 0; - - if (!irte) - return -1; - - for (i = 0; i < MAX_IO_APICS; i++) { - if (ir_ioapic[i].id == apic) { - sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; - break; - } - } - - if (sid == 0) { - pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); - return -1; - } - - set_irte_sid(irte, 1, 0, sid); - - return 0; -} - -static int set_hpet_sid(struct irte *irte, u8 id) -{ - int i; - u16 sid = 0; - - if (!irte) - return -1; - - for (i = 0; i < MAX_HPET_TBS; i++) { - if (ir_hpet[i].id == id) { - sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; - break; - } - } - - if (sid == 0) { - pr_warning("Failed to set source-id of HPET block (%d)\n", id); - return -1; - } - - /* - * Should really use SQ_ALL_16. Some platforms are broken. - * While we figure out the right quirks for these broken platforms, use - * SQ_13_IGNORE_3 for now. - */ - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); - - return 0; -} - -static int set_msi_sid(struct irte *irte, struct pci_dev *dev) -{ - struct pci_dev *bridge; - - if (!irte || !dev) - return -1; - - /* PCIe device or Root Complex integrated PCI device */ - if (pci_is_pcie(dev) || !dev->bus->parent) { - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, - (dev->bus->number << 8) | dev->devfn); - return 0; - } - - bridge = pci_find_upstream_pcie_bridge(dev); - if (bridge) { - if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */ - set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, - (bridge->bus->number << 8) | dev->bus->number); - else /* this is a legacy PCI bridge */ - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, - (bridge->bus->number << 8) | bridge->devfn); - } - - return 0; -} - -static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) -{ - u64 addr; - u32 sts; - unsigned long flags; - - addr = virt_to_phys((void *)iommu->ir_table->base); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - dmar_writeq(iommu->reg + DMAR_IRTA_REG, - (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); - - /* Set interrupt-remapping table pointer */ - iommu->gcmd |= DMA_GCMD_SIRTP; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, (sts & DMA_GSTS_IRTPS), sts); - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); - - /* - * global invalidation of interrupt entry cache before enabling - * interrupt-remapping. - */ - qi_global_iec(iommu); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - /* Enable interrupt-remapping */ - iommu->gcmd |= DMA_GCMD_IRE; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, (sts & DMA_GSTS_IRES), sts); - - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); -} - - -static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode) -{ - struct ir_table *ir_table; - struct page *pages; - - ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), - GFP_ATOMIC); - - if (!iommu->ir_table) - return -ENOMEM; - - pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, - INTR_REMAP_PAGE_ORDER); - - if (!pages) { - printk(KERN_ERR "failed to allocate pages of order %d\n", - INTR_REMAP_PAGE_ORDER); - kfree(iommu->ir_table); - return -ENOMEM; - } - - ir_table->base = page_address(pages); - - iommu_set_irq_remapping(iommu, mode); - return 0; -} - -/* - * Disable Interrupt Remapping. - */ -static void iommu_disable_irq_remapping(struct intel_iommu *iommu) -{ - unsigned long flags; - u32 sts; - - if (!ecap_ir_support(iommu->ecap)) - return; - - /* - * global invalidation of interrupt entry cache before disabling - * interrupt-remapping. - */ - qi_global_iec(iommu); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); - if (!(sts & DMA_GSTS_IRES)) - goto end; - - iommu->gcmd &= ~DMA_GCMD_IRE; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, !(sts & DMA_GSTS_IRES), sts); - -end: - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); -} - -static int __init dmar_x2apic_optout(void) -{ - struct acpi_table_dmar *dmar; - dmar = (struct acpi_table_dmar *)dmar_tbl; - if (!dmar || no_x2apic_optout) - return 0; - return dmar->flags & DMAR_X2APIC_OPT_OUT; -} - -static int __init intel_irq_remapping_supported(void) -{ - struct dmar_drhd_unit *drhd; - - if (disable_irq_remap) - return 0; - - if (!dmar_ir_support()) - return 0; - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - return 0; - } - - return 1; -} - -static int __init intel_enable_irq_remapping(void) -{ - struct dmar_drhd_unit *drhd; - int setup = 0; - int eim = 0; - - if (parse_ioapics_under_ir() != 1) { - printk(KERN_INFO "Not enable interrupt remapping\n"); - return -1; - } - - if (x2apic_supported()) { - eim = !dmar_x2apic_optout(); - WARN(!eim, KERN_WARNING - "Your BIOS is broken and requested that x2apic be disabled\n" - "This will leave your machine vulnerable to irq-injection attacks\n" - "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); - } - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - /* - * If the queued invalidation is already initialized, - * shouldn't disable it. - */ - if (iommu->qi) - continue; - - /* - * Clear previous faults. - */ - dmar_fault(-1, iommu); - - /* - * Disable intr remapping and queued invalidation, if already - * enabled prior to OS handover. - */ - iommu_disable_irq_remapping(iommu); - - dmar_disable_qi(iommu); - } - - /* - * check for the Interrupt-remapping support - */ - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - continue; - - if (eim && !ecap_eim_support(iommu->ecap)) { - printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " - " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); - return -1; - } - } - - /* - * Enable queued invalidation for all the DRHD's. - */ - for_each_drhd_unit(drhd) { - int ret; - struct intel_iommu *iommu = drhd->iommu; - ret = dmar_enable_qi(iommu); - - if (ret) { - printk(KERN_ERR "DRHD %Lx: failed to enable queued, " - " invalidation, ecap %Lx, ret %d\n", - drhd->reg_base_addr, iommu->ecap, ret); - return -1; - } - } - - /* - * Setup Interrupt-remapping for all the DRHD's now. - */ - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - continue; - - if (intel_setup_irq_remapping(iommu, eim)) - goto error; - - setup = 1; - } - - if (!setup) - goto error; - - irq_remapping_enabled = 1; - pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); - - return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; - -error: - /* - * handle error condition gracefully here! - */ - return -1; -} - -static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, - struct intel_iommu *iommu) -{ - struct acpi_dmar_pci_path *path; - u8 bus; - int count; - - bus = scope->bus; - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (--count > 0) { - /* - * Access PCI directly due to the PCI - * subsystem isn't initialized yet. - */ - bus = read_pci_config_byte(bus, path->dev, path->fn, - PCI_SECONDARY_BUS); - path++; - } - ir_hpet[ir_hpet_num].bus = bus; - ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn); - ir_hpet[ir_hpet_num].iommu = iommu; - ir_hpet[ir_hpet_num].id = scope->enumeration_id; - ir_hpet_num++; -} - -static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, - struct intel_iommu *iommu) -{ - struct acpi_dmar_pci_path *path; - u8 bus; - int count; - - bus = scope->bus; - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (--count > 0) { - /* - * Access PCI directly due to the PCI - * subsystem isn't initialized yet. - */ - bus = read_pci_config_byte(bus, path->dev, path->fn, - PCI_SECONDARY_BUS); - path++; - } - - ir_ioapic[ir_ioapic_num].bus = bus; - ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn); - ir_ioapic[ir_ioapic_num].iommu = iommu; - ir_ioapic[ir_ioapic_num].id = scope->enumeration_id; - ir_ioapic_num++; -} - -static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, - struct intel_iommu *iommu) -{ - struct acpi_dmar_hardware_unit *drhd; - struct acpi_dmar_device_scope *scope; - void *start, *end; - - drhd = (struct acpi_dmar_hardware_unit *)header; - - start = (void *)(drhd + 1); - end = ((void *)drhd) + header->length; - - while (start < end) { - scope = start; - if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { - if (ir_ioapic_num == MAX_IO_APICS) { - printk(KERN_WARNING "Exceeded Max IO APICS\n"); - return -1; - } - - printk(KERN_INFO "IOAPIC id %d under DRHD base " - " 0x%Lx IOMMU %d\n", scope->enumeration_id, - drhd->address, iommu->seq_id); - - ir_parse_one_ioapic_scope(scope, iommu); - } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) { - if (ir_hpet_num == MAX_HPET_TBS) { - printk(KERN_WARNING "Exceeded Max HPET blocks\n"); - return -1; - } - - printk(KERN_INFO "HPET id %d under DRHD base" - " 0x%Lx\n", scope->enumeration_id, - drhd->address); - - ir_parse_one_hpet_scope(scope, iommu); - } - start += scope->length; - } - - return 0; -} - -/* - * Finds the assocaition between IOAPIC's and its Interrupt-remapping - * hardware unit. - */ -int __init parse_ioapics_under_ir(void) -{ - struct dmar_drhd_unit *drhd; - int ir_supported = 0; - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (ecap_ir_support(iommu->ecap)) { - if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) - return -1; - - ir_supported = 1; - } - } - - if (ir_supported && ir_ioapic_num != nr_ioapics) { - printk(KERN_WARNING - "Not all IO-APIC's listed under remapping hardware\n"); - return -1; - } - - return ir_supported; -} - -int __init ir_dev_scope_init(void) -{ - if (!irq_remapping_enabled) - return 0; - - return dmar_dev_scope_init(); -} -rootfs_initcall(ir_dev_scope_init); - -static void disable_irq_remapping(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu = NULL; - - /* - * Disable Interrupt-remapping for all the DRHD's now. - */ - for_each_iommu(iommu, drhd) { - if (!ecap_ir_support(iommu->ecap)) - continue; - - iommu_disable_irq_remapping(iommu); - } -} - -static int reenable_irq_remapping(int eim) -{ - struct dmar_drhd_unit *drhd; - int setup = 0; - struct intel_iommu *iommu = NULL; - - for_each_iommu(iommu, drhd) - if (iommu->qi) - dmar_reenable_qi(iommu); - - /* - * Setup Interrupt-remapping for all the DRHD's now. - */ - for_each_iommu(iommu, drhd) { - if (!ecap_ir_support(iommu->ecap)) - continue; - - /* Set up interrupt remapping for iommu.*/ - iommu_set_irq_remapping(iommu, eim); - setup = 1; - } - - if (!setup) - goto error; - - return 0; - -error: - /* - * handle error condition gracefully here! - */ - return -1; -} - -static void prepare_irte(struct irte *irte, int vector, - unsigned int dest) -{ - memset(irte, 0, sizeof(*irte)); - - irte->present = 1; - irte->dst_mode = apic->irq_dest_mode; - /* - * Trigger mode in the IRTE will always be edge, and for IO-APIC, the - * actual level or edge trigger will be setup in the IO-APIC - * RTE. This will help simplify level triggered irq migration. - * For more details, see the comments (in io_apic.c) explainig IO-APIC - * irq migration in the presence of interrupt-remapping. - */ - irte->trigger_mode = 0; - irte->dlvry_mode = apic->irq_delivery_mode; - irte->vector = vector; - irte->dest_id = IRTE_DEST(dest); - irte->redir_hint = 1; -} - -static int intel_setup_ioapic_entry(int irq, - struct IO_APIC_route_entry *route_entry, - unsigned int destination, int vector, - struct io_apic_irq_attr *attr) -{ - int ioapic_id = mpc_ioapic_id(attr->ioapic); - struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id); - struct IR_IO_APIC_route_entry *entry; - struct irte irte; - int index; - - if (!iommu) { - pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); - return -ENODEV; - } - - entry = (struct IR_IO_APIC_route_entry *)route_entry; - - index = alloc_irte(iommu, irq, 1); - if (index < 0) { - pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id); - return -ENOMEM; - } - - prepare_irte(&irte, vector, destination); - - /* Set source-id of interrupt request */ - set_ioapic_sid(&irte, ioapic_id); - - modify_irte(irq, &irte); - - apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " - "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " - "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " - "Avail:%X Vector:%02X Dest:%08X " - "SID:%04X SQ:%X SVT:%X)\n", - attr->ioapic, irte.present, irte.fpd, irte.dst_mode, - irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, - irte.avail, irte.vector, irte.dest_id, - irte.sid, irte.sq, irte.svt); - - memset(entry, 0, sizeof(*entry)); - - entry->index2 = (index >> 15) & 0x1; - entry->zero = 0; - entry->format = 1; - entry->index = (index & 0x7fff); - /* - * IO-APIC RTE will be configured with virtual vector. - * irq handler will do the explicit EOI to the io-apic. - */ - entry->vector = attr->ioapic_pin; - entry->mask = 0; /* enable IRQ */ - entry->trigger = attr->trigger; - entry->polarity = attr->polarity; - - /* Mask level triggered irqs. - * Use IRQ_DELAYED_DISABLE for edge triggered irqs. - */ - if (attr->trigger) - entry->mask = 1; - - return 0; -} - -/* - * Migrate the IO-APIC irq in the presence of intr-remapping. - * - * For both level and edge triggered, irq migration is a simple atomic - * update(of vector and cpu destination) of IRTE and flush the hardware cache. - * - * For level triggered, we eliminate the io-apic RTE modification (with the - * updated vector information), by using a virtual vector (io-apic pin number). - * Real vector that is used for interrupting cpu will be coming from - * the interrupt-remapping table entry. - * - * As the migration is a simple atomic update of IRTE, the same mechanism - * is used to migrate MSI irq's in the presence of interrupt-remapping. - */ -static int -intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) -{ - struct irq_cfg *cfg = data->chip_data; - unsigned int dest, irq = data->irq; - struct irte irte; - - if (!cpumask_intersects(mask, cpu_online_mask)) - return -EINVAL; - - if (get_irte(irq, &irte)) - return -EBUSY; - - if (assign_irq_vector(irq, cfg, mask)) - return -EBUSY; - - dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); - - irte.vector = cfg->vector; - irte.dest_id = IRTE_DEST(dest); - - /* - * Atomically updates the IRTE with the new destination, vector - * and flushes the interrupt entry cache. - */ - modify_irte(irq, &irte); - - /* - * After this point, all the interrupts will start arriving - * at the new destination. So, time to cleanup the previous - * vector allocation. - */ - if (cfg->move_in_progress) - send_cleanup_vector(cfg); - - cpumask_copy(data->affinity, mask); - return 0; -} - -static void intel_compose_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id) -{ - struct irq_cfg *cfg; - struct irte irte; - u16 sub_handle; - int ir_index; - - cfg = irq_get_chip_data(irq); - - ir_index = map_irq_to_irte_handle(irq, &sub_handle); - BUG_ON(ir_index == -1); - - prepare_irte(&irte, cfg->vector, dest); - - /* Set source-id of interrupt request */ - if (pdev) - set_msi_sid(&irte, pdev); - else - set_hpet_sid(&irte, hpet_id); - - modify_irte(irq, &irte); - - msg->address_hi = MSI_ADDR_BASE_HI; - msg->data = sub_handle; - msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | - MSI_ADDR_IR_SHV | - MSI_ADDR_IR_INDEX1(ir_index) | - MSI_ADDR_IR_INDEX2(ir_index); -} - -/* - * Map the PCI dev to the corresponding remapping hardware unit - * and allocate 'nvec' consecutive interrupt-remapping table entries - * in it. - */ -static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) -{ - struct intel_iommu *iommu; - int index; - - iommu = map_dev_to_ir(dev); - if (!iommu) { - printk(KERN_ERR - "Unable to map PCI %s to iommu\n", pci_name(dev)); - return -ENOENT; - } - - index = alloc_irte(iommu, irq, nvec); - if (index < 0) { - printk(KERN_ERR - "Unable to allocate %d IRTE for PCI %s\n", nvec, - pci_name(dev)); - return -ENOSPC; - } - return index; -} - -static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) -{ - struct intel_iommu *iommu; - - iommu = map_dev_to_ir(pdev); - if (!iommu) - return -ENOENT; - /* - * setup the mapping between the irq and the IRTE - * base index, the sub_handle pointing to the - * appropriate interrupt remap table entry. - */ - set_irte_irq(irq, iommu, index, sub_handle); - - return 0; -} - -static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) -{ - struct intel_iommu *iommu = map_hpet_to_ir(id); - int index; - - if (!iommu) - return -1; - - index = alloc_irte(iommu, irq, 1); - if (index < 0) - return -1; - - return 0; -} - -struct irq_remap_ops intel_irq_remap_ops = { - .supported = intel_irq_remapping_supported, - .prepare = dmar_table_init, - .enable = intel_enable_irq_remapping, - .disable = disable_irq_remapping, - .reenable = reenable_irq_remapping, - .enable_faulting = enable_drhd_fault_handling, - .setup_ioapic_entry = intel_setup_ioapic_entry, - .set_affinity = intel_ioapic_set_affinity, - .free_irq = free_irte, - .compose_msi_msg = intel_compose_msi_msg, - .msi_alloc_irq = intel_msi_alloc_irq, - .msi_setup_irq = intel_msi_setup_irq, - .setup_hpet_msi = intel_setup_hpet_msi, -}; diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c new file mode 100644 index 000000000000..b4d39507681a --- /dev/null +++ b/drivers/iommu/intel_irq_remapping.c @@ -0,0 +1,1065 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "irq_remapping.h" + +struct ioapic_scope { + struct intel_iommu *iommu; + unsigned int id; + unsigned int bus; /* PCI bus number */ + unsigned int devfn; /* PCI devfn number */ +}; + +struct hpet_scope { + struct intel_iommu *iommu; + u8 id; + unsigned int bus; + unsigned int devfn; +}; + +#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) +#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8) + +static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; +static struct hpet_scope ir_hpet[MAX_HPET_TBS]; +static int ir_ioapic_num, ir_hpet_num; + +static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); + +static struct irq_2_iommu *irq_2_iommu(unsigned int irq) +{ + struct irq_cfg *cfg = irq_get_chip_data(irq); + return cfg ? &cfg->irq_2_iommu : NULL; +} + +int get_irte(int irq, struct irte *entry) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int index; + + if (!entry || !irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + index = irq_iommu->irte_index + irq_iommu->sub_handle; + *entry = *(irq_iommu->iommu->ir_table->base + index); + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + return 0; +} + +static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) +{ + struct ir_table *table = iommu->ir_table; + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + u16 index, start_index; + unsigned int mask = 0; + unsigned long flags; + int i; + + if (!count || !irq_iommu) + return -1; + + /* + * start the IRTE search from index 0. + */ + index = start_index = 0; + + if (count > 1) { + count = __roundup_pow_of_two(count); + mask = ilog2(count); + } + + if (mask > ecap_max_handle_mask(iommu->ecap)) { + printk(KERN_ERR + "Requested mask %x exceeds the max invalidation handle" + " mask value %Lx\n", mask, + ecap_max_handle_mask(iommu->ecap)); + return -1; + } + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + do { + for (i = index; i < index + count; i++) + if (table->base[i].present) + break; + /* empty index found */ + if (i == index + count) + break; + + index = (index + count) % INTR_REMAP_TABLE_ENTRIES; + + if (index == start_index) { + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + printk(KERN_ERR "can't allocate an IRTE\n"); + return -1; + } + } while (1); + + for (i = index; i < index + count; i++) + table->base[i].present = 1; + + irq_iommu->iommu = iommu; + irq_iommu->irte_index = index; + irq_iommu->sub_handle = 0; + irq_iommu->irte_mask = mask; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return index; +} + +static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) +{ + struct qi_desc desc; + + desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) + | QI_IEC_SELECTIVE; + desc.high = 0; + + return qi_submit_sync(&desc, iommu); +} + +static int map_irq_to_irte_handle(int irq, u16 *sub_handle) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int index; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + *sub_handle = irq_iommu->sub_handle; + index = irq_iommu->irte_index; + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + return index; +} + +static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + irq_iommu->iommu = iommu; + irq_iommu->irte_index = index; + irq_iommu->sub_handle = subhandle; + irq_iommu->irte_mask = 0; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return 0; +} + +static int modify_irte(int irq, struct irte *irte_modified) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + struct intel_iommu *iommu; + unsigned long flags; + struct irte *irte; + int rc, index; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + iommu = irq_iommu->iommu; + + index = irq_iommu->irte_index + irq_iommu->sub_handle; + irte = &iommu->ir_table->base[index]; + + set_64bit(&irte->low, irte_modified->low); + set_64bit(&irte->high, irte_modified->high); + __iommu_flush_cache(iommu, irte, sizeof(*irte)); + + rc = qi_flush_iec(iommu, index, 0); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return rc; +} + +static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) +{ + int i; + + for (i = 0; i < MAX_HPET_TBS; i++) + if (ir_hpet[i].id == hpet_id) + return ir_hpet[i].iommu; + return NULL; +} + +static struct intel_iommu *map_ioapic_to_ir(int apic) +{ + int i; + + for (i = 0; i < MAX_IO_APICS; i++) + if (ir_ioapic[i].id == apic) + return ir_ioapic[i].iommu; + return NULL; +} + +static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) +{ + struct dmar_drhd_unit *drhd; + + drhd = dmar_find_matched_drhd_unit(dev); + if (!drhd) + return NULL; + + return drhd->iommu; +} + +static int clear_entries(struct irq_2_iommu *irq_iommu) +{ + struct irte *start, *entry, *end; + struct intel_iommu *iommu; + int index; + + if (irq_iommu->sub_handle) + return 0; + + iommu = irq_iommu->iommu; + index = irq_iommu->irte_index + irq_iommu->sub_handle; + + start = iommu->ir_table->base + index; + end = start + (1 << irq_iommu->irte_mask); + + for (entry = start; entry < end; entry++) { + set_64bit(&entry->low, 0); + set_64bit(&entry->high, 0); + } + + return qi_flush_iec(iommu, index, irq_iommu->irte_mask); +} + +static int free_irte(int irq) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int rc; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + rc = clear_entries(irq_iommu); + + irq_iommu->iommu = NULL; + irq_iommu->irte_index = 0; + irq_iommu->sub_handle = 0; + irq_iommu->irte_mask = 0; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return rc; +} + +/* + * source validation type + */ +#define SVT_NO_VERIFY 0x0 /* no verification is required */ +#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ +#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ + +/* + * source-id qualifier + */ +#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ +#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore + * the third least significant bit + */ +#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore + * the second and third least significant bits + */ +#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore + * the least three significant bits + */ + +/* + * set SVT, SQ and SID fields of irte to verify + * source ids of interrupt requests + */ +static void set_irte_sid(struct irte *irte, unsigned int svt, + unsigned int sq, unsigned int sid) +{ + if (disable_sourceid_checking) + svt = SVT_NO_VERIFY; + irte->svt = svt; + irte->sq = sq; + irte->sid = sid; +} + +static int set_ioapic_sid(struct irte *irte, int apic) +{ + int i; + u16 sid = 0; + + if (!irte) + return -1; + + for (i = 0; i < MAX_IO_APICS; i++) { + if (ir_ioapic[i].id == apic) { + sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; + break; + } + } + + if (sid == 0) { + pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); + return -1; + } + + set_irte_sid(irte, 1, 0, sid); + + return 0; +} + +static int set_hpet_sid(struct irte *irte, u8 id) +{ + int i; + u16 sid = 0; + + if (!irte) + return -1; + + for (i = 0; i < MAX_HPET_TBS; i++) { + if (ir_hpet[i].id == id) { + sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; + break; + } + } + + if (sid == 0) { + pr_warning("Failed to set source-id of HPET block (%d)\n", id); + return -1; + } + + /* + * Should really use SQ_ALL_16. Some platforms are broken. + * While we figure out the right quirks for these broken platforms, use + * SQ_13_IGNORE_3 for now. + */ + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); + + return 0; +} + +static int set_msi_sid(struct irte *irte, struct pci_dev *dev) +{ + struct pci_dev *bridge; + + if (!irte || !dev) + return -1; + + /* PCIe device or Root Complex integrated PCI device */ + if (pci_is_pcie(dev) || !dev->bus->parent) { + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, + (dev->bus->number << 8) | dev->devfn); + return 0; + } + + bridge = pci_find_upstream_pcie_bridge(dev); + if (bridge) { + if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */ + set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, + (bridge->bus->number << 8) | dev->bus->number); + else /* this is a legacy PCI bridge */ + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, + (bridge->bus->number << 8) | bridge->devfn); + } + + return 0; +} + +static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) +{ + u64 addr; + u32 sts; + unsigned long flags; + + addr = virt_to_phys((void *)iommu->ir_table->base); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + dmar_writeq(iommu->reg + DMAR_IRTA_REG, + (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); + + /* Set interrupt-remapping table pointer */ + iommu->gcmd |= DMA_GCMD_SIRTP; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_IRTPS), sts); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); + + /* + * global invalidation of interrupt entry cache before enabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + /* Enable interrupt-remapping */ + iommu->gcmd |= DMA_GCMD_IRE; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_IRES), sts); + + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + + +static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode) +{ + struct ir_table *ir_table; + struct page *pages; + + ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), + GFP_ATOMIC); + + if (!iommu->ir_table) + return -ENOMEM; + + pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, + INTR_REMAP_PAGE_ORDER); + + if (!pages) { + printk(KERN_ERR "failed to allocate pages of order %d\n", + INTR_REMAP_PAGE_ORDER); + kfree(iommu->ir_table); + return -ENOMEM; + } + + ir_table->base = page_address(pages); + + iommu_set_irq_remapping(iommu, mode); + return 0; +} + +/* + * Disable Interrupt Remapping. + */ +static void iommu_disable_irq_remapping(struct intel_iommu *iommu) +{ + unsigned long flags; + u32 sts; + + if (!ecap_ir_support(iommu->ecap)) + return; + + /* + * global invalidation of interrupt entry cache before disabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); + if (!(sts & DMA_GSTS_IRES)) + goto end; + + iommu->gcmd &= ~DMA_GCMD_IRE; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, !(sts & DMA_GSTS_IRES), sts); + +end: + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + +static int __init dmar_x2apic_optout(void) +{ + struct acpi_table_dmar *dmar; + dmar = (struct acpi_table_dmar *)dmar_tbl; + if (!dmar || no_x2apic_optout) + return 0; + return dmar->flags & DMAR_X2APIC_OPT_OUT; +} + +static int __init intel_irq_remapping_supported(void) +{ + struct dmar_drhd_unit *drhd; + + if (disable_irq_remap) + return 0; + + if (!dmar_ir_support()) + return 0; + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + return 0; + } + + return 1; +} + +static int __init intel_enable_irq_remapping(void) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + int eim = 0; + + if (parse_ioapics_under_ir() != 1) { + printk(KERN_INFO "Not enable interrupt remapping\n"); + return -1; + } + + if (x2apic_supported()) { + eim = !dmar_x2apic_optout(); + WARN(!eim, KERN_WARNING + "Your BIOS is broken and requested that x2apic be disabled\n" + "This will leave your machine vulnerable to irq-injection attacks\n" + "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); + } + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + /* + * If the queued invalidation is already initialized, + * shouldn't disable it. + */ + if (iommu->qi) + continue; + + /* + * Clear previous faults. + */ + dmar_fault(-1, iommu); + + /* + * Disable intr remapping and queued invalidation, if already + * enabled prior to OS handover. + */ + iommu_disable_irq_remapping(iommu); + + dmar_disable_qi(iommu); + } + + /* + * check for the Interrupt-remapping support + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + continue; + + if (eim && !ecap_eim_support(iommu->ecap)) { + printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " + " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); + return -1; + } + } + + /* + * Enable queued invalidation for all the DRHD's. + */ + for_each_drhd_unit(drhd) { + int ret; + struct intel_iommu *iommu = drhd->iommu; + ret = dmar_enable_qi(iommu); + + if (ret) { + printk(KERN_ERR "DRHD %Lx: failed to enable queued, " + " invalidation, ecap %Lx, ret %d\n", + drhd->reg_base_addr, iommu->ecap, ret); + return -1; + } + } + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + continue; + + if (intel_setup_irq_remapping(iommu, eim)) + goto error; + + setup = 1; + } + + if (!setup) + goto error; + + irq_remapping_enabled = 1; + pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); + + return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + +static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, + struct intel_iommu *iommu) +{ + struct acpi_dmar_pci_path *path; + u8 bus; + int count; + + bus = scope->bus; + path = (struct acpi_dmar_pci_path *)(scope + 1); + count = (scope->length - sizeof(struct acpi_dmar_device_scope)) + / sizeof(struct acpi_dmar_pci_path); + + while (--count > 0) { + /* + * Access PCI directly due to the PCI + * subsystem isn't initialized yet. + */ + bus = read_pci_config_byte(bus, path->dev, path->fn, + PCI_SECONDARY_BUS); + path++; + } + ir_hpet[ir_hpet_num].bus = bus; + ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn); + ir_hpet[ir_hpet_num].iommu = iommu; + ir_hpet[ir_hpet_num].id = scope->enumeration_id; + ir_hpet_num++; +} + +static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, + struct intel_iommu *iommu) +{ + struct acpi_dmar_pci_path *path; + u8 bus; + int count; + + bus = scope->bus; + path = (struct acpi_dmar_pci_path *)(scope + 1); + count = (scope->length - sizeof(struct acpi_dmar_device_scope)) + / sizeof(struct acpi_dmar_pci_path); + + while (--count > 0) { + /* + * Access PCI directly due to the PCI + * subsystem isn't initialized yet. + */ + bus = read_pci_config_byte(bus, path->dev, path->fn, + PCI_SECONDARY_BUS); + path++; + } + + ir_ioapic[ir_ioapic_num].bus = bus; + ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn); + ir_ioapic[ir_ioapic_num].iommu = iommu; + ir_ioapic[ir_ioapic_num].id = scope->enumeration_id; + ir_ioapic_num++; +} + +static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, + struct intel_iommu *iommu) +{ + struct acpi_dmar_hardware_unit *drhd; + struct acpi_dmar_device_scope *scope; + void *start, *end; + + drhd = (struct acpi_dmar_hardware_unit *)header; + + start = (void *)(drhd + 1); + end = ((void *)drhd) + header->length; + + while (start < end) { + scope = start; + if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { + if (ir_ioapic_num == MAX_IO_APICS) { + printk(KERN_WARNING "Exceeded Max IO APICS\n"); + return -1; + } + + printk(KERN_INFO "IOAPIC id %d under DRHD base " + " 0x%Lx IOMMU %d\n", scope->enumeration_id, + drhd->address, iommu->seq_id); + + ir_parse_one_ioapic_scope(scope, iommu); + } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) { + if (ir_hpet_num == MAX_HPET_TBS) { + printk(KERN_WARNING "Exceeded Max HPET blocks\n"); + return -1; + } + + printk(KERN_INFO "HPET id %d under DRHD base" + " 0x%Lx\n", scope->enumeration_id, + drhd->address); + + ir_parse_one_hpet_scope(scope, iommu); + } + start += scope->length; + } + + return 0; +} + +/* + * Finds the assocaition between IOAPIC's and its Interrupt-remapping + * hardware unit. + */ +int __init parse_ioapics_under_ir(void) +{ + struct dmar_drhd_unit *drhd; + int ir_supported = 0; + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (ecap_ir_support(iommu->ecap)) { + if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) + return -1; + + ir_supported = 1; + } + } + + if (ir_supported && ir_ioapic_num != nr_ioapics) { + printk(KERN_WARNING + "Not all IO-APIC's listed under remapping hardware\n"); + return -1; + } + + return ir_supported; +} + +int __init ir_dev_scope_init(void) +{ + if (!irq_remapping_enabled) + return 0; + + return dmar_dev_scope_init(); +} +rootfs_initcall(ir_dev_scope_init); + +static void disable_irq_remapping(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + + /* + * Disable Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + iommu_disable_irq_remapping(iommu); + } +} + +static int reenable_irq_remapping(int eim) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + struct intel_iommu *iommu = NULL; + + for_each_iommu(iommu, drhd) + if (iommu->qi) + dmar_reenable_qi(iommu); + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + /* Set up interrupt remapping for iommu.*/ + iommu_set_irq_remapping(iommu, eim); + setup = 1; + } + + if (!setup) + goto error; + + return 0; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + +static void prepare_irte(struct irte *irte, int vector, + unsigned int dest) +{ + memset(irte, 0, sizeof(*irte)); + + irte->present = 1; + irte->dst_mode = apic->irq_dest_mode; + /* + * Trigger mode in the IRTE will always be edge, and for IO-APIC, the + * actual level or edge trigger will be setup in the IO-APIC + * RTE. This will help simplify level triggered irq migration. + * For more details, see the comments (in io_apic.c) explainig IO-APIC + * irq migration in the presence of interrupt-remapping. + */ + irte->trigger_mode = 0; + irte->dlvry_mode = apic->irq_delivery_mode; + irte->vector = vector; + irte->dest_id = IRTE_DEST(dest); + irte->redir_hint = 1; +} + +static int intel_setup_ioapic_entry(int irq, + struct IO_APIC_route_entry *route_entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) +{ + int ioapic_id = mpc_ioapic_id(attr->ioapic); + struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id); + struct IR_IO_APIC_route_entry *entry; + struct irte irte; + int index; + + if (!iommu) { + pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); + return -ENODEV; + } + + entry = (struct IR_IO_APIC_route_entry *)route_entry; + + index = alloc_irte(iommu, irq, 1); + if (index < 0) { + pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id); + return -ENOMEM; + } + + prepare_irte(&irte, vector, destination); + + /* Set source-id of interrupt request */ + set_ioapic_sid(&irte, ioapic_id); + + modify_irte(irq, &irte); + + apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " + "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " + "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " + "Avail:%X Vector:%02X Dest:%08X " + "SID:%04X SQ:%X SVT:%X)\n", + attr->ioapic, irte.present, irte.fpd, irte.dst_mode, + irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, + irte.avail, irte.vector, irte.dest_id, + irte.sid, irte.sq, irte.svt); + + memset(entry, 0, sizeof(*entry)); + + entry->index2 = (index >> 15) & 0x1; + entry->zero = 0; + entry->format = 1; + entry->index = (index & 0x7fff); + /* + * IO-APIC RTE will be configured with virtual vector. + * irq handler will do the explicit EOI to the io-apic. + */ + entry->vector = attr->ioapic_pin; + entry->mask = 0; /* enable IRQ */ + entry->trigger = attr->trigger; + entry->polarity = attr->polarity; + + /* Mask level triggered irqs. + * Use IRQ_DELAYED_DISABLE for edge triggered irqs. + */ + if (attr->trigger) + entry->mask = 1; + + return 0; +} + +/* + * Migrate the IO-APIC irq in the presence of intr-remapping. + * + * For both level and edge triggered, irq migration is a simple atomic + * update(of vector and cpu destination) of IRTE and flush the hardware cache. + * + * For level triggered, we eliminate the io-apic RTE modification (with the + * updated vector information), by using a virtual vector (io-apic pin number). + * Real vector that is used for interrupting cpu will be coming from + * the interrupt-remapping table entry. + * + * As the migration is a simple atomic update of IRTE, the same mechanism + * is used to migrate MSI irq's in the presence of interrupt-remapping. + */ +static int +intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) +{ + struct irq_cfg *cfg = data->chip_data; + unsigned int dest, irq = data->irq; + struct irte irte; + + if (!cpumask_intersects(mask, cpu_online_mask)) + return -EINVAL; + + if (get_irte(irq, &irte)) + return -EBUSY; + + if (assign_irq_vector(irq, cfg, mask)) + return -EBUSY; + + dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); + + irte.vector = cfg->vector; + irte.dest_id = IRTE_DEST(dest); + + /* + * Atomically updates the IRTE with the new destination, vector + * and flushes the interrupt entry cache. + */ + modify_irte(irq, &irte); + + /* + * After this point, all the interrupts will start arriving + * at the new destination. So, time to cleanup the previous + * vector allocation. + */ + if (cfg->move_in_progress) + send_cleanup_vector(cfg); + + cpumask_copy(data->affinity, mask); + return 0; +} + +static void intel_compose_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ + struct irq_cfg *cfg; + struct irte irte; + u16 sub_handle; + int ir_index; + + cfg = irq_get_chip_data(irq); + + ir_index = map_irq_to_irte_handle(irq, &sub_handle); + BUG_ON(ir_index == -1); + + prepare_irte(&irte, cfg->vector, dest); + + /* Set source-id of interrupt request */ + if (pdev) + set_msi_sid(&irte, pdev); + else + set_hpet_sid(&irte, hpet_id); + + modify_irte(irq, &irte); + + msg->address_hi = MSI_ADDR_BASE_HI; + msg->data = sub_handle; + msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | + MSI_ADDR_IR_SHV | + MSI_ADDR_IR_INDEX1(ir_index) | + MSI_ADDR_IR_INDEX2(ir_index); +} + +/* + * Map the PCI dev to the corresponding remapping hardware unit + * and allocate 'nvec' consecutive interrupt-remapping table entries + * in it. + */ +static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) +{ + struct intel_iommu *iommu; + int index; + + iommu = map_dev_to_ir(dev); + if (!iommu) { + printk(KERN_ERR + "Unable to map PCI %s to iommu\n", pci_name(dev)); + return -ENOENT; + } + + index = alloc_irte(iommu, irq, nvec); + if (index < 0) { + printk(KERN_ERR + "Unable to allocate %d IRTE for PCI %s\n", nvec, + pci_name(dev)); + return -ENOSPC; + } + return index; +} + +static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + struct intel_iommu *iommu; + + iommu = map_dev_to_ir(pdev); + if (!iommu) + return -ENOENT; + /* + * setup the mapping between the irq and the IRTE + * base index, the sub_handle pointing to the + * appropriate interrupt remap table entry. + */ + set_irte_irq(irq, iommu, index, sub_handle); + + return 0; +} + +static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) +{ + struct intel_iommu *iommu = map_hpet_to_ir(id); + int index; + + if (!iommu) + return -1; + + index = alloc_irte(iommu, irq, 1); + if (index < 0) + return -1; + + return 0; +} + +struct irq_remap_ops intel_irq_remap_ops = { + .supported = intel_irq_remapping_supported, + .prepare = dmar_table_init, + .enable = intel_enable_irq_remapping, + .disable = disable_irq_remapping, + .reenable = reenable_irq_remapping, + .enable_faulting = enable_drhd_fault_handling, + .setup_ioapic_entry = intel_setup_ioapic_entry, + .set_affinity = intel_ioapic_set_affinity, + .free_irq = free_irte, + .compose_msi_msg = intel_compose_msi_msg, + .msi_alloc_irq = intel_msi_alloc_irq, + .msi_setup_irq = intel_msi_setup_irq, + .setup_hpet_msi = intel_setup_hpet_msi, +}; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c deleted file mode 100644 index 523a7b3a1205..000000000000 --- a/drivers/iommu/intr_remapping.c +++ /dev/null @@ -1,164 +0,0 @@ -#include -#include -#include - -#include "intr_remapping.h" - -int irq_remapping_enabled; - -int disable_irq_remap; -int disable_sourceid_checking; -int no_x2apic_optout; - -static struct irq_remap_ops *remap_ops; - -static __init int setup_nointremap(char *str) -{ - disable_irq_remap = 1; - return 0; -} -early_param("nointremap", setup_nointremap); - -static __init int setup_irqremap(char *str) -{ - if (!str) - return -EINVAL; - - while (*str) { - if (!strncmp(str, "on", 2)) - disable_irq_remap = 0; - else if (!strncmp(str, "off", 3)) - disable_irq_remap = 1; - else if (!strncmp(str, "nosid", 5)) - disable_sourceid_checking = 1; - else if (!strncmp(str, "no_x2apic_optout", 16)) - no_x2apic_optout = 1; - - str += strcspn(str, ","); - while (*str == ',') - str++; - } - - return 0; -} -early_param("intremap", setup_irqremap); - -void __init setup_irq_remapping_ops(void) -{ - remap_ops = &intel_irq_remap_ops; -} - -int irq_remapping_supported(void) -{ - if (disable_irq_remap) - return 0; - - if (!remap_ops || !remap_ops->supported) - return 0; - - return remap_ops->supported(); -} - -int __init irq_remapping_prepare(void) -{ - if (!remap_ops || !remap_ops->prepare) - return -ENODEV; - - return remap_ops->prepare(); -} - -int __init irq_remapping_enable(void) -{ - if (!remap_ops || !remap_ops->enable) - return -ENODEV; - - return remap_ops->enable(); -} - -void irq_remapping_disable(void) -{ - if (!remap_ops || !remap_ops->disable) - return; - - remap_ops->disable(); -} - -int irq_remapping_reenable(int mode) -{ - if (!remap_ops || !remap_ops->reenable) - return 0; - - return remap_ops->reenable(mode); -} - -int __init irq_remap_enable_fault_handling(void) -{ - if (!remap_ops || !remap_ops->enable_faulting) - return -ENODEV; - - return remap_ops->enable_faulting(); -} - -int setup_ioapic_remapped_entry(int irq, - struct IO_APIC_route_entry *entry, - unsigned int destination, int vector, - struct io_apic_irq_attr *attr) -{ - if (!remap_ops || !remap_ops->setup_ioapic_entry) - return -ENODEV; - - return remap_ops->setup_ioapic_entry(irq, entry, destination, - vector, attr); -} - -int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) -{ - if (!remap_ops || !remap_ops->set_affinity) - return 0; - - return remap_ops->set_affinity(data, mask, force); -} - -void free_remapped_irq(int irq) -{ - if (!remap_ops || !remap_ops->free_irq) - return; - - remap_ops->free_irq(irq); -} - -void compose_remapped_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id) -{ - if (!remap_ops || !remap_ops->compose_msi_msg) - return; - - remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); -} - -int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) -{ - if (!remap_ops || !remap_ops->msi_alloc_irq) - return -ENODEV; - - return remap_ops->msi_alloc_irq(pdev, irq, nvec); -} - -int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) -{ - if (!remap_ops || !remap_ops->msi_setup_irq) - return -ENODEV; - - return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle); -} - -int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) -{ - if (!remap_ops || !remap_ops->setup_hpet_msi) - return -ENODEV; - - return remap_ops->setup_hpet_msi(irq, id); -} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h deleted file mode 100644 index bd5d98fec148..000000000000 --- a/drivers/iommu/intr_remapping.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2012 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * This header file contains stuff that is shared between different interrupt - * remapping drivers but with no need to be visible outside of the IOMMU layer. - */ - -#ifndef __INTR_REMAPPING_H -#define __INTR_REMAPPING_H - -#ifdef CONFIG_IRQ_REMAP - -struct IO_APIC_route_entry; -struct io_apic_irq_attr; -struct irq_data; -struct cpumask; -struct pci_dev; -struct msi_msg; - -extern int disable_irq_remap; -extern int disable_sourceid_checking; -extern int no_x2apic_optout; - -struct irq_remap_ops { - /* Check whether Interrupt Remapping is supported */ - int (*supported)(void); - - /* Initializes hardware and makes it ready for remapping interrupts */ - int (*prepare)(void); - - /* Enables the remapping hardware */ - int (*enable)(void); - - /* Disables the remapping hardware */ - void (*disable)(void); - - /* Reenables the remapping hardware */ - int (*reenable)(int); - - /* Enable fault handling */ - int (*enable_faulting)(void); - - /* IO-APIC setup routine */ - int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *, - unsigned int, int, - struct io_apic_irq_attr *); - - /* Set the CPU affinity of a remapped interrupt */ - int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, - bool force); - - /* Free an IRQ */ - int (*free_irq)(int); - - /* Create MSI msg to use for interrupt remapping */ - void (*compose_msi_msg)(struct pci_dev *, - unsigned int, unsigned int, - struct msi_msg *, u8); - - /* Allocate remapping resources for MSI */ - int (*msi_alloc_irq)(struct pci_dev *, int, int); - - /* Setup the remapped MSI irq */ - int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int); - - /* Setup interrupt remapping for an HPET MSI */ - int (*setup_hpet_msi)(unsigned int, unsigned int); -}; - -extern struct irq_remap_ops intel_irq_remap_ops; - -#endif /* CONFIG_IRQ_REMAP */ - -#endif /* __INTR_REMAPPING_H */ diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c new file mode 100644 index 000000000000..1cf350e02da8 --- /dev/null +++ b/drivers/iommu/irq_remapping.c @@ -0,0 +1,164 @@ +#include +#include +#include + +#include "irq_remapping.h" + +int irq_remapping_enabled; + +int disable_irq_remap; +int disable_sourceid_checking; +int no_x2apic_optout; + +static struct irq_remap_ops *remap_ops; + +static __init int setup_nointremap(char *str) +{ + disable_irq_remap = 1; + return 0; +} +early_param("nointremap", setup_nointremap); + +static __init int setup_irqremap(char *str) +{ + if (!str) + return -EINVAL; + + while (*str) { + if (!strncmp(str, "on", 2)) + disable_irq_remap = 0; + else if (!strncmp(str, "off", 3)) + disable_irq_remap = 1; + else if (!strncmp(str, "nosid", 5)) + disable_sourceid_checking = 1; + else if (!strncmp(str, "no_x2apic_optout", 16)) + no_x2apic_optout = 1; + + str += strcspn(str, ","); + while (*str == ',') + str++; + } + + return 0; +} +early_param("intremap", setup_irqremap); + +void __init setup_irq_remapping_ops(void) +{ + remap_ops = &intel_irq_remap_ops; +} + +int irq_remapping_supported(void) +{ + if (disable_irq_remap) + return 0; + + if (!remap_ops || !remap_ops->supported) + return 0; + + return remap_ops->supported(); +} + +int __init irq_remapping_prepare(void) +{ + if (!remap_ops || !remap_ops->prepare) + return -ENODEV; + + return remap_ops->prepare(); +} + +int __init irq_remapping_enable(void) +{ + if (!remap_ops || !remap_ops->enable) + return -ENODEV; + + return remap_ops->enable(); +} + +void irq_remapping_disable(void) +{ + if (!remap_ops || !remap_ops->disable) + return; + + remap_ops->disable(); +} + +int irq_remapping_reenable(int mode) +{ + if (!remap_ops || !remap_ops->reenable) + return 0; + + return remap_ops->reenable(mode); +} + +int __init irq_remap_enable_fault_handling(void) +{ + if (!remap_ops || !remap_ops->enable_faulting) + return -ENODEV; + + return remap_ops->enable_faulting(); +} + +int setup_ioapic_remapped_entry(int irq, + struct IO_APIC_route_entry *entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) +{ + if (!remap_ops || !remap_ops->setup_ioapic_entry) + return -ENODEV; + + return remap_ops->setup_ioapic_entry(irq, entry, destination, + vector, attr); +} + +int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) +{ + if (!remap_ops || !remap_ops->set_affinity) + return 0; + + return remap_ops->set_affinity(data, mask, force); +} + +void free_remapped_irq(int irq) +{ + if (!remap_ops || !remap_ops->free_irq) + return; + + remap_ops->free_irq(irq); +} + +void compose_remapped_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ + if (!remap_ops || !remap_ops->compose_msi_msg) + return; + + remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); +} + +int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) +{ + if (!remap_ops || !remap_ops->msi_alloc_irq) + return -ENODEV; + + return remap_ops->msi_alloc_irq(pdev, irq, nvec); +} + +int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + if (!remap_ops || !remap_ops->msi_setup_irq) + return -ENODEV; + + return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle); +} + +int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) +{ + if (!remap_ops || !remap_ops->setup_hpet_msi) + return -ENODEV; + + return remap_ops->setup_hpet_msi(irq, id); +} diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h new file mode 100644 index 000000000000..b12974cc1dfe --- /dev/null +++ b/drivers/iommu/irq_remapping.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2012 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * This header file contains stuff that is shared between different interrupt + * remapping drivers but with no need to be visible outside of the IOMMU layer. + */ + +#ifndef __IRQ_REMAPPING_H +#define __IRQ_REMAPPING_H + +#ifdef CONFIG_IRQ_REMAP + +struct IO_APIC_route_entry; +struct io_apic_irq_attr; +struct irq_data; +struct cpumask; +struct pci_dev; +struct msi_msg; + +extern int disable_irq_remap; +extern int disable_sourceid_checking; +extern int no_x2apic_optout; + +struct irq_remap_ops { + /* Check whether Interrupt Remapping is supported */ + int (*supported)(void); + + /* Initializes hardware and makes it ready for remapping interrupts */ + int (*prepare)(void); + + /* Enables the remapping hardware */ + int (*enable)(void); + + /* Disables the remapping hardware */ + void (*disable)(void); + + /* Reenables the remapping hardware */ + int (*reenable)(int); + + /* Enable fault handling */ + int (*enable_faulting)(void); + + /* IO-APIC setup routine */ + int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *, + unsigned int, int, + struct io_apic_irq_attr *); + + /* Set the CPU affinity of a remapped interrupt */ + int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, + bool force); + + /* Free an IRQ */ + int (*free_irq)(int); + + /* Create MSI msg to use for interrupt remapping */ + void (*compose_msi_msg)(struct pci_dev *, + unsigned int, unsigned int, + struct msi_msg *, u8); + + /* Allocate remapping resources for MSI */ + int (*msi_alloc_irq)(struct pci_dev *, int, int); + + /* Setup the remapped MSI irq */ + int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int); + + /* Setup interrupt remapping for an HPET MSI */ + int (*setup_hpet_msi)(unsigned int, unsigned int); +}; + +extern struct irq_remap_ops intel_irq_remap_ops; + +#endif /* CONFIG_IRQ_REMAP */ + +#endif /* __IRQ_REMAPPING_H */ -- cgit v1.2.3 From 82b481e80d8a17720b5805393684a95184cfb6bb Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Tue, 8 May 2012 00:08:53 -0700 Subject: irq_remap: Fix UP build failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the below UP build failure with CONFIG_IRQ_REMAP enabled. drivers/iommu/intel_irq_remapping.c:955:19: error: ‘struct irq_data’ has no member named ‘affinity’ Reported-by: Ingo Molnar Signed-off-by: Suresh Siddha Cc: joro@8bytes.org Cc: iommu@lists.linux-foundation.org Cc: Joerg Roedel Link: http://lkml.kernel.org/r/1336460934-23592-2-git-send-email-suresh.b.siddha@intel.com Signed-off-by: Ingo Molnar --- drivers/iommu/intel_irq_remapping.c | 4 ++++ drivers/iommu/irq_remapping.c | 2 ++ drivers/iommu/irq_remapping.h | 2 ++ 3 files changed, 8 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index b4d39507681a..1c0255e4ff29 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -902,6 +902,7 @@ static int intel_setup_ioapic_entry(int irq, return 0; } +#ifdef CONFIG_SMP /* * Migrate the IO-APIC irq in the presence of intr-remapping. * @@ -955,6 +956,7 @@ intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, cpumask_copy(data->affinity, mask); return 0; } +#endif static void intel_compose_msi_msg(struct pci_dev *pdev, unsigned int irq, unsigned int dest, @@ -1056,7 +1058,9 @@ struct irq_remap_ops intel_irq_remap_ops = { .reenable = reenable_irq_remapping, .enable_faulting = enable_drhd_fault_handling, .setup_ioapic_entry = intel_setup_ioapic_entry, +#ifdef CONFIG_SMP .set_affinity = intel_ioapic_set_affinity, +#endif .free_irq = free_irte, .compose_msi_msg = intel_compose_msi_msg, .msi_alloc_irq = intel_msi_alloc_irq, diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 1cf350e02da8..40cda8e98d87 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -111,6 +111,7 @@ int setup_ioapic_remapped_entry(int irq, vector, attr); } +#ifdef CONFIG_SMP int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { @@ -119,6 +120,7 @@ int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, return remap_ops->set_affinity(data, mask, force); } +#endif void free_remapped_irq(int irq) { diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h index b12974cc1dfe..be9d72950c51 100644 --- a/drivers/iommu/irq_remapping.h +++ b/drivers/iommu/irq_remapping.h @@ -59,9 +59,11 @@ struct irq_remap_ops { unsigned int, int, struct io_apic_irq_attr *); +#ifdef CONFIG_SMP /* Set the CPU affinity of a remapped interrupt */ int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, bool force); +#endif /* Free an IRQ */ int (*free_irq)(int); -- cgit v1.2.3 From c558df4a01a9ec7b02303aba808d1e5044822add Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Tue, 8 May 2012 00:08:54 -0700 Subject: irq_remap: Fix the 'sub_handle' uninitialized warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix this uninitialized variable warning: drivers/iommu/intel_irq_remapping.c:986:12: warning: ‘sub_handle’ may be used uninitialized in this function [-Wuninitialized] GCC is wrong, help it out. Signed-off-by: Suresh Siddha Cc: joro@8bytes.org Cc: iommu@lists.linux-foundation.org Cc: Joerg Roedel Link: http://lkml.kernel.org/r/1336460934-23592-3-git-send-email-suresh.b.siddha@intel.com Signed-off-by: Ingo Molnar --- drivers/iommu/intel_irq_remapping.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 1c0255e4ff29..6d347064b8b0 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -964,7 +964,7 @@ static void intel_compose_msi_msg(struct pci_dev *pdev, { struct irq_cfg *cfg; struct irte irte; - u16 sub_handle; + u16 sub_handle = 0; int ir_index; cfg = irq_get_chip_data(irq); -- cgit v1.2.3 From fefe1ed1398b81e3fadc92d11d91162d343c8836 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Sun, 13 May 2012 20:09:38 +0300 Subject: iommu: Fix off by one in dmar_get_fault_reason() fault_reason - 0x20 == ARRAY_SIZE(irq_remap_fault_reasons) is one past the end of the array. Signed-off-by: Dan Carpenter Cc: Joerg Roedel Cc: Youquan Song Cc: walter harms Cc: Suresh Siddha Cc: Link: http://lkml.kernel.org/r/20120513170938.GA4280@elgon.mountain Signed-off-by: Ingo Molnar --- drivers/iommu/dmar.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 5ef65cf66152..3a74e4410fc0 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1057,8 +1057,8 @@ static const char *irq_remap_fault_reasons[] = const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type) { - if (fault_reason >= 0x20 && (fault_reason <= 0x20 + - ARRAY_SIZE(irq_remap_fault_reasons))) { + if (fault_reason >= 0x20 && (fault_reason - 0x20 < + ARRAY_SIZE(irq_remap_fault_reasons))) { *fault_type = INTR_REMAP; return irq_remap_fault_reasons[fault_reason - 0x20]; } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) { -- cgit v1.2.3