From 8a8f422d3b4f2cde8e0e1d31638279a26a886a82 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Fri, 30 Mar 2012 11:47:08 -0700 Subject: iommu: rename intr_remapping.[ch] to irq_remapping.[ch] Make the file names consistent with the naming conventions of irq subsystem. Signed-off-by: Suresh Siddha Cc: Joerg Roedel Cc: Yinghai Lu Cc: David Woodhouse Cc: Alex Williamson Signed-off-by: Joerg Roedel --- arch/ia64/include/asm/intr_remapping.h | 4 - arch/ia64/include/asm/irq_remapping.h | 4 + arch/x86/include/asm/intr_remapping.h | 103 --- arch/x86/include/asm/irq_remapping.h | 103 +++ arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/apic/io_apic.c | 2 +- drivers/iommu/Makefile | 2 +- drivers/iommu/dmar.c | 2 +- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/intel_intr_remapping.c | 1065 -------------------------------- drivers/iommu/intel_irq_remapping.c | 1065 ++++++++++++++++++++++++++++++++ drivers/iommu/intr_remapping.c | 164 ----- drivers/iommu/intr_remapping.h | 88 --- drivers/iommu/irq_remapping.c | 164 +++++ drivers/iommu/irq_remapping.h | 88 +++ 15 files changed, 1429 insertions(+), 1429 deletions(-) delete mode 100644 arch/ia64/include/asm/intr_remapping.h create mode 100644 arch/ia64/include/asm/irq_remapping.h delete mode 100644 arch/x86/include/asm/intr_remapping.h create mode 100644 arch/x86/include/asm/irq_remapping.h delete mode 100644 drivers/iommu/intel_intr_remapping.c create mode 100644 drivers/iommu/intel_irq_remapping.c delete mode 100644 drivers/iommu/intr_remapping.c delete mode 100644 drivers/iommu/intr_remapping.h create mode 100644 drivers/iommu/irq_remapping.c create mode 100644 drivers/iommu/irq_remapping.h diff --git a/arch/ia64/include/asm/intr_remapping.h b/arch/ia64/include/asm/intr_remapping.h deleted file mode 100644 index a8687b1d890..00000000000 --- a/arch/ia64/include/asm/intr_remapping.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef __IA64_INTR_REMAPPING_H -#define __IA64_INTR_REMAPPING_H -#define irq_remapping_enabled 0 -#endif diff --git a/arch/ia64/include/asm/irq_remapping.h b/arch/ia64/include/asm/irq_remapping.h new file mode 100644 index 00000000000..a8687b1d890 --- /dev/null +++ b/arch/ia64/include/asm/irq_remapping.h @@ -0,0 +1,4 @@ +#ifndef __IA64_INTR_REMAPPING_H +#define __IA64_INTR_REMAPPING_H +#define irq_remapping_enabled 0 +#endif diff --git a/arch/x86/include/asm/intr_remapping.h b/arch/x86/include/asm/intr_remapping.h deleted file mode 100644 index f9cbbcb2956..00000000000 --- a/arch/x86/include/asm/intr_remapping.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2012 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * This header file contains the interface of the interrupt remapping code to - * the x86 interrupt management code. - */ - -#ifndef __X86_INTR_REMAPPING_H -#define __X86_INTR_REMAPPING_H - -#ifdef CONFIG_IRQ_REMAP - -struct IO_APIC_route_entry; -struct io_apic_irq_attr; -struct pci_dev; - -extern int irq_remapping_enabled; - -extern void setup_irq_remapping_ops(void); -extern int irq_remapping_supported(void); -extern int irq_remapping_prepare(void); -extern int irq_remapping_enable(void); -extern void irq_remapping_disable(void); -extern int irq_remapping_reenable(int); -extern int irq_remap_enable_fault_handling(void); -extern int setup_ioapic_remapped_entry(int irq, - struct IO_APIC_route_entry *entry, - unsigned int destination, - int vector, - struct io_apic_irq_attr *attr); -extern int set_remapped_irq_affinity(struct irq_data *data, - const struct cpumask *mask, - bool force); -extern void free_remapped_irq(int irq); -extern void compose_remapped_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id); -extern int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec); -extern int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle); -extern int setup_hpet_msi_remapped(unsigned int irq, unsigned int id); - -#else /* CONFIG_IRQ_REMAP */ - -#define irq_remapping_enabled 0 - -static inline void setup_irq_remapping_ops(void) { } -static inline int irq_remapping_supported(void) { return 0; } -static inline int irq_remapping_prepare(void) { return -ENODEV; } -static inline int irq_remapping_enable(void) { return -ENODEV; } -static inline void irq_remapping_disable(void) { } -static inline int irq_remapping_reenable(int eim) { return -ENODEV; } -static inline int irq_remap_enable_fault_handling(void) { return -ENODEV; } -static inline int setup_ioapic_remapped_entry(int irq, - struct IO_APIC_route_entry *entry, - unsigned int destination, - int vector, - struct io_apic_irq_attr *attr) -{ - return -ENODEV; -} -static inline int set_remapped_irq_affinity(struct irq_data *data, - const struct cpumask *mask, - bool force) -{ - return 0; -} -static inline void free_remapped_irq(int irq) { } -static inline void compose_remapped_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id) -{ -} -static inline int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) -{ - return -ENODEV; -} -static inline int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) -{ - return -ENODEV; -} -static inline int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) -{ - return -ENODEV; -} -#endif /* CONFIG_IRQ_REMAP */ - -#endif /* __X86_INTR_REMAPPING_H */ diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h new file mode 100644 index 00000000000..dcb0c723102 --- /dev/null +++ b/arch/x86/include/asm/irq_remapping.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2012 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * This header file contains the interface of the interrupt remapping code to + * the x86 interrupt management code. + */ + +#ifndef __X86_IRQ_REMAPPING_H +#define __X86_IRQ_REMAPPING_H + +#ifdef CONFIG_IRQ_REMAP + +struct IO_APIC_route_entry; +struct io_apic_irq_attr; +struct pci_dev; + +extern int irq_remapping_enabled; + +extern void setup_irq_remapping_ops(void); +extern int irq_remapping_supported(void); +extern int irq_remapping_prepare(void); +extern int irq_remapping_enable(void); +extern void irq_remapping_disable(void); +extern int irq_remapping_reenable(int); +extern int irq_remap_enable_fault_handling(void); +extern int setup_ioapic_remapped_entry(int irq, + struct IO_APIC_route_entry *entry, + unsigned int destination, + int vector, + struct io_apic_irq_attr *attr); +extern int set_remapped_irq_affinity(struct irq_data *data, + const struct cpumask *mask, + bool force); +extern void free_remapped_irq(int irq); +extern void compose_remapped_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id); +extern int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec); +extern int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle); +extern int setup_hpet_msi_remapped(unsigned int irq, unsigned int id); + +#else /* CONFIG_IRQ_REMAP */ + +#define irq_remapping_enabled 0 + +static inline void setup_irq_remapping_ops(void) { } +static inline int irq_remapping_supported(void) { return 0; } +static inline int irq_remapping_prepare(void) { return -ENODEV; } +static inline int irq_remapping_enable(void) { return -ENODEV; } +static inline void irq_remapping_disable(void) { } +static inline int irq_remapping_reenable(int eim) { return -ENODEV; } +static inline int irq_remap_enable_fault_handling(void) { return -ENODEV; } +static inline int setup_ioapic_remapped_entry(int irq, + struct IO_APIC_route_entry *entry, + unsigned int destination, + int vector, + struct io_apic_irq_attr *attr) +{ + return -ENODEV; +} +static inline int set_remapped_irq_affinity(struct irq_data *data, + const struct cpumask *mask, + bool force) +{ + return 0; +} +static inline void free_remapped_irq(int irq) { } +static inline void compose_remapped_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ +} +static inline int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) +{ + return -ENODEV; +} +static inline int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + return -ENODEV; +} +static inline int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) +{ + return -ENODEV; +} +#endif /* CONFIG_IRQ_REMAP */ + +#endif /* __X86_IRQ_REMAPPING_H */ diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index c02c666c462..3722179a49d 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -35,7 +35,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index abbbcd4d1d7..ef0648cd708 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -57,7 +57,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 823e1cf8708..3e5e82ae9f0 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o -obj-$(CONFIG_IRQ_REMAP) += intel_intr_remapping.o intr_remapping.o +obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index ee74f698eef..5ef65cf6615 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #define PREFIX "DMAR: " diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index cef5b8226f3..bf2fbaad5e2 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/iommu/intel_intr_remapping.c b/drivers/iommu/intel_intr_remapping.c deleted file mode 100644 index efeb601c782..00000000000 --- a/drivers/iommu/intel_intr_remapping.c +++ /dev/null @@ -1,1065 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "intr_remapping.h" - -struct ioapic_scope { - struct intel_iommu *iommu; - unsigned int id; - unsigned int bus; /* PCI bus number */ - unsigned int devfn; /* PCI devfn number */ -}; - -struct hpet_scope { - struct intel_iommu *iommu; - u8 id; - unsigned int bus; - unsigned int devfn; -}; - -#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) -#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8) - -static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; -static struct hpet_scope ir_hpet[MAX_HPET_TBS]; -static int ir_ioapic_num, ir_hpet_num; - -static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); - -static struct irq_2_iommu *irq_2_iommu(unsigned int irq) -{ - struct irq_cfg *cfg = irq_get_chip_data(irq); - return cfg ? &cfg->irq_2_iommu : NULL; -} - -int get_irte(int irq, struct irte *entry) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int index; - - if (!entry || !irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - index = irq_iommu->irte_index + irq_iommu->sub_handle; - *entry = *(irq_iommu->iommu->ir_table->base + index); - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - return 0; -} - -static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) -{ - struct ir_table *table = iommu->ir_table; - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - u16 index, start_index; - unsigned int mask = 0; - unsigned long flags; - int i; - - if (!count || !irq_iommu) - return -1; - - /* - * start the IRTE search from index 0. - */ - index = start_index = 0; - - if (count > 1) { - count = __roundup_pow_of_two(count); - mask = ilog2(count); - } - - if (mask > ecap_max_handle_mask(iommu->ecap)) { - printk(KERN_ERR - "Requested mask %x exceeds the max invalidation handle" - " mask value %Lx\n", mask, - ecap_max_handle_mask(iommu->ecap)); - return -1; - } - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - do { - for (i = index; i < index + count; i++) - if (table->base[i].present) - break; - /* empty index found */ - if (i == index + count) - break; - - index = (index + count) % INTR_REMAP_TABLE_ENTRIES; - - if (index == start_index) { - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - printk(KERN_ERR "can't allocate an IRTE\n"); - return -1; - } - } while (1); - - for (i = index; i < index + count; i++) - table->base[i].present = 1; - - irq_iommu->iommu = iommu; - irq_iommu->irte_index = index; - irq_iommu->sub_handle = 0; - irq_iommu->irte_mask = mask; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return index; -} - -static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) -{ - struct qi_desc desc; - - desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) - | QI_IEC_SELECTIVE; - desc.high = 0; - - return qi_submit_sync(&desc, iommu); -} - -static int map_irq_to_irte_handle(int irq, u16 *sub_handle) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int index; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - *sub_handle = irq_iommu->sub_handle; - index = irq_iommu->irte_index; - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - return index; -} - -static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - irq_iommu->iommu = iommu; - irq_iommu->irte_index = index; - irq_iommu->sub_handle = subhandle; - irq_iommu->irte_mask = 0; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return 0; -} - -static int modify_irte(int irq, struct irte *irte_modified) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - struct intel_iommu *iommu; - unsigned long flags; - struct irte *irte; - int rc, index; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - iommu = irq_iommu->iommu; - - index = irq_iommu->irte_index + irq_iommu->sub_handle; - irte = &iommu->ir_table->base[index]; - - set_64bit(&irte->low, irte_modified->low); - set_64bit(&irte->high, irte_modified->high); - __iommu_flush_cache(iommu, irte, sizeof(*irte)); - - rc = qi_flush_iec(iommu, index, 0); - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return rc; -} - -static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) -{ - int i; - - for (i = 0; i < MAX_HPET_TBS; i++) - if (ir_hpet[i].id == hpet_id) - return ir_hpet[i].iommu; - return NULL; -} - -static struct intel_iommu *map_ioapic_to_ir(int apic) -{ - int i; - - for (i = 0; i < MAX_IO_APICS; i++) - if (ir_ioapic[i].id == apic) - return ir_ioapic[i].iommu; - return NULL; -} - -static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) -{ - struct dmar_drhd_unit *drhd; - - drhd = dmar_find_matched_drhd_unit(dev); - if (!drhd) - return NULL; - - return drhd->iommu; -} - -static int clear_entries(struct irq_2_iommu *irq_iommu) -{ - struct irte *start, *entry, *end; - struct intel_iommu *iommu; - int index; - - if (irq_iommu->sub_handle) - return 0; - - iommu = irq_iommu->iommu; - index = irq_iommu->irte_index + irq_iommu->sub_handle; - - start = iommu->ir_table->base + index; - end = start + (1 << irq_iommu->irte_mask); - - for (entry = start; entry < end; entry++) { - set_64bit(&entry->low, 0); - set_64bit(&entry->high, 0); - } - - return qi_flush_iec(iommu, index, irq_iommu->irte_mask); -} - -static int free_irte(int irq) -{ - struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); - unsigned long flags; - int rc; - - if (!irq_iommu) - return -1; - - raw_spin_lock_irqsave(&irq_2_ir_lock, flags); - - rc = clear_entries(irq_iommu); - - irq_iommu->iommu = NULL; - irq_iommu->irte_index = 0; - irq_iommu->sub_handle = 0; - irq_iommu->irte_mask = 0; - - raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); - - return rc; -} - -/* - * source validation type - */ -#define SVT_NO_VERIFY 0x0 /* no verification is required */ -#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ -#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ - -/* - * source-id qualifier - */ -#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ -#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore - * the third least significant bit - */ -#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore - * the second and third least significant bits - */ -#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore - * the least three significant bits - */ - -/* - * set SVT, SQ and SID fields of irte to verify - * source ids of interrupt requests - */ -static void set_irte_sid(struct irte *irte, unsigned int svt, - unsigned int sq, unsigned int sid) -{ - if (disable_sourceid_checking) - svt = SVT_NO_VERIFY; - irte->svt = svt; - irte->sq = sq; - irte->sid = sid; -} - -static int set_ioapic_sid(struct irte *irte, int apic) -{ - int i; - u16 sid = 0; - - if (!irte) - return -1; - - for (i = 0; i < MAX_IO_APICS; i++) { - if (ir_ioapic[i].id == apic) { - sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; - break; - } - } - - if (sid == 0) { - pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); - return -1; - } - - set_irte_sid(irte, 1, 0, sid); - - return 0; -} - -static int set_hpet_sid(struct irte *irte, u8 id) -{ - int i; - u16 sid = 0; - - if (!irte) - return -1; - - for (i = 0; i < MAX_HPET_TBS; i++) { - if (ir_hpet[i].id == id) { - sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; - break; - } - } - - if (sid == 0) { - pr_warning("Failed to set source-id of HPET block (%d)\n", id); - return -1; - } - - /* - * Should really use SQ_ALL_16. Some platforms are broken. - * While we figure out the right quirks for these broken platforms, use - * SQ_13_IGNORE_3 for now. - */ - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); - - return 0; -} - -static int set_msi_sid(struct irte *irte, struct pci_dev *dev) -{ - struct pci_dev *bridge; - - if (!irte || !dev) - return -1; - - /* PCIe device or Root Complex integrated PCI device */ - if (pci_is_pcie(dev) || !dev->bus->parent) { - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, - (dev->bus->number << 8) | dev->devfn); - return 0; - } - - bridge = pci_find_upstream_pcie_bridge(dev); - if (bridge) { - if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */ - set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, - (bridge->bus->number << 8) | dev->bus->number); - else /* this is a legacy PCI bridge */ - set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, - (bridge->bus->number << 8) | bridge->devfn); - } - - return 0; -} - -static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) -{ - u64 addr; - u32 sts; - unsigned long flags; - - addr = virt_to_phys((void *)iommu->ir_table->base); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - dmar_writeq(iommu->reg + DMAR_IRTA_REG, - (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); - - /* Set interrupt-remapping table pointer */ - iommu->gcmd |= DMA_GCMD_SIRTP; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, (sts & DMA_GSTS_IRTPS), sts); - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); - - /* - * global invalidation of interrupt entry cache before enabling - * interrupt-remapping. - */ - qi_global_iec(iommu); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - /* Enable interrupt-remapping */ - iommu->gcmd |= DMA_GCMD_IRE; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, (sts & DMA_GSTS_IRES), sts); - - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); -} - - -static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode) -{ - struct ir_table *ir_table; - struct page *pages; - - ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), - GFP_ATOMIC); - - if (!iommu->ir_table) - return -ENOMEM; - - pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, - INTR_REMAP_PAGE_ORDER); - - if (!pages) { - printk(KERN_ERR "failed to allocate pages of order %d\n", - INTR_REMAP_PAGE_ORDER); - kfree(iommu->ir_table); - return -ENOMEM; - } - - ir_table->base = page_address(pages); - - iommu_set_irq_remapping(iommu, mode); - return 0; -} - -/* - * Disable Interrupt Remapping. - */ -static void iommu_disable_irq_remapping(struct intel_iommu *iommu) -{ - unsigned long flags; - u32 sts; - - if (!ecap_ir_support(iommu->ecap)) - return; - - /* - * global invalidation of interrupt entry cache before disabling - * interrupt-remapping. - */ - qi_global_iec(iommu); - - raw_spin_lock_irqsave(&iommu->register_lock, flags); - - sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); - if (!(sts & DMA_GSTS_IRES)) - goto end; - - iommu->gcmd &= ~DMA_GCMD_IRE; - writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); - - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, - readl, !(sts & DMA_GSTS_IRES), sts); - -end: - raw_spin_unlock_irqrestore(&iommu->register_lock, flags); -} - -static int __init dmar_x2apic_optout(void) -{ - struct acpi_table_dmar *dmar; - dmar = (struct acpi_table_dmar *)dmar_tbl; - if (!dmar || no_x2apic_optout) - return 0; - return dmar->flags & DMAR_X2APIC_OPT_OUT; -} - -static int __init intel_irq_remapping_supported(void) -{ - struct dmar_drhd_unit *drhd; - - if (disable_irq_remap) - return 0; - - if (!dmar_ir_support()) - return 0; - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - return 0; - } - - return 1; -} - -static int __init intel_enable_irq_remapping(void) -{ - struct dmar_drhd_unit *drhd; - int setup = 0; - int eim = 0; - - if (parse_ioapics_under_ir() != 1) { - printk(KERN_INFO "Not enable interrupt remapping\n"); - return -1; - } - - if (x2apic_supported()) { - eim = !dmar_x2apic_optout(); - WARN(!eim, KERN_WARNING - "Your BIOS is broken and requested that x2apic be disabled\n" - "This will leave your machine vulnerable to irq-injection attacks\n" - "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); - } - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - /* - * If the queued invalidation is already initialized, - * shouldn't disable it. - */ - if (iommu->qi) - continue; - - /* - * Clear previous faults. - */ - dmar_fault(-1, iommu); - - /* - * Disable intr remapping and queued invalidation, if already - * enabled prior to OS handover. - */ - iommu_disable_irq_remapping(iommu); - - dmar_disable_qi(iommu); - } - - /* - * check for the Interrupt-remapping support - */ - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - continue; - - if (eim && !ecap_eim_support(iommu->ecap)) { - printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " - " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); - return -1; - } - } - - /* - * Enable queued invalidation for all the DRHD's. - */ - for_each_drhd_unit(drhd) { - int ret; - struct intel_iommu *iommu = drhd->iommu; - ret = dmar_enable_qi(iommu); - - if (ret) { - printk(KERN_ERR "DRHD %Lx: failed to enable queued, " - " invalidation, ecap %Lx, ret %d\n", - drhd->reg_base_addr, iommu->ecap, ret); - return -1; - } - } - - /* - * Setup Interrupt-remapping for all the DRHD's now. - */ - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (!ecap_ir_support(iommu->ecap)) - continue; - - if (intel_setup_irq_remapping(iommu, eim)) - goto error; - - setup = 1; - } - - if (!setup) - goto error; - - irq_remapping_enabled = 1; - pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); - - return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; - -error: - /* - * handle error condition gracefully here! - */ - return -1; -} - -static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, - struct intel_iommu *iommu) -{ - struct acpi_dmar_pci_path *path; - u8 bus; - int count; - - bus = scope->bus; - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (--count > 0) { - /* - * Access PCI directly due to the PCI - * subsystem isn't initialized yet. - */ - bus = read_pci_config_byte(bus, path->dev, path->fn, - PCI_SECONDARY_BUS); - path++; - } - ir_hpet[ir_hpet_num].bus = bus; - ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn); - ir_hpet[ir_hpet_num].iommu = iommu; - ir_hpet[ir_hpet_num].id = scope->enumeration_id; - ir_hpet_num++; -} - -static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, - struct intel_iommu *iommu) -{ - struct acpi_dmar_pci_path *path; - u8 bus; - int count; - - bus = scope->bus; - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (--count > 0) { - /* - * Access PCI directly due to the PCI - * subsystem isn't initialized yet. - */ - bus = read_pci_config_byte(bus, path->dev, path->fn, - PCI_SECONDARY_BUS); - path++; - } - - ir_ioapic[ir_ioapic_num].bus = bus; - ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn); - ir_ioapic[ir_ioapic_num].iommu = iommu; - ir_ioapic[ir_ioapic_num].id = scope->enumeration_id; - ir_ioapic_num++; -} - -static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, - struct intel_iommu *iommu) -{ - struct acpi_dmar_hardware_unit *drhd; - struct acpi_dmar_device_scope *scope; - void *start, *end; - - drhd = (struct acpi_dmar_hardware_unit *)header; - - start = (void *)(drhd + 1); - end = ((void *)drhd) + header->length; - - while (start < end) { - scope = start; - if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { - if (ir_ioapic_num == MAX_IO_APICS) { - printk(KERN_WARNING "Exceeded Max IO APICS\n"); - return -1; - } - - printk(KERN_INFO "IOAPIC id %d under DRHD base " - " 0x%Lx IOMMU %d\n", scope->enumeration_id, - drhd->address, iommu->seq_id); - - ir_parse_one_ioapic_scope(scope, iommu); - } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) { - if (ir_hpet_num == MAX_HPET_TBS) { - printk(KERN_WARNING "Exceeded Max HPET blocks\n"); - return -1; - } - - printk(KERN_INFO "HPET id %d under DRHD base" - " 0x%Lx\n", scope->enumeration_id, - drhd->address); - - ir_parse_one_hpet_scope(scope, iommu); - } - start += scope->length; - } - - return 0; -} - -/* - * Finds the assocaition between IOAPIC's and its Interrupt-remapping - * hardware unit. - */ -int __init parse_ioapics_under_ir(void) -{ - struct dmar_drhd_unit *drhd; - int ir_supported = 0; - - for_each_drhd_unit(drhd) { - struct intel_iommu *iommu = drhd->iommu; - - if (ecap_ir_support(iommu->ecap)) { - if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) - return -1; - - ir_supported = 1; - } - } - - if (ir_supported && ir_ioapic_num != nr_ioapics) { - printk(KERN_WARNING - "Not all IO-APIC's listed under remapping hardware\n"); - return -1; - } - - return ir_supported; -} - -int __init ir_dev_scope_init(void) -{ - if (!irq_remapping_enabled) - return 0; - - return dmar_dev_scope_init(); -} -rootfs_initcall(ir_dev_scope_init); - -static void disable_irq_remapping(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu = NULL; - - /* - * Disable Interrupt-remapping for all the DRHD's now. - */ - for_each_iommu(iommu, drhd) { - if (!ecap_ir_support(iommu->ecap)) - continue; - - iommu_disable_irq_remapping(iommu); - } -} - -static int reenable_irq_remapping(int eim) -{ - struct dmar_drhd_unit *drhd; - int setup = 0; - struct intel_iommu *iommu = NULL; - - for_each_iommu(iommu, drhd) - if (iommu->qi) - dmar_reenable_qi(iommu); - - /* - * Setup Interrupt-remapping for all the DRHD's now. - */ - for_each_iommu(iommu, drhd) { - if (!ecap_ir_support(iommu->ecap)) - continue; - - /* Set up interrupt remapping for iommu.*/ - iommu_set_irq_remapping(iommu, eim); - setup = 1; - } - - if (!setup) - goto error; - - return 0; - -error: - /* - * handle error condition gracefully here! - */ - return -1; -} - -static void prepare_irte(struct irte *irte, int vector, - unsigned int dest) -{ - memset(irte, 0, sizeof(*irte)); - - irte->present = 1; - irte->dst_mode = apic->irq_dest_mode; - /* - * Trigger mode in the IRTE will always be edge, and for IO-APIC, the - * actual level or edge trigger will be setup in the IO-APIC - * RTE. This will help simplify level triggered irq migration. - * For more details, see the comments (in io_apic.c) explainig IO-APIC - * irq migration in the presence of interrupt-remapping. - */ - irte->trigger_mode = 0; - irte->dlvry_mode = apic->irq_delivery_mode; - irte->vector = vector; - irte->dest_id = IRTE_DEST(dest); - irte->redir_hint = 1; -} - -static int intel_setup_ioapic_entry(int irq, - struct IO_APIC_route_entry *route_entry, - unsigned int destination, int vector, - struct io_apic_irq_attr *attr) -{ - int ioapic_id = mpc_ioapic_id(attr->ioapic); - struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id); - struct IR_IO_APIC_route_entry *entry; - struct irte irte; - int index; - - if (!iommu) { - pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); - return -ENODEV; - } - - entry = (struct IR_IO_APIC_route_entry *)route_entry; - - index = alloc_irte(iommu, irq, 1); - if (index < 0) { - pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id); - return -ENOMEM; - } - - prepare_irte(&irte, vector, destination); - - /* Set source-id of interrupt request */ - set_ioapic_sid(&irte, ioapic_id); - - modify_irte(irq, &irte); - - apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " - "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " - "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " - "Avail:%X Vector:%02X Dest:%08X " - "SID:%04X SQ:%X SVT:%X)\n", - attr->ioapic, irte.present, irte.fpd, irte.dst_mode, - irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, - irte.avail, irte.vector, irte.dest_id, - irte.sid, irte.sq, irte.svt); - - memset(entry, 0, sizeof(*entry)); - - entry->index2 = (index >> 15) & 0x1; - entry->zero = 0; - entry->format = 1; - entry->index = (index & 0x7fff); - /* - * IO-APIC RTE will be configured with virtual vector. - * irq handler will do the explicit EOI to the io-apic. - */ - entry->vector = attr->ioapic_pin; - entry->mask = 0; /* enable IRQ */ - entry->trigger = attr->trigger; - entry->polarity = attr->polarity; - - /* Mask level triggered irqs. - * Use IRQ_DELAYED_DISABLE for edge triggered irqs. - */ - if (attr->trigger) - entry->mask = 1; - - return 0; -} - -/* - * Migrate the IO-APIC irq in the presence of intr-remapping. - * - * For both level and edge triggered, irq migration is a simple atomic - * update(of vector and cpu destination) of IRTE and flush the hardware cache. - * - * For level triggered, we eliminate the io-apic RTE modification (with the - * updated vector information), by using a virtual vector (io-apic pin number). - * Real vector that is used for interrupting cpu will be coming from - * the interrupt-remapping table entry. - * - * As the migration is a simple atomic update of IRTE, the same mechanism - * is used to migrate MSI irq's in the presence of interrupt-remapping. - */ -static int -intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) -{ - struct irq_cfg *cfg = data->chip_data; - unsigned int dest, irq = data->irq; - struct irte irte; - - if (!cpumask_intersects(mask, cpu_online_mask)) - return -EINVAL; - - if (get_irte(irq, &irte)) - return -EBUSY; - - if (assign_irq_vector(irq, cfg, mask)) - return -EBUSY; - - dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); - - irte.vector = cfg->vector; - irte.dest_id = IRTE_DEST(dest); - - /* - * Atomically updates the IRTE with the new destination, vector - * and flushes the interrupt entry cache. - */ - modify_irte(irq, &irte); - - /* - * After this point, all the interrupts will start arriving - * at the new destination. So, time to cleanup the previous - * vector allocation. - */ - if (cfg->move_in_progress) - send_cleanup_vector(cfg); - - cpumask_copy(data->affinity, mask); - return 0; -} - -static void intel_compose_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id) -{ - struct irq_cfg *cfg; - struct irte irte; - u16 sub_handle; - int ir_index; - - cfg = irq_get_chip_data(irq); - - ir_index = map_irq_to_irte_handle(irq, &sub_handle); - BUG_ON(ir_index == -1); - - prepare_irte(&irte, cfg->vector, dest); - - /* Set source-id of interrupt request */ - if (pdev) - set_msi_sid(&irte, pdev); - else - set_hpet_sid(&irte, hpet_id); - - modify_irte(irq, &irte); - - msg->address_hi = MSI_ADDR_BASE_HI; - msg->data = sub_handle; - msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | - MSI_ADDR_IR_SHV | - MSI_ADDR_IR_INDEX1(ir_index) | - MSI_ADDR_IR_INDEX2(ir_index); -} - -/* - * Map the PCI dev to the corresponding remapping hardware unit - * and allocate 'nvec' consecutive interrupt-remapping table entries - * in it. - */ -static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) -{ - struct intel_iommu *iommu; - int index; - - iommu = map_dev_to_ir(dev); - if (!iommu) { - printk(KERN_ERR - "Unable to map PCI %s to iommu\n", pci_name(dev)); - return -ENOENT; - } - - index = alloc_irte(iommu, irq, nvec); - if (index < 0) { - printk(KERN_ERR - "Unable to allocate %d IRTE for PCI %s\n", nvec, - pci_name(dev)); - return -ENOSPC; - } - return index; -} - -static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) -{ - struct intel_iommu *iommu; - - iommu = map_dev_to_ir(pdev); - if (!iommu) - return -ENOENT; - /* - * setup the mapping between the irq and the IRTE - * base index, the sub_handle pointing to the - * appropriate interrupt remap table entry. - */ - set_irte_irq(irq, iommu, index, sub_handle); - - return 0; -} - -static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) -{ - struct intel_iommu *iommu = map_hpet_to_ir(id); - int index; - - if (!iommu) - return -1; - - index = alloc_irte(iommu, irq, 1); - if (index < 0) - return -1; - - return 0; -} - -struct irq_remap_ops intel_irq_remap_ops = { - .supported = intel_irq_remapping_supported, - .prepare = dmar_table_init, - .enable = intel_enable_irq_remapping, - .disable = disable_irq_remapping, - .reenable = reenable_irq_remapping, - .enable_faulting = enable_drhd_fault_handling, - .setup_ioapic_entry = intel_setup_ioapic_entry, - .set_affinity = intel_ioapic_set_affinity, - .free_irq = free_irte, - .compose_msi_msg = intel_compose_msi_msg, - .msi_alloc_irq = intel_msi_alloc_irq, - .msi_setup_irq = intel_msi_setup_irq, - .setup_hpet_msi = intel_setup_hpet_msi, -}; diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c new file mode 100644 index 00000000000..b4d39507681 --- /dev/null +++ b/drivers/iommu/intel_irq_remapping.c @@ -0,0 +1,1065 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "irq_remapping.h" + +struct ioapic_scope { + struct intel_iommu *iommu; + unsigned int id; + unsigned int bus; /* PCI bus number */ + unsigned int devfn; /* PCI devfn number */ +}; + +struct hpet_scope { + struct intel_iommu *iommu; + u8 id; + unsigned int bus; + unsigned int devfn; +}; + +#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) +#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8) + +static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; +static struct hpet_scope ir_hpet[MAX_HPET_TBS]; +static int ir_ioapic_num, ir_hpet_num; + +static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); + +static struct irq_2_iommu *irq_2_iommu(unsigned int irq) +{ + struct irq_cfg *cfg = irq_get_chip_data(irq); + return cfg ? &cfg->irq_2_iommu : NULL; +} + +int get_irte(int irq, struct irte *entry) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int index; + + if (!entry || !irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + index = irq_iommu->irte_index + irq_iommu->sub_handle; + *entry = *(irq_iommu->iommu->ir_table->base + index); + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + return 0; +} + +static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) +{ + struct ir_table *table = iommu->ir_table; + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + u16 index, start_index; + unsigned int mask = 0; + unsigned long flags; + int i; + + if (!count || !irq_iommu) + return -1; + + /* + * start the IRTE search from index 0. + */ + index = start_index = 0; + + if (count > 1) { + count = __roundup_pow_of_two(count); + mask = ilog2(count); + } + + if (mask > ecap_max_handle_mask(iommu->ecap)) { + printk(KERN_ERR + "Requested mask %x exceeds the max invalidation handle" + " mask value %Lx\n", mask, + ecap_max_handle_mask(iommu->ecap)); + return -1; + } + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + do { + for (i = index; i < index + count; i++) + if (table->base[i].present) + break; + /* empty index found */ + if (i == index + count) + break; + + index = (index + count) % INTR_REMAP_TABLE_ENTRIES; + + if (index == start_index) { + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + printk(KERN_ERR "can't allocate an IRTE\n"); + return -1; + } + } while (1); + + for (i = index; i < index + count; i++) + table->base[i].present = 1; + + irq_iommu->iommu = iommu; + irq_iommu->irte_index = index; + irq_iommu->sub_handle = 0; + irq_iommu->irte_mask = mask; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return index; +} + +static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) +{ + struct qi_desc desc; + + desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) + | QI_IEC_SELECTIVE; + desc.high = 0; + + return qi_submit_sync(&desc, iommu); +} + +static int map_irq_to_irte_handle(int irq, u16 *sub_handle) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int index; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + *sub_handle = irq_iommu->sub_handle; + index = irq_iommu->irte_index; + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + return index; +} + +static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + irq_iommu->iommu = iommu; + irq_iommu->irte_index = index; + irq_iommu->sub_handle = subhandle; + irq_iommu->irte_mask = 0; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return 0; +} + +static int modify_irte(int irq, struct irte *irte_modified) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + struct intel_iommu *iommu; + unsigned long flags; + struct irte *irte; + int rc, index; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + iommu = irq_iommu->iommu; + + index = irq_iommu->irte_index + irq_iommu->sub_handle; + irte = &iommu->ir_table->base[index]; + + set_64bit(&irte->low, irte_modified->low); + set_64bit(&irte->high, irte_modified->high); + __iommu_flush_cache(iommu, irte, sizeof(*irte)); + + rc = qi_flush_iec(iommu, index, 0); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return rc; +} + +static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) +{ + int i; + + for (i = 0; i < MAX_HPET_TBS; i++) + if (ir_hpet[i].id == hpet_id) + return ir_hpet[i].iommu; + return NULL; +} + +static struct intel_iommu *map_ioapic_to_ir(int apic) +{ + int i; + + for (i = 0; i < MAX_IO_APICS; i++) + if (ir_ioapic[i].id == apic) + return ir_ioapic[i].iommu; + return NULL; +} + +static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) +{ + struct dmar_drhd_unit *drhd; + + drhd = dmar_find_matched_drhd_unit(dev); + if (!drhd) + return NULL; + + return drhd->iommu; +} + +static int clear_entries(struct irq_2_iommu *irq_iommu) +{ + struct irte *start, *entry, *end; + struct intel_iommu *iommu; + int index; + + if (irq_iommu->sub_handle) + return 0; + + iommu = irq_iommu->iommu; + index = irq_iommu->irte_index + irq_iommu->sub_handle; + + start = iommu->ir_table->base + index; + end = start + (1 << irq_iommu->irte_mask); + + for (entry = start; entry < end; entry++) { + set_64bit(&entry->low, 0); + set_64bit(&entry->high, 0); + } + + return qi_flush_iec(iommu, index, irq_iommu->irte_mask); +} + +static int free_irte(int irq) +{ + struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + unsigned long flags; + int rc; + + if (!irq_iommu) + return -1; + + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + + rc = clear_entries(irq_iommu); + + irq_iommu->iommu = NULL; + irq_iommu->irte_index = 0; + irq_iommu->sub_handle = 0; + irq_iommu->irte_mask = 0; + + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); + + return rc; +} + +/* + * source validation type + */ +#define SVT_NO_VERIFY 0x0 /* no verification is required */ +#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ +#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ + +/* + * source-id qualifier + */ +#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ +#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore + * the third least significant bit + */ +#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore + * the second and third least significant bits + */ +#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore + * the least three significant bits + */ + +/* + * set SVT, SQ and SID fields of irte to verify + * source ids of interrupt requests + */ +static void set_irte_sid(struct irte *irte, unsigned int svt, + unsigned int sq, unsigned int sid) +{ + if (disable_sourceid_checking) + svt = SVT_NO_VERIFY; + irte->svt = svt; + irte->sq = sq; + irte->sid = sid; +} + +static int set_ioapic_sid(struct irte *irte, int apic) +{ + int i; + u16 sid = 0; + + if (!irte) + return -1; + + for (i = 0; i < MAX_IO_APICS; i++) { + if (ir_ioapic[i].id == apic) { + sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; + break; + } + } + + if (sid == 0) { + pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); + return -1; + } + + set_irte_sid(irte, 1, 0, sid); + + return 0; +} + +static int set_hpet_sid(struct irte *irte, u8 id) +{ + int i; + u16 sid = 0; + + if (!irte) + return -1; + + for (i = 0; i < MAX_HPET_TBS; i++) { + if (ir_hpet[i].id == id) { + sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; + break; + } + } + + if (sid == 0) { + pr_warning("Failed to set source-id of HPET block (%d)\n", id); + return -1; + } + + /* + * Should really use SQ_ALL_16. Some platforms are broken. + * While we figure out the right quirks for these broken platforms, use + * SQ_13_IGNORE_3 for now. + */ + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); + + return 0; +} + +static int set_msi_sid(struct irte *irte, struct pci_dev *dev) +{ + struct pci_dev *bridge; + + if (!irte || !dev) + return -1; + + /* PCIe device or Root Complex integrated PCI device */ + if (pci_is_pcie(dev) || !dev->bus->parent) { + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, + (dev->bus->number << 8) | dev->devfn); + return 0; + } + + bridge = pci_find_upstream_pcie_bridge(dev); + if (bridge) { + if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */ + set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, + (bridge->bus->number << 8) | dev->bus->number); + else /* this is a legacy PCI bridge */ + set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, + (bridge->bus->number << 8) | bridge->devfn); + } + + return 0; +} + +static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) +{ + u64 addr; + u32 sts; + unsigned long flags; + + addr = virt_to_phys((void *)iommu->ir_table->base); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + dmar_writeq(iommu->reg + DMAR_IRTA_REG, + (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); + + /* Set interrupt-remapping table pointer */ + iommu->gcmd |= DMA_GCMD_SIRTP; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_IRTPS), sts); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); + + /* + * global invalidation of interrupt entry cache before enabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + /* Enable interrupt-remapping */ + iommu->gcmd |= DMA_GCMD_IRE; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_IRES), sts); + + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + + +static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode) +{ + struct ir_table *ir_table; + struct page *pages; + + ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), + GFP_ATOMIC); + + if (!iommu->ir_table) + return -ENOMEM; + + pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, + INTR_REMAP_PAGE_ORDER); + + if (!pages) { + printk(KERN_ERR "failed to allocate pages of order %d\n", + INTR_REMAP_PAGE_ORDER); + kfree(iommu->ir_table); + return -ENOMEM; + } + + ir_table->base = page_address(pages); + + iommu_set_irq_remapping(iommu, mode); + return 0; +} + +/* + * Disable Interrupt Remapping. + */ +static void iommu_disable_irq_remapping(struct intel_iommu *iommu) +{ + unsigned long flags; + u32 sts; + + if (!ecap_ir_support(iommu->ecap)) + return; + + /* + * global invalidation of interrupt entry cache before disabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + + raw_spin_lock_irqsave(&iommu->register_lock, flags); + + sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); + if (!(sts & DMA_GSTS_IRES)) + goto end; + + iommu->gcmd &= ~DMA_GCMD_IRE; + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, !(sts & DMA_GSTS_IRES), sts); + +end: + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + +static int __init dmar_x2apic_optout(void) +{ + struct acpi_table_dmar *dmar; + dmar = (struct acpi_table_dmar *)dmar_tbl; + if (!dmar || no_x2apic_optout) + return 0; + return dmar->flags & DMAR_X2APIC_OPT_OUT; +} + +static int __init intel_irq_remapping_supported(void) +{ + struct dmar_drhd_unit *drhd; + + if (disable_irq_remap) + return 0; + + if (!dmar_ir_support()) + return 0; + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + return 0; + } + + return 1; +} + +static int __init intel_enable_irq_remapping(void) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + int eim = 0; + + if (parse_ioapics_under_ir() != 1) { + printk(KERN_INFO "Not enable interrupt remapping\n"); + return -1; + } + + if (x2apic_supported()) { + eim = !dmar_x2apic_optout(); + WARN(!eim, KERN_WARNING + "Your BIOS is broken and requested that x2apic be disabled\n" + "This will leave your machine vulnerable to irq-injection attacks\n" + "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); + } + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + /* + * If the queued invalidation is already initialized, + * shouldn't disable it. + */ + if (iommu->qi) + continue; + + /* + * Clear previous faults. + */ + dmar_fault(-1, iommu); + + /* + * Disable intr remapping and queued invalidation, if already + * enabled prior to OS handover. + */ + iommu_disable_irq_remapping(iommu); + + dmar_disable_qi(iommu); + } + + /* + * check for the Interrupt-remapping support + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + continue; + + if (eim && !ecap_eim_support(iommu->ecap)) { + printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " + " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); + return -1; + } + } + + /* + * Enable queued invalidation for all the DRHD's. + */ + for_each_drhd_unit(drhd) { + int ret; + struct intel_iommu *iommu = drhd->iommu; + ret = dmar_enable_qi(iommu); + + if (ret) { + printk(KERN_ERR "DRHD %Lx: failed to enable queued, " + " invalidation, ecap %Lx, ret %d\n", + drhd->reg_base_addr, iommu->ecap, ret); + return -1; + } + } + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (!ecap_ir_support(iommu->ecap)) + continue; + + if (intel_setup_irq_remapping(iommu, eim)) + goto error; + + setup = 1; + } + + if (!setup) + goto error; + + irq_remapping_enabled = 1; + pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); + + return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + +static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, + struct intel_iommu *iommu) +{ + struct acpi_dmar_pci_path *path; + u8 bus; + int count; + + bus = scope->bus; + path = (struct acpi_dmar_pci_path *)(scope + 1); + count = (scope->length - sizeof(struct acpi_dmar_device_scope)) + / sizeof(struct acpi_dmar_pci_path); + + while (--count > 0) { + /* + * Access PCI directly due to the PCI + * subsystem isn't initialized yet. + */ + bus = read_pci_config_byte(bus, path->dev, path->fn, + PCI_SECONDARY_BUS); + path++; + } + ir_hpet[ir_hpet_num].bus = bus; + ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn); + ir_hpet[ir_hpet_num].iommu = iommu; + ir_hpet[ir_hpet_num].id = scope->enumeration_id; + ir_hpet_num++; +} + +static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, + struct intel_iommu *iommu) +{ + struct acpi_dmar_pci_path *path; + u8 bus; + int count; + + bus = scope->bus; + path = (struct acpi_dmar_pci_path *)(scope + 1); + count = (scope->length - sizeof(struct acpi_dmar_device_scope)) + / sizeof(struct acpi_dmar_pci_path); + + while (--count > 0) { + /* + * Access PCI directly due to the PCI + * subsystem isn't initialized yet. + */ + bus = read_pci_config_byte(bus, path->dev, path->fn, + PCI_SECONDARY_BUS); + path++; + } + + ir_ioapic[ir_ioapic_num].bus = bus; + ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn); + ir_ioapic[ir_ioapic_num].iommu = iommu; + ir_ioapic[ir_ioapic_num].id = scope->enumeration_id; + ir_ioapic_num++; +} + +static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, + struct intel_iommu *iommu) +{ + struct acpi_dmar_hardware_unit *drhd; + struct acpi_dmar_device_scope *scope; + void *start, *end; + + drhd = (struct acpi_dmar_hardware_unit *)header; + + start = (void *)(drhd + 1); + end = ((void *)drhd) + header->length; + + while (start < end) { + scope = start; + if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { + if (ir_ioapic_num == MAX_IO_APICS) { + printk(KERN_WARNING "Exceeded Max IO APICS\n"); + return -1; + } + + printk(KERN_INFO "IOAPIC id %d under DRHD base " + " 0x%Lx IOMMU %d\n", scope->enumeration_id, + drhd->address, iommu->seq_id); + + ir_parse_one_ioapic_scope(scope, iommu); + } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) { + if (ir_hpet_num == MAX_HPET_TBS) { + printk(KERN_WARNING "Exceeded Max HPET blocks\n"); + return -1; + } + + printk(KERN_INFO "HPET id %d under DRHD base" + " 0x%Lx\n", scope->enumeration_id, + drhd->address); + + ir_parse_one_hpet_scope(scope, iommu); + } + start += scope->length; + } + + return 0; +} + +/* + * Finds the assocaition between IOAPIC's and its Interrupt-remapping + * hardware unit. + */ +int __init parse_ioapics_under_ir(void) +{ + struct dmar_drhd_unit *drhd; + int ir_supported = 0; + + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu = drhd->iommu; + + if (ecap_ir_support(iommu->ecap)) { + if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) + return -1; + + ir_supported = 1; + } + } + + if (ir_supported && ir_ioapic_num != nr_ioapics) { + printk(KERN_WARNING + "Not all IO-APIC's listed under remapping hardware\n"); + return -1; + } + + return ir_supported; +} + +int __init ir_dev_scope_init(void) +{ + if (!irq_remapping_enabled) + return 0; + + return dmar_dev_scope_init(); +} +rootfs_initcall(ir_dev_scope_init); + +static void disable_irq_remapping(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + + /* + * Disable Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + iommu_disable_irq_remapping(iommu); + } +} + +static int reenable_irq_remapping(int eim) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + struct intel_iommu *iommu = NULL; + + for_each_iommu(iommu, drhd) + if (iommu->qi) + dmar_reenable_qi(iommu); + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + /* Set up interrupt remapping for iommu.*/ + iommu_set_irq_remapping(iommu, eim); + setup = 1; + } + + if (!setup) + goto error; + + return 0; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + +static void prepare_irte(struct irte *irte, int vector, + unsigned int dest) +{ + memset(irte, 0, sizeof(*irte)); + + irte->present = 1; + irte->dst_mode = apic->irq_dest_mode; + /* + * Trigger mode in the IRTE will always be edge, and for IO-APIC, the + * actual level or edge trigger will be setup in the IO-APIC + * RTE. This will help simplify level triggered irq migration. + * For more details, see the comments (in io_apic.c) explainig IO-APIC + * irq migration in the presence of interrupt-remapping. + */ + irte->trigger_mode = 0; + irte->dlvry_mode = apic->irq_delivery_mode; + irte->vector = vector; + irte->dest_id = IRTE_DEST(dest); + irte->redir_hint = 1; +} + +static int intel_setup_ioapic_entry(int irq, + struct IO_APIC_route_entry *route_entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) +{ + int ioapic_id = mpc_ioapic_id(attr->ioapic); + struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id); + struct IR_IO_APIC_route_entry *entry; + struct irte irte; + int index; + + if (!iommu) { + pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); + return -ENODEV; + } + + entry = (struct IR_IO_APIC_route_entry *)route_entry; + + index = alloc_irte(iommu, irq, 1); + if (index < 0) { + pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id); + return -ENOMEM; + } + + prepare_irte(&irte, vector, destination); + + /* Set source-id of interrupt request */ + set_ioapic_sid(&irte, ioapic_id); + + modify_irte(irq, &irte); + + apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " + "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " + "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " + "Avail:%X Vector:%02X Dest:%08X " + "SID:%04X SQ:%X SVT:%X)\n", + attr->ioapic, irte.present, irte.fpd, irte.dst_mode, + irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, + irte.avail, irte.vector, irte.dest_id, + irte.sid, irte.sq, irte.svt); + + memset(entry, 0, sizeof(*entry)); + + entry->index2 = (index >> 15) & 0x1; + entry->zero = 0; + entry->format = 1; + entry->index = (index & 0x7fff); + /* + * IO-APIC RTE will be configured with virtual vector. + * irq handler will do the explicit EOI to the io-apic. + */ + entry->vector = attr->ioapic_pin; + entry->mask = 0; /* enable IRQ */ + entry->trigger = attr->trigger; + entry->polarity = attr->polarity; + + /* Mask level triggered irqs. + * Use IRQ_DELAYED_DISABLE for edge triggered irqs. + */ + if (attr->trigger) + entry->mask = 1; + + return 0; +} + +/* + * Migrate the IO-APIC irq in the presence of intr-remapping. + * + * For both level and edge triggered, irq migration is a simple atomic + * update(of vector and cpu destination) of IRTE and flush the hardware cache. + * + * For level triggered, we eliminate the io-apic RTE modification (with the + * updated vector information), by using a virtual vector (io-apic pin number). + * Real vector that is used for interrupting cpu will be coming from + * the interrupt-remapping table entry. + * + * As the migration is a simple atomic update of IRTE, the same mechanism + * is used to migrate MSI irq's in the presence of interrupt-remapping. + */ +static int +intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) +{ + struct irq_cfg *cfg = data->chip_data; + unsigned int dest, irq = data->irq; + struct irte irte; + + if (!cpumask_intersects(mask, cpu_online_mask)) + return -EINVAL; + + if (get_irte(irq, &irte)) + return -EBUSY; + + if (assign_irq_vector(irq, cfg, mask)) + return -EBUSY; + + dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); + + irte.vector = cfg->vector; + irte.dest_id = IRTE_DEST(dest); + + /* + * Atomically updates the IRTE with the new destination, vector + * and flushes the interrupt entry cache. + */ + modify_irte(irq, &irte); + + /* + * After this point, all the interrupts will start arriving + * at the new destination. So, time to cleanup the previous + * vector allocation. + */ + if (cfg->move_in_progress) + send_cleanup_vector(cfg); + + cpumask_copy(data->affinity, mask); + return 0; +} + +static void intel_compose_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ + struct irq_cfg *cfg; + struct irte irte; + u16 sub_handle; + int ir_index; + + cfg = irq_get_chip_data(irq); + + ir_index = map_irq_to_irte_handle(irq, &sub_handle); + BUG_ON(ir_index == -1); + + prepare_irte(&irte, cfg->vector, dest); + + /* Set source-id of interrupt request */ + if (pdev) + set_msi_sid(&irte, pdev); + else + set_hpet_sid(&irte, hpet_id); + + modify_irte(irq, &irte); + + msg->address_hi = MSI_ADDR_BASE_HI; + msg->data = sub_handle; + msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | + MSI_ADDR_IR_SHV | + MSI_ADDR_IR_INDEX1(ir_index) | + MSI_ADDR_IR_INDEX2(ir_index); +} + +/* + * Map the PCI dev to the corresponding remapping hardware unit + * and allocate 'nvec' consecutive interrupt-remapping table entries + * in it. + */ +static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) +{ + struct intel_iommu *iommu; + int index; + + iommu = map_dev_to_ir(dev); + if (!iommu) { + printk(KERN_ERR + "Unable to map PCI %s to iommu\n", pci_name(dev)); + return -ENOENT; + } + + index = alloc_irte(iommu, irq, nvec); + if (index < 0) { + printk(KERN_ERR + "Unable to allocate %d IRTE for PCI %s\n", nvec, + pci_name(dev)); + return -ENOSPC; + } + return index; +} + +static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + struct intel_iommu *iommu; + + iommu = map_dev_to_ir(pdev); + if (!iommu) + return -ENOENT; + /* + * setup the mapping between the irq and the IRTE + * base index, the sub_handle pointing to the + * appropriate interrupt remap table entry. + */ + set_irte_irq(irq, iommu, index, sub_handle); + + return 0; +} + +static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) +{ + struct intel_iommu *iommu = map_hpet_to_ir(id); + int index; + + if (!iommu) + return -1; + + index = alloc_irte(iommu, irq, 1); + if (index < 0) + return -1; + + return 0; +} + +struct irq_remap_ops intel_irq_remap_ops = { + .supported = intel_irq_remapping_supported, + .prepare = dmar_table_init, + .enable = intel_enable_irq_remapping, + .disable = disable_irq_remapping, + .reenable = reenable_irq_remapping, + .enable_faulting = enable_drhd_fault_handling, + .setup_ioapic_entry = intel_setup_ioapic_entry, + .set_affinity = intel_ioapic_set_affinity, + .free_irq = free_irte, + .compose_msi_msg = intel_compose_msi_msg, + .msi_alloc_irq = intel_msi_alloc_irq, + .msi_setup_irq = intel_msi_setup_irq, + .setup_hpet_msi = intel_setup_hpet_msi, +}; diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c deleted file mode 100644 index 523a7b3a120..00000000000 --- a/drivers/iommu/intr_remapping.c +++ /dev/null @@ -1,164 +0,0 @@ -#include -#include -#include - -#include "intr_remapping.h" - -int irq_remapping_enabled; - -int disable_irq_remap; -int disable_sourceid_checking; -int no_x2apic_optout; - -static struct irq_remap_ops *remap_ops; - -static __init int setup_nointremap(char *str) -{ - disable_irq_remap = 1; - return 0; -} -early_param("nointremap", setup_nointremap); - -static __init int setup_irqremap(char *str) -{ - if (!str) - return -EINVAL; - - while (*str) { - if (!strncmp(str, "on", 2)) - disable_irq_remap = 0; - else if (!strncmp(str, "off", 3)) - disable_irq_remap = 1; - else if (!strncmp(str, "nosid", 5)) - disable_sourceid_checking = 1; - else if (!strncmp(str, "no_x2apic_optout", 16)) - no_x2apic_optout = 1; - - str += strcspn(str, ","); - while (*str == ',') - str++; - } - - return 0; -} -early_param("intremap", setup_irqremap); - -void __init setup_irq_remapping_ops(void) -{ - remap_ops = &intel_irq_remap_ops; -} - -int irq_remapping_supported(void) -{ - if (disable_irq_remap) - return 0; - - if (!remap_ops || !remap_ops->supported) - return 0; - - return remap_ops->supported(); -} - -int __init irq_remapping_prepare(void) -{ - if (!remap_ops || !remap_ops->prepare) - return -ENODEV; - - return remap_ops->prepare(); -} - -int __init irq_remapping_enable(void) -{ - if (!remap_ops || !remap_ops->enable) - return -ENODEV; - - return remap_ops->enable(); -} - -void irq_remapping_disable(void) -{ - if (!remap_ops || !remap_ops->disable) - return; - - remap_ops->disable(); -} - -int irq_remapping_reenable(int mode) -{ - if (!remap_ops || !remap_ops->reenable) - return 0; - - return remap_ops->reenable(mode); -} - -int __init irq_remap_enable_fault_handling(void) -{ - if (!remap_ops || !remap_ops->enable_faulting) - return -ENODEV; - - return remap_ops->enable_faulting(); -} - -int setup_ioapic_remapped_entry(int irq, - struct IO_APIC_route_entry *entry, - unsigned int destination, int vector, - struct io_apic_irq_attr *attr) -{ - if (!remap_ops || !remap_ops->setup_ioapic_entry) - return -ENODEV; - - return remap_ops->setup_ioapic_entry(irq, entry, destination, - vector, attr); -} - -int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) -{ - if (!remap_ops || !remap_ops->set_affinity) - return 0; - - return remap_ops->set_affinity(data, mask, force); -} - -void free_remapped_irq(int irq) -{ - if (!remap_ops || !remap_ops->free_irq) - return; - - remap_ops->free_irq(irq); -} - -void compose_remapped_msi_msg(struct pci_dev *pdev, - unsigned int irq, unsigned int dest, - struct msi_msg *msg, u8 hpet_id) -{ - if (!remap_ops || !remap_ops->compose_msi_msg) - return; - - remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); -} - -int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) -{ - if (!remap_ops || !remap_ops->msi_alloc_irq) - return -ENODEV; - - return remap_ops->msi_alloc_irq(pdev, irq, nvec); -} - -int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) -{ - if (!remap_ops || !remap_ops->msi_setup_irq) - return -ENODEV; - - return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle); -} - -int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) -{ - if (!remap_ops || !remap_ops->setup_hpet_msi) - return -ENODEV; - - return remap_ops->setup_hpet_msi(irq, id); -} diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h deleted file mode 100644 index bd5d98fec14..00000000000 --- a/drivers/iommu/intr_remapping.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2012 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * This header file contains stuff that is shared between different interrupt - * remapping drivers but with no need to be visible outside of the IOMMU layer. - */ - -#ifndef __INTR_REMAPPING_H -#define __INTR_REMAPPING_H - -#ifdef CONFIG_IRQ_REMAP - -struct IO_APIC_route_entry; -struct io_apic_irq_attr; -struct irq_data; -struct cpumask; -struct pci_dev; -struct msi_msg; - -extern int disable_irq_remap; -extern int disable_sourceid_checking; -extern int no_x2apic_optout; - -struct irq_remap_ops { - /* Check whether Interrupt Remapping is supported */ - int (*supported)(void); - - /* Initializes hardware and makes it ready for remapping interrupts */ - int (*prepare)(void); - - /* Enables the remapping hardware */ - int (*enable)(void); - - /* Disables the remapping hardware */ - void (*disable)(void); - - /* Reenables the remapping hardware */ - int (*reenable)(int); - - /* Enable fault handling */ - int (*enable_faulting)(void); - - /* IO-APIC setup routine */ - int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *, - unsigned int, int, - struct io_apic_irq_attr *); - - /* Set the CPU affinity of a remapped interrupt */ - int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, - bool force); - - /* Free an IRQ */ - int (*free_irq)(int); - - /* Create MSI msg to use for interrupt remapping */ - void (*compose_msi_msg)(struct pci_dev *, - unsigned int, unsigned int, - struct msi_msg *, u8); - - /* Allocate remapping resources for MSI */ - int (*msi_alloc_irq)(struct pci_dev *, int, int); - - /* Setup the remapped MSI irq */ - int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int); - - /* Setup interrupt remapping for an HPET MSI */ - int (*setup_hpet_msi)(unsigned int, unsigned int); -}; - -extern struct irq_remap_ops intel_irq_remap_ops; - -#endif /* CONFIG_IRQ_REMAP */ - -#endif /* __INTR_REMAPPING_H */ diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c new file mode 100644 index 00000000000..1cf350e02da --- /dev/null +++ b/drivers/iommu/irq_remapping.c @@ -0,0 +1,164 @@ +#include +#include +#include + +#include "irq_remapping.h" + +int irq_remapping_enabled; + +int disable_irq_remap; +int disable_sourceid_checking; +int no_x2apic_optout; + +static struct irq_remap_ops *remap_ops; + +static __init int setup_nointremap(char *str) +{ + disable_irq_remap = 1; + return 0; +} +early_param("nointremap", setup_nointremap); + +static __init int setup_irqremap(char *str) +{ + if (!str) + return -EINVAL; + + while (*str) { + if (!strncmp(str, "on", 2)) + disable_irq_remap = 0; + else if (!strncmp(str, "off", 3)) + disable_irq_remap = 1; + else if (!strncmp(str, "nosid", 5)) + disable_sourceid_checking = 1; + else if (!strncmp(str, "no_x2apic_optout", 16)) + no_x2apic_optout = 1; + + str += strcspn(str, ","); + while (*str == ',') + str++; + } + + return 0; +} +early_param("intremap", setup_irqremap); + +void __init setup_irq_remapping_ops(void) +{ + remap_ops = &intel_irq_remap_ops; +} + +int irq_remapping_supported(void) +{ + if (disable_irq_remap) + return 0; + + if (!remap_ops || !remap_ops->supported) + return 0; + + return remap_ops->supported(); +} + +int __init irq_remapping_prepare(void) +{ + if (!remap_ops || !remap_ops->prepare) + return -ENODEV; + + return remap_ops->prepare(); +} + +int __init irq_remapping_enable(void) +{ + if (!remap_ops || !remap_ops->enable) + return -ENODEV; + + return remap_ops->enable(); +} + +void irq_remapping_disable(void) +{ + if (!remap_ops || !remap_ops->disable) + return; + + remap_ops->disable(); +} + +int irq_remapping_reenable(int mode) +{ + if (!remap_ops || !remap_ops->reenable) + return 0; + + return remap_ops->reenable(mode); +} + +int __init irq_remap_enable_fault_handling(void) +{ + if (!remap_ops || !remap_ops->enable_faulting) + return -ENODEV; + + return remap_ops->enable_faulting(); +} + +int setup_ioapic_remapped_entry(int irq, + struct IO_APIC_route_entry *entry, + unsigned int destination, int vector, + struct io_apic_irq_attr *attr) +{ + if (!remap_ops || !remap_ops->setup_ioapic_entry) + return -ENODEV; + + return remap_ops->setup_ioapic_entry(irq, entry, destination, + vector, attr); +} + +int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) +{ + if (!remap_ops || !remap_ops->set_affinity) + return 0; + + return remap_ops->set_affinity(data, mask, force); +} + +void free_remapped_irq(int irq) +{ + if (!remap_ops || !remap_ops->free_irq) + return; + + remap_ops->free_irq(irq); +} + +void compose_remapped_msi_msg(struct pci_dev *pdev, + unsigned int irq, unsigned int dest, + struct msi_msg *msg, u8 hpet_id) +{ + if (!remap_ops || !remap_ops->compose_msi_msg) + return; + + remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); +} + +int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) +{ + if (!remap_ops || !remap_ops->msi_alloc_irq) + return -ENODEV; + + return remap_ops->msi_alloc_irq(pdev, irq, nvec); +} + +int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) +{ + if (!remap_ops || !remap_ops->msi_setup_irq) + return -ENODEV; + + return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle); +} + +int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) +{ + if (!remap_ops || !remap_ops->setup_hpet_msi) + return -ENODEV; + + return remap_ops->setup_hpet_msi(irq, id); +} diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h new file mode 100644 index 00000000000..b12974cc1df --- /dev/null +++ b/drivers/iommu/irq_remapping.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2012 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * This header file contains stuff that is shared between different interrupt + * remapping drivers but with no need to be visible outside of the IOMMU layer. + */ + +#ifndef __IRQ_REMAPPING_H +#define __IRQ_REMAPPING_H + +#ifdef CONFIG_IRQ_REMAP + +struct IO_APIC_route_entry; +struct io_apic_irq_attr; +struct irq_data; +struct cpumask; +struct pci_dev; +struct msi_msg; + +extern int disable_irq_remap; +extern int disable_sourceid_checking; +extern int no_x2apic_optout; + +struct irq_remap_ops { + /* Check whether Interrupt Remapping is supported */ + int (*supported)(void); + + /* Initializes hardware and makes it ready for remapping interrupts */ + int (*prepare)(void); + + /* Enables the remapping hardware */ + int (*enable)(void); + + /* Disables the remapping hardware */ + void (*disable)(void); + + /* Reenables the remapping hardware */ + int (*reenable)(int); + + /* Enable fault handling */ + int (*enable_faulting)(void); + + /* IO-APIC setup routine */ + int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *, + unsigned int, int, + struct io_apic_irq_attr *); + + /* Set the CPU affinity of a remapped interrupt */ + int (*set_affinity)(struct irq_data *data, const struct cpumask *mask, + bool force); + + /* Free an IRQ */ + int (*free_irq)(int); + + /* Create MSI msg to use for interrupt remapping */ + void (*compose_msi_msg)(struct pci_dev *, + unsigned int, unsigned int, + struct msi_msg *, u8); + + /* Allocate remapping resources for MSI */ + int (*msi_alloc_irq)(struct pci_dev *, int, int); + + /* Setup the remapped MSI irq */ + int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int); + + /* Setup interrupt remapping for an HPET MSI */ + int (*setup_hpet_msi)(unsigned int, unsigned int); +}; + +extern struct irq_remap_ops intel_irq_remap_ops; + +#endif /* CONFIG_IRQ_REMAP */ + +#endif /* __IRQ_REMAPPING_H */ -- cgit v1.2.3