diff options
author | Marc Zyngier <marc.zyngier@arm.com> | 2014-11-24 14:35:11 +0000 |
---|---|---|
committer | Jason Cooper <jason@lakedaemon.net> | 2014-11-26 15:55:12 +0000 |
commit | cc2d3216f53c9fff0030eb71cacc4ce5f39d1d7e (patch) | |
tree | 5ee3d58aab214a297e5dc906caf08211cef3d21d /drivers/irqchip | |
parent | f5c1434c217fd72ac0d24d3142d09e49a3d4e72e (diff) | |
download | linux-stable-cc2d3216f53c9fff0030eb71cacc4ce5f39d1d7e.tar.gz linux-stable-cc2d3216f53c9fff0030eb71cacc4ce5f39d1d7e.tar.bz2 linux-stable-cc2d3216f53c9fff0030eb71cacc4ce5f39d1d7e.zip |
irqchip: GICv3: ITS command queue
The ITS is configured through a number commands that the driver
issues to the HW using a memory-based circular buffer.
This patch implements the subset of commands that are required
for Linux.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Link: https://lkml.kernel.org/r/1416839720-18400-5-git-send-email-marc.zyngier@arm.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
Diffstat (limited to 'drivers/irqchip')
-rw-r--r-- | drivers/irqchip/irq-gic-v3-its.c | 511 |
1 files changed, 511 insertions, 0 deletions
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c new file mode 100644 index 000000000000..a5ab12c7d7fc --- /dev/null +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -0,0 +1,511 @@ +/* + * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/bitmap.h> +#include <linux/cpu.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/log2.h> +#include <linux/mm.h> +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/percpu.h> +#include <linux/slab.h> + +#include <linux/irqchip/arm-gic-v3.h> + +#include <asm/cacheflush.h> +#include <asm/cputype.h> +#include <asm/exception.h> + +#include "irqchip.h" + +#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1 << 0) + +/* + * Collection structure - just an ID, and a redistributor address to + * ping. We use one per CPU as a bag of interrupts assigned to this + * CPU. + */ +struct its_collection { + u64 target_address; + u16 col_id; +}; + +/* + * The ITS structure - contains most of the infrastructure, with the + * msi_controller, the command queue, the collections, and the list of + * devices writing to it. + */ +struct its_node { + raw_spinlock_t lock; + struct list_head entry; + struct msi_controller msi_chip; + struct irq_domain *domain; + void __iomem *base; + unsigned long phys_base; + struct its_cmd_block *cmd_base; + struct its_cmd_block *cmd_write; + void *tables[GITS_BASER_NR_REGS]; + struct its_collection *collections; + struct list_head its_device_list; + u64 flags; + u32 ite_size; +}; + +#define ITS_ITT_ALIGN SZ_256 + +/* + * The ITS view of a device - belongs to an ITS, a collection, owns an + * interrupt translation table, and a list of interrupts. + */ +struct its_device { + struct list_head entry; + struct its_node *its; + struct its_collection *collection; + void *itt; + unsigned long *lpi_map; + irq_hw_number_t lpi_base; + int nr_lpis; + u32 nr_ites; + u32 device_id; +}; + +/* + * ITS command descriptors - parameters to be encoded in a command + * block. + */ +struct its_cmd_desc { + union { + struct { + struct its_device *dev; + u32 event_id; + } its_inv_cmd; + + struct { + struct its_device *dev; + u32 event_id; + } its_int_cmd; + + struct { + struct its_device *dev; + int valid; + } its_mapd_cmd; + + struct { + struct its_collection *col; + int valid; + } its_mapc_cmd; + + struct { + struct its_device *dev; + u32 phys_id; + u32 event_id; + } its_mapvi_cmd; + + struct { + struct its_device *dev; + struct its_collection *col; + u32 id; + } its_movi_cmd; + + struct { + struct its_device *dev; + u32 event_id; + } its_discard_cmd; + + struct { + struct its_collection *col; + } its_invall_cmd; + }; +}; + +/* + * The ITS command block, which is what the ITS actually parses. + */ +struct its_cmd_block { + u64 raw_cmd[4]; +}; + +#define ITS_CMD_QUEUE_SZ SZ_64K +#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) + +typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, + struct its_cmd_desc *); + +static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) +{ + cmd->raw_cmd[0] &= ~0xffUL; + cmd->raw_cmd[0] |= cmd_nr; +} + +static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) +{ + cmd->raw_cmd[0] &= ~(0xffffUL << 32); + cmd->raw_cmd[0] |= ((u64)devid) << 32; +} + +static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) +{ + cmd->raw_cmd[1] &= ~0xffffffffUL; + cmd->raw_cmd[1] |= id; +} + +static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) +{ + cmd->raw_cmd[1] &= 0xffffffffUL; + cmd->raw_cmd[1] |= ((u64)phys_id) << 32; +} + +static void its_encode_size(struct its_cmd_block *cmd, u8 size) +{ + cmd->raw_cmd[1] &= ~0x1fUL; + cmd->raw_cmd[1] |= size & 0x1f; +} + +static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) +{ + cmd->raw_cmd[2] &= ~0xffffffffffffUL; + cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL; +} + +static void its_encode_valid(struct its_cmd_block *cmd, int valid) +{ + cmd->raw_cmd[2] &= ~(1UL << 63); + cmd->raw_cmd[2] |= ((u64)!!valid) << 63; +} + +static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) +{ + cmd->raw_cmd[2] &= ~(0xffffffffUL << 16); + cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16)); +} + +static void its_encode_collection(struct its_cmd_block *cmd, u16 col) +{ + cmd->raw_cmd[2] &= ~0xffffUL; + cmd->raw_cmd[2] |= col; +} + +static inline void its_fixup_cmd(struct its_cmd_block *cmd) +{ + /* Let's fixup BE commands */ + cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); + cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); + cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); + cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); +} + +static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + unsigned long itt_addr; + u8 size = order_base_2(desc->its_mapd_cmd.dev->nr_ites); + + itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); + itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); + + its_encode_cmd(cmd, GITS_CMD_MAPD); + its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); + its_encode_size(cmd, size - 1); + its_encode_itt(cmd, itt_addr); + its_encode_valid(cmd, desc->its_mapd_cmd.valid); + + its_fixup_cmd(cmd); + + return desc->its_mapd_cmd.dev->collection; +} + +static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MAPC); + its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); + its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); + its_encode_valid(cmd, desc->its_mapc_cmd.valid); + + its_fixup_cmd(cmd); + + return desc->its_mapc_cmd.col; +} + +static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MAPVI); + its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id); + its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id); + its_encode_collection(cmd, desc->its_mapvi_cmd.dev->collection->col_id); + + its_fixup_cmd(cmd); + + return desc->its_mapvi_cmd.dev->collection; +} + +static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MOVI); + its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_movi_cmd.id); + its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); + + its_fixup_cmd(cmd); + + return desc->its_movi_cmd.dev->collection; +} + +static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_DISCARD); + its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_discard_cmd.event_id); + + its_fixup_cmd(cmd); + + return desc->its_discard_cmd.dev->collection; +} + +static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_INV); + its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_inv_cmd.event_id); + + its_fixup_cmd(cmd); + + return desc->its_inv_cmd.dev->collection; +} + +static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_INVALL); + its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); + + its_fixup_cmd(cmd); + + return NULL; +} + +static u64 its_cmd_ptr_to_offset(struct its_node *its, + struct its_cmd_block *ptr) +{ + return (ptr - its->cmd_base) * sizeof(*ptr); +} + +static int its_queue_full(struct its_node *its) +{ + int widx; + int ridx; + + widx = its->cmd_write - its->cmd_base; + ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); + + /* This is incredibly unlikely to happen, unless the ITS locks up. */ + if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) + return 1; + + return 0; +} + +static struct its_cmd_block *its_allocate_entry(struct its_node *its) +{ + struct its_cmd_block *cmd; + u32 count = 1000000; /* 1s! */ + + while (its_queue_full(its)) { + count--; + if (!count) { + pr_err_ratelimited("ITS queue not draining\n"); + return NULL; + } + cpu_relax(); + udelay(1); + } + + cmd = its->cmd_write++; + + /* Handle queue wrapping */ + if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) + its->cmd_write = its->cmd_base; + + return cmd; +} + +static struct its_cmd_block *its_post_commands(struct its_node *its) +{ + u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); + + writel_relaxed(wr, its->base + GITS_CWRITER); + + return its->cmd_write; +} + +static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) +{ + /* + * Make sure the commands written to memory are observable by + * the ITS. + */ + if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) + __flush_dcache_area(cmd, sizeof(*cmd)); + else + dsb(ishst); +} + +static void its_wait_for_range_completion(struct its_node *its, + struct its_cmd_block *from, + struct its_cmd_block *to) +{ + u64 rd_idx, from_idx, to_idx; + u32 count = 1000000; /* 1s! */ + + from_idx = its_cmd_ptr_to_offset(its, from); + to_idx = its_cmd_ptr_to_offset(its, to); + + while (1) { + rd_idx = readl_relaxed(its->base + GITS_CREADR); + if (rd_idx >= to_idx || rd_idx < from_idx) + break; + + count--; + if (!count) { + pr_err_ratelimited("ITS queue timeout\n"); + return; + } + cpu_relax(); + udelay(1); + } +} + +static void its_send_single_command(struct its_node *its, + its_cmd_builder_t builder, + struct its_cmd_desc *desc) +{ + struct its_cmd_block *cmd, *sync_cmd, *next_cmd; + struct its_collection *sync_col; + + raw_spin_lock(&its->lock); + + cmd = its_allocate_entry(its); + if (!cmd) { /* We're soooooo screewed... */ + pr_err_ratelimited("ITS can't allocate, dropping command\n"); + raw_spin_unlock(&its->lock); + return; + } + sync_col = builder(cmd, desc); + its_flush_cmd(its, cmd); + + if (sync_col) { + sync_cmd = its_allocate_entry(its); + if (!sync_cmd) { + pr_err_ratelimited("ITS can't SYNC, skipping\n"); + goto post; + } + its_encode_cmd(sync_cmd, GITS_CMD_SYNC); + its_encode_target(sync_cmd, sync_col->target_address); + its_fixup_cmd(sync_cmd); + its_flush_cmd(its, sync_cmd); + } + +post: + next_cmd = its_post_commands(its); + raw_spin_unlock(&its->lock); + + its_wait_for_range_completion(its, cmd, next_cmd); +} + +static void its_send_inv(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; + + desc.its_inv_cmd.dev = dev; + desc.its_inv_cmd.event_id = event_id; + + its_send_single_command(dev->its, its_build_inv_cmd, &desc); +} + +static void its_send_mapd(struct its_device *dev, int valid) +{ + struct its_cmd_desc desc; + + desc.its_mapd_cmd.dev = dev; + desc.its_mapd_cmd.valid = !!valid; + + its_send_single_command(dev->its, its_build_mapd_cmd, &desc); +} + +static void its_send_mapc(struct its_node *its, struct its_collection *col, + int valid) +{ + struct its_cmd_desc desc; + + desc.its_mapc_cmd.col = col; + desc.its_mapc_cmd.valid = !!valid; + + its_send_single_command(its, its_build_mapc_cmd, &desc); +} + +static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_mapvi_cmd.dev = dev; + desc.its_mapvi_cmd.phys_id = irq_id; + desc.its_mapvi_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_mapvi_cmd, &desc); +} + +static void its_send_movi(struct its_device *dev, + struct its_collection *col, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_movi_cmd.dev = dev; + desc.its_movi_cmd.col = col; + desc.its_movi_cmd.id = id; + + its_send_single_command(dev->its, its_build_movi_cmd, &desc); +} + +static void its_send_discard(struct its_device *dev, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_discard_cmd.dev = dev; + desc.its_discard_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_discard_cmd, &desc); +} + +static void its_send_invall(struct its_node *its, struct its_collection *col) +{ + struct its_cmd_desc desc; + + desc.its_invall_cmd.col = col; + + its_send_single_command(its, its_build_invall_cmd, &desc); +} |