armv7: Add fine-grained page table support

This patch adds an mmu_config_range_kb() function, which can set memory
types at the 4KB level by chaining a fine-grained page table to an
existing superpage entry. It is only intended for special cases where
this level of precision is really necessary and therefore comes with a
few practical limitations (the area for each invocation must be confined
within a single superpage, and you are not allowed to remap the same
region with mmu_config_range() again later). Since the fine-grained page
tables need some space, boards intending to use this feature must define
a TTB_SUBTABLES() region in their memlayout.ld.

BUG=chrome-os-partner:32848
TEST=Booted both Veyron_Pinky (normal) and Nyan_Blaze (LPAE), ensured
that they still work. Checksummed the page tables with and without this
patch, confirmed that they end up equal. Hacked in some subtable test
entries, hexdumped all tables and manually confirmed that they look as
expected.

Change-Id: I8c3eb7c2eb9c82e2abc5f2c0dda91f5b2eee7023
Signed-off-by: Patrick Georgi <pgeorgi@chromium.org>
Original-Commit-Id: 2f13e60cf5509b9a63fb7b8d84846daf889dc1b7
Original-Change-Id: Iedf7ca435ae337ead85115200d6987fb0d4828d7
Original-Signed-off-by: Julius Werner <jwerner@chromium.org>
Original-Reviewed-on: https://chromium-review.googlesource.com/223781
Reviewed-on: http://review.coreboot.org/9341
Tested-by: build bot (Jenkins)
Reviewed-by: Stefan Reinauer <stefan.reinauer@coreboot.org>

5 files changed, 140 insertions, 87 deletions

diff --git a/src/arch/arm/armv7/mmu.c b/src/arch/arm/armv7/mmu.c
index e9c681530cb2..de7fa40f7e08 100644
--- a/src/arch/arm/armv7/mmu.c
+++ b/src/arch/arm/armv7/mmu.c
@@ -42,7 +42,7 @@
 #if CONFIG_ARM_LPAE
 /* See B3.6.2 of ARMv7 Architecture Reference Manual */
 /* TODO: Utilize the contiguous hint flag */
-#define ATTR_BASE (\
+#define ATTR_BLOCK (\
 	0ULL << 54 |	/* XN. 0:Not restricted */ \
 	0ULL << 53 |	/* PXN. 0:Not restricted */ \
 	1 << 10 |	/* AF. 1:Accessed. This is to prevent access \
@@ -52,16 +52,18 @@
 	0 << 1 | 	/* block/table. 0:block entry */ \
 	1 << 0		/* validity. 1:valid */ \
 	)
-#define ATTR_NC		(ATTR_BASE | (MAIR_INDX_NC << 2) | \
-			(1ULL << 53) | (1ULL << 54))
-#define ATTR_WT		(ATTR_BASE | (MAIR_INDX_WT << 2))
-#define ATTR_WB		(ATTR_BASE | (MAIR_INDX_WB << 2))
+#define ATTR_PAGE	(ATTR_BLOCK | 1 << 1)
+#define ATTR_NEXTLEVEL	(0x3)
+#define ATTR_NC		((MAIR_INDX_NC << 2) | (1ULL << 53) | (1ULL << 54))
+#define ATTR_WT		(MAIR_INDX_WT << 2)
+#define ATTR_WB		(MAIR_INDX_WB << 2)
 
+#define PAGE_MASK	0x000ffffffffff000ULL
+#define BLOCK_MASK	0x000fffffffe00000ULL
+#define NEXTLEVEL_MASK	PAGE_MASK
 #define BLOCK_SHIFT	21
 
-typedef uint64_t pgd_t;
-typedef uint64_t pmd_t;
-static const unsigned int denom = 2;
+typedef uint64_t pte_t;
 #else	/* CONFIG_ARM_LPAE */
 	/*
 	 * Section entry bits:
@@ -79,27 +81,28 @@ static const unsigned int denom = 2;
 	 *     2 - B, 1 for bufferable
 	 *  1: 0 - 0b10 to indicate section entry
 	 */
-#define ATTR_BASE	((3 << 10) | 0x2)
-#define ATTR_NC		(ATTR_BASE | (1 << 4))
-#define ATTR_WT		(ATTR_BASE | (1 << 3))
-#define ATTR_WB		(ATTR_BASE | (1 << 3) | (1 << 2))
+#define ATTR_BLOCK	((3 << 10) | 0x2)
+#define ATTR_PAGE	((3 << 4) | 0x2)
+#define ATTR_NEXTLEVEL	(0x1)
+#define ATTR_NC		(1 << 4)
+#define ATTR_WT		(1 << 3)
+#define ATTR_WB		((1 << 3) | (1 << 2))
 
+#define PAGE_MASK	0xfffff000UL
+#define BLOCK_MASK	0xfff00000UL
+#define NEXTLEVEL_MASK	0xfffffc00UL
 #define BLOCK_SHIFT	20
 
-typedef uint32_t pgd_t;
-typedef uint32_t pmd_t;
-static const unsigned int denom = 1;
+typedef uint32_t pte_t;
 #endif	/* CONFIG_ARM_LPAE */
 
-static pmd_t *const ttb_buff = (pmd_t *)_ttb;
-
 /*
  * mask/shift/size for pages and blocks
  */
 #define PAGE_SHIFT	12
 #define PAGE_SIZE	(1UL << PAGE_SHIFT)
-#define PAGE_MASK	~((1UL << PAGE_SHIFT) - 1)
 #define BLOCK_SIZE	(1UL << BLOCK_SHIFT)
+#define SUBTABLE_SIZE	((1 << (BLOCK_SHIFT - PAGE_SHIFT)) * sizeof(pte_t))
 
 /*
  * MAIR Index
@@ -108,69 +111,124 @@ static pmd_t *const ttb_buff = (pmd_t *)_ttb;
 #define MAIR_INDX_WT	1
 #define MAIR_INDX_WB	2
 
-static void mmu_flush_page_table_entry_range(
-		unsigned long start_mb, unsigned long size_mb)
+static pte_t *const ttb_buff = (void *)_ttb;
+static int used_tables = 0;
+
+/* Not all boards want to use subtables and declare them in memlayout.ld. This
+ * outputs two 0x00000000 symbols if they don't, making _ttb_subtables_size 0.
+ * (I would like to explicitly assign them to 0 here, but that triggers
+ * https://sourceware.org/bugzilla/show_bug.cgi?id=1038 in GNU as.) */
+asm (".weak _ttb_subtables, _ettb_subtables");
+
+static struct {
+	pte_t value;
+	const char *name;
+} attrs[] = {
+	[DCACHE_OFF] = {.value = ATTR_NC, .name = "uncached"},
+	[DCACHE_WRITEBACK] = {.value = ATTR_WB, .name = "writeback"},
+	[DCACHE_WRITETHROUGH] = {.value = ATTR_WT, .name = "writethrough"},
+};
+
+/* Fills page table entries in |table| from |start_idx| to |end_idx| with |attr|
+ * and performs necessary invalidations. |offset| is the start address of the
+ * area described by |table|, and |shift| is the size-shift of each frame. */
+static void mmu_fill_table(pte_t *table, u32 start_idx, u32 end_idx,
+			   uintptr_t offset, u32 shift, pte_t attr)
 {
 	int i;
 
+	/* Write out page table entries. */
+	for (i = start_idx; i < end_idx; i++)
+		table[i] = (offset + (i << shift)) | attr;
+
 	/* Flush the page table entries from the dcache. */
-	for (i = start_mb/denom; i*denom < start_mb + size_mb; i++)
-		dccmvac((uintptr_t)&ttb_buff[i]);
+	for (i = start_idx; i < end_idx; i++)
+		dccmvac((uintptr_t)&table[i]);
 	dsb();
+
 	/* Invalidate the TLB entries. */
-	for (i = start_mb/denom; i*denom < start_mb + size_mb; i++)
-		tlbimvaa(i*denom*MiB);
+	for (i = start_idx; i < end_idx; i++)
+		tlbimvaa(offset + (i << shift));
 	dsb();
 	isb();
 }
 
-void mmu_disable_range(unsigned long start_mb, unsigned long size_mb)
+static pte_t *mmu_create_subtable(pte_t *pgd_entry)
 {
-	int i;
+	if (used_tables >= _ttb_subtables_size / SUBTABLE_SIZE)
+		die("Not enough room for another sub-pagetable!");
 
-	printk(BIOS_DEBUG, "Disabling: [0x%08lx:0x%08lx)\n",
-			start_mb*MiB, start_mb*MiB + size_mb*MiB);
+	/* We assume that *pgd_entry must already be a valid block mapping. */
+	uintptr_t start_addr = (uintptr_t)(*pgd_entry & BLOCK_MASK);
+	pte_t *table = (void *)(_ttb_subtables + used_tables++ * SUBTABLE_SIZE);
+	printk(BIOS_DEBUG, "Creating new subtable @%p for [%#.8x:%#.8lx)\n",
+	       table, start_addr, start_addr + BLOCK_SIZE);
 
-	for (i = start_mb/denom; i*denom < start_mb + size_mb; i++)
-		ttb_buff[i] = 0;
+	/* Initialize the new subtable with entries of the same attributes
+	 * (XN bit moves from 4 to 0, set PAGE unless block was unmapped). */
+	pte_t attr = *pgd_entry & ~(BLOCK_MASK);
+	if (!IS_ENABLED(CONFIG_ARM_LPAE) && (attr & (1 << 4)))
+		attr = ((attr & ~(1 << 4)) | (1 << 0));
+	if (attr & ATTR_BLOCK)
+		attr = (attr & ~ATTR_BLOCK) | ATTR_PAGE;
+	mmu_fill_table(table, 0, SUBTABLE_SIZE / sizeof(pte_t),
+		       start_addr, PAGE_SHIFT, attr);
 
-	mmu_flush_page_table_entry_range(start_mb, size_mb);
+	/* Replace old entry in upper level table to point at subtable. */
+	*pgd_entry = (pte_t)(uintptr_t)table | ATTR_NEXTLEVEL;
+	dccmvac((uintptr_t)pgd_entry);
+	dsb();
+	tlbimvaa(start_addr);
+	dsb();
+	isb();
+
+	return table;
 }
 
-void mmu_config_range(unsigned long start_mb, unsigned long size_mb,
-		enum dcache_policy policy)
+void mmu_config_range_kb(u32 start_kb, u32 size_kb, enum dcache_policy policy)
 {
-	const char *str = NULL;
-	pmd_t attr;
-	int i;
+	pte_t *pgd_entry = &ttb_buff[start_kb / (BLOCK_SIZE/KiB)];
+	pte_t *table = (void *)(uintptr_t)(*pgd_entry & NEXTLEVEL_MASK);
 
-	switch(policy) {
-	case DCACHE_OFF:
-		/* XN set to avoid prefetches to uncached/unbuffered regions */
-		attr = ATTR_NC;
-		str = "off";
-		break;
-	case DCACHE_WRITEBACK:
-		attr = ATTR_WB;
-		str = "writeback";
-		break;
-	case DCACHE_WRITETHROUGH:
-		attr = ATTR_WT;
-		str = "writethrough";
-		break;
-	default:
-		printk(BIOS_ERR, "unknown dcache policy: %02x\n", policy);
-		return;
-	}
-
-	printk(BIOS_DEBUG, "Setting dcache policy: [0x%08lx:0x%08lx) [%s]\n",
-			start_mb << 20, ((start_mb + size_mb) << 20), str);
+	/* Make sure the range is contained within a single superpage. */
+	assert(((start_kb + size_kb - 1) & (BLOCK_MASK/KiB))
+	       == (start_kb & (BLOCK_MASK/KiB)) && start_kb < 4 * (GiB/KiB));
 
-	/* Write out page table entries. */
-	for (i = start_mb/denom; i*denom < start_mb + size_mb; i++)
-		ttb_buff[i] = ((pmd_t)i << BLOCK_SHIFT) | attr;
+	if ((*pgd_entry & ~NEXTLEVEL_MASK) != ATTR_NEXTLEVEL)
+		table = mmu_create_subtable(pgd_entry);
+
+	/* Always _one_ _damn_ bit that won't fit... (XN moves from 4 to 0) */
+	pte_t attr = attrs[policy].value;
+	if (!IS_ENABLED(CONFIG_ARM_LPAE) && (attr & (1 << 4)))
+		attr = ((attr & ~(1 << 4)) | (1 << 0));
+
+	/* Mask away high address bits that are handled by upper level table. */
+	u32 mask = BLOCK_SIZE/KiB - 1;
+	printk(BIOS_DEBUG, "Mapping address range [%#.8x:%#.8x) as %s\n",
+	       start_kb * KiB, (start_kb + size_kb) * KiB, attrs[policy].name);
+	mmu_fill_table(table, (start_kb & mask) / (PAGE_SIZE/KiB),
+		       div_round_up((start_kb + size_kb) & mask, PAGE_SIZE/KiB),
+		       (start_kb & ~mask) * KiB, PAGE_SHIFT, ATTR_PAGE | attr);
+}
 
-	mmu_flush_page_table_entry_range(start_mb, size_mb);
+void mmu_disable_range(u32 start_mb, u32 size_mb)
+{
+	printk(BIOS_DEBUG, "Setting address range [%#.8x:%#.8x) as unmapped\n",
+	       start_mb * MiB, (start_mb + size_mb) * MiB);
+	assert(start_mb + size_mb <= 4 * (GiB/MiB));
+	mmu_fill_table(ttb_buff, start_mb / (BLOCK_SIZE/MiB),
+		       div_round_up(start_mb + size_mb, BLOCK_SIZE/MiB),
+		       0, BLOCK_SHIFT, 0);
+}
+
+void mmu_config_range(u32 start_mb, u32 size_mb, enum dcache_policy policy)
+{
+	printk(BIOS_DEBUG, "Mapping address range [%#.8x:%#.8x) as %s\n",
+	       start_mb * MiB, (start_mb + size_mb) * MiB, attrs[policy].name);
+	assert(start_mb + size_mb <= 4 * (GiB/MiB));
+	mmu_fill_table(ttb_buff, start_mb / (BLOCK_SIZE/MiB),
+		       div_round_up(start_mb + size_mb, BLOCK_SIZE/MiB),
+		       0, BLOCK_SHIFT, ATTR_BLOCK | attrs[policy].value);
 }
 
 /*
@@ -187,8 +245,8 @@ void mmu_config_range(unsigned long start_mb, unsigned long size_mb,
 void mmu_init(void)
 {
         if (CONFIG_ARM_LPAE) {
-                pgd_t *const pgd_buff = (pgd_t*)(_ttb + 16*KiB);
-                pmd_t *pmd = ttb_buff;
+                pte_t *const pgd_buff = (pte_t*)(_ttb + 16*KiB);
+                pte_t *pmd = ttb_buff;
                 int i;
 
                 printk(BIOS_DEBUG, "LPAE Translation tables are @ %p\n",
@@ -214,8 +272,8 @@ void mmu_init(void)
                  * See B3.6.1 of ARMv7 Architecture Reference Manual
                  */
                 for (i = 0; i < 4; i++) {
-                        pgd_buff[i] = ((uint32_t)pmd & PAGE_MASK) |
-                                3;	/* 0b11: valid table entry */
+                        pgd_buff[i] = ((uint32_t)pmd & NEXTLEVEL_MASK) |
+                                ATTR_NEXTLEVEL;
                         pmd += BLOCK_SIZE / PAGE_SIZE;
                 }
 
diff --git a/src/arch/arm/include/arch/memlayout.h b/src/arch/arm/include/arch/memlayout.h
index 9f5b3831603b..2c733946a20e 100644
--- a/src/arch/arm/include/arch/memlayout.h
+++ b/src/arch/arm/include/arch/memlayout.h
@@ -27,6 +27,11 @@
 	_ = ASSERT(size >= 16K + IS_ENABLED(CONFIG_ARM_LPAE) * 32, \
 		"TTB must be 16K (+ 32 for LPAE)!");
 
+#define TTB_SUBTABLES(addr, size) \
+	REGION(ttb_subtables, addr, size, IS_ENABLED(CONFIG_ARM_LPAE)*3K + 1K) \
+	_ = ASSERT(size % (1K + 3K * IS_ENABLED(CONFIG_ARM_LPAE)) == 0, \
+		"TTB subtable region must be evenly divisible by table size!");
+
 /* ARM stacks need 8-byte alignment and stay in one place through ramstage. */
 #define STACK(addr, size) REGION(stack, addr, size, 8)
 
diff --git a/src/arch/arm/include/armv4/arch/cache.h b/src/arch/arm/include/armv4/arch/cache.h
index e41ff9a49f33..65f59662a678 100644
--- a/src/arch/arm/include/armv4/arch/cache.h
+++ b/src/arch/arm/include/armv4/arch/cache.h
@@ -72,23 +72,4 @@ void cache_sync_instructions(void);
 /* tlb invalidate all */
 void tlb_invalidate_all(void);
 
-/*
- * Generalized setup/init functions
- */
-
-/* mmu initialization (set page table address, set permissions, etc) */
-void mmu_init(void);
-
-enum dcache_policy {
-	DCACHE_OFF,
-	DCACHE_WRITEBACK,
-	DCACHE_WRITETHROUGH,
-};
-
-/* disable the mmu for a range. Primarily useful to lock out address 0. */
-void mmu_disable_range(unsigned long start_mb, unsigned long size_mb);
-/* mmu range configuration (set dcache policy) */
-void mmu_config_range(unsigned long start_mb, unsigned long size_mb,
-						enum dcache_policy policy);
-
 #endif /* ARM_CACHE_H */
diff --git a/src/arch/arm/include/armv7/arch/cache.h b/src/arch/arm/include/armv7/arch/cache.h
index dde2c08c1d25..f23dec1bbbf0 100644
--- a/src/arch/arm/include/armv7/arch/cache.h
+++ b/src/arch/arm/include/armv7/arch/cache.h
@@ -345,9 +345,14 @@ enum dcache_policy {
 };
 
 /* disable the mmu for a range. Primarily useful to lock out address 0. */
-void mmu_disable_range(unsigned long start_mb, unsigned long size_mb);
+void mmu_disable_range(u32 start_mb, u32 size_mb);
 /* mmu range configuration (set dcache policy) */
-void mmu_config_range(unsigned long start_mb, unsigned long size_mb,
-						enum dcache_policy policy);
+void mmu_config_range(u32 start_mb, u32 size_mb, enum dcache_policy policy);
+
+/* Reconfigure memory mappings at the fine-grained (4K) page level. Must be
+ * called on a range contained within a single, already mapped block/superpage.
+ * Careful: Do NOT map over this address range with mmu_config_range() again
+ * later, or you will leak resources and may desync your TLB! */
+void mmu_config_range_kb(u32 start_kb, u32 size_kb, enum dcache_policy policy);
 
 #endif /* ARM_CACHE_H */
diff --git a/src/include/symbols.h b/src/include/symbols.h
index c343fd9a1418..2259dcb5b628 100644
--- a/src/include/symbols.h
+++ b/src/include/symbols.h
@@ -69,6 +69,10 @@ extern u8 _ttb[];
 extern u8 _ettb[];
 #define _ttb_size (_ettb - _ttb)
 
+extern u8 _ttb_subtables[];
+extern u8 _ettb_subtables[];
+#define _ttb_subtables_size (_ettb_subtables - _ttb_subtables)
+
 extern u8 _dma_coherent[];
 extern u8 _edma_coherent[];
 #define _dma_coherent_size (_edma_coherent - _dma_coherent)