Automatically select initial xlation lookup level

Instead of hardcoding a level 1 table as the base translation level
table, let the code decide which level is the most appropriate given
the virtual address space size.

As the table granularity is 4 KB, this allows the code to select
level 0, 1 or 2 as base level for AArch64. This way, instead of
limiting the virtual address space width to 39-31 bits, widths of
48-25 bit can be used.

For AArch32, this change allows the code to select level 1 or 2
as the base translation level table and use virtual address space
width of 32-25 bits.

Also removed some unused definitions related to translation tables.

Fixes ARM-software/tf-issues#362

Change-Id: Ie3bb5d6d1a4730a26700b09827c79f37ca3cdb65

include/lib/aarch32/arch.h

@@ -254,6 +254,8 @@
 
 #define TTBCR_T1SZ_SHIFT	16
 #define TTBCR_T1SZ_MASK		(0x7)
+#define TTBCR_TxSZ_MIN		0
+#define TTBCR_TxSZ_MAX		7
 
 #define TTBCR_SH0_NON_SHAREABLE		(0x0 << 12)
 #define TTBCR_SH0_OUTER_SHAREABLE	(0x2 << 12)

include/lib/aarch64/arch.h

@@ -243,6 +243,9 @@
 #define TCR_EL1_IPS_SHIFT	32
 #define TCR_EL3_PS_SHIFT	16
 
+#define TCR_TxSZ_MIN		16
+#define TCR_TxSZ_MAX		39
+
 /* (internal) physical address size bits in EL3/EL1 */
 #define TCR_PS_BITS_4GB		(0x0)
 #define TCR_PS_BITS_64GB	(0x1)

include/lib/xlat_tables.h

@@ -32,29 +32,12 @@
 #define __XLAT_TABLES_H__
 
 /* Miscellaneous MMU related constants */
-#define NUM_2MB_IN_GB		(1 << 9)
-#define NUM_4K_IN_2MB		(1 << 9)
-#define NUM_GB_IN_4GB		(1 << 2)
-
-#define TWO_MB_SHIFT		21
-#define ONE_GB_SHIFT		30
 #define FOUR_KB_SHIFT		12
 
-#define ONE_GB_INDEX(x)		((x) >> ONE_GB_SHIFT)
-#define TWO_MB_INDEX(x)		((x) >> TWO_MB_SHIFT)
-#define FOUR_KB_INDEX(x)	((x) >> FOUR_KB_SHIFT)
-
 #define INVALID_DESC		0x0
-#define BLOCK_DESC		0x1
+#define BLOCK_DESC		0x1 /* Table levels 0-2 */
-#define TABLE_DESC		0x3
+#define TABLE_DESC		0x3 /* Table levels 0-2 */
-
+#define PAGE_DESC		0x3 /* Table level 3 */
-#define FIRST_LEVEL_DESC_N	ONE_GB_SHIFT
-#define SECOND_LEVEL_DESC_N	TWO_MB_SHIFT
-#define THIRD_LEVEL_DESC_N	FOUR_KB_SHIFT
-
-#define LEVEL1			1
-#define LEVEL2			2
-#define LEVEL3			3
 
 #define XN			(1ull << 2)
 #define PXN			(1ull << 1)
@@ -78,6 +61,14 @@
 #define XLAT_TABLE_SIZE_SHIFT	PAGE_SIZE_SHIFT
 #define XLAT_TABLE_SIZE		(1 << XLAT_TABLE_SIZE_SHIFT)
 
+#ifdef AARCH32
+#define XLAT_TABLE_LEVEL_MIN	1
+#else
+#define XLAT_TABLE_LEVEL_MIN	0
+#endif /* AARCH32 */
+
+#define XLAT_TABLE_LEVEL_MAX	3
+
 /* Values for number of entries in each MMU translation table */
 #define XLAT_TABLE_ENTRIES_SHIFT (XLAT_TABLE_SIZE_SHIFT - XLAT_ENTRY_SIZE_SHIFT)
 #define XLAT_TABLE_ENTRIES	(1 << XLAT_TABLE_ENTRIES_SHIFT)
@@ -87,6 +78,7 @@
 #define L3_XLAT_ADDRESS_SHIFT	PAGE_SIZE_SHIFT
 #define L2_XLAT_ADDRESS_SHIFT	(L3_XLAT_ADDRESS_SHIFT + XLAT_TABLE_ENTRIES_SHIFT)
 #define L1_XLAT_ADDRESS_SHIFT	(L2_XLAT_ADDRESS_SHIFT + XLAT_TABLE_ENTRIES_SHIFT)
+#define L0_XLAT_ADDRESS_SHIFT	(L1_XLAT_ADDRESS_SHIFT + XLAT_TABLE_ENTRIES_SHIFT)
 
 /*
  * AP[1] bit is ignored by hardware and is

lib/xlat_tables/aarch32/xlat_tables.c

@@ -38,25 +38,57 @@
 #include "../xlat_tables_private.h"
 
 /*
- * The virtual address space size must be a power of two. As we start the initial
+ * Each platform can define the size of the virtual address space, which is
- * lookup at level 1, it must also be between 2 GB and 4 GB. See section
+ * defined in ADDR_SPACE_SIZE. TTBCR.TxSZ is calculated as 32 minus the width
- * G4.6.5 in the ARMv8-A Architecture Reference Manual (DDI 0487A.j) for more
+ * of said address space. The value of TTBCR.TxSZ must be in the range 0 to
- * information.
+ * 7 [1], which means that the virtual address space width must be in the range
+ * 32 to 25 bits.
+ *
+ * Here we calculate the initial lookup level from the value of ADDR_SPACE_SIZE.
+ * For a 4 KB page size, level 1 supports virtual address spaces of widths 32
+ * to 31 bits, and level 2 from 30 to 25. Wider or narrower address spaces are
+ * not supported. As a result, level 3 cannot be used as initial lookup level
+ * with 4 KB granularity [1].
+ *
+ * For example, for a 31-bit address space (i.e. ADDR_SPACE_SIZE == 1 << 31),
+ * TTBCR.TxSZ will be programmed to (32 - 31) = 1. According to Table G4-5 in
+ * the ARM ARM, the initial lookup level for such an address space is 1.
+ *
+ * See the ARMv8-A Architecture Reference Manual (DDI 0487A.j) for more
+ * information:
+ * [1] Section G4.6.5
  */
-CASSERT(ADDR_SPACE_SIZE >= (1ull << 31) && ADDR_SPACE_SIZE <= (1ull << 32) &&
-	IS_POWER_OF_TWO(ADDR_SPACE_SIZE), assert_valid_addr_space_size);
 
-#define NUM_L1_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT)
+#if ADDR_SPACE_SIZE > (1ULL << (32 - TTBCR_TxSZ_MIN))
 
-static uint64_t l1_xlation_table[NUM_L1_ENTRIES]
+# error "ADDR_SPACE_SIZE is too big."
-		__aligned(NUM_L1_ENTRIES * sizeof(uint64_t));
+
+#elif ADDR_SPACE_SIZE > (1 << L1_XLAT_ADDRESS_SHIFT)
+
+# define XLAT_TABLE_LEVEL_BASE	1
+# define NUM_BASE_LEVEL_ENTRIES	(ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT)
+
+#elif ADDR_SPACE_SIZE >= (1 << (32 - TTBCR_TxSZ_MAX))
+
+# define XLAT_TABLE_LEVEL_BASE	2
+# define NUM_BASE_LEVEL_ENTRIES	(ADDR_SPACE_SIZE >> L2_XLAT_ADDRESS_SHIFT)
+
+#else
+
+# error "ADDR_SPACE_SIZE is too small."
+
+#endif
+
+static uint64_t base_xlation_table[NUM_BASE_LEVEL_ENTRIES]
+		__aligned(NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t));
 
 void init_xlat_tables(void)
 {
 	unsigned long long max_pa;
 	uintptr_t max_va;
 	print_mmap();
-	init_xlation_table(0, l1_xlation_table, 1, &max_va, &max_pa);
+	init_xlation_table(0, base_xlation_table, XLAT_TABLE_LEVEL_BASE,
+						&max_va, &max_pa);
 	assert(max_va < ADDR_SPACE_SIZE);
 }
 
@@ -95,7 +127,7 @@ void enable_mmu_secure(unsigned int flags)
 	write_ttbcr(ttbcr);
 
 	/* Set TTBR0 bits as well */
-	ttbr0 = (uintptr_t) l1_xlation_table;
+	ttbr0 = (uintptr_t) base_xlation_table;
 	write64_ttbr0(ttbr0);
 	write64_ttbr1(0);
 

lib/xlat_tables/aarch64/xlat_tables.c

@@ -38,19 +38,55 @@
 #include "../xlat_tables_private.h"
 
 /*
- * The virtual address space size must be a power of two (as set in TCR.T0SZ).
+ * Each platform can define the size of the virtual address space, which is
- * As we start the initial lookup at level 1, it must also be between 2 GB and
+ * defined in ADDR_SPACE_SIZE. TCR.TxSZ is calculated as 64 minus the width of
- * 512 GB (with the virtual address size therefore 31 to 39 bits). See section
+ * said address space. The value of TCR.TxSZ must be in the range 16 to 39 [1],
- * D4.2.5 in the ARMv8-A Architecture Reference Manual (DDI 0487A.i) for more
+ * which means that the virtual address space width must be in the range 48 to
- * information.
+ * 25 bits.
+ *
+ * Here we calculate the initial lookup level from the value of ADDR_SPACE_SIZE.
+ * For a 4 KB page size, level 0 supports virtual address spaces of widths 48 to
+ * 40 bits, level 1 from 39 to 31, and level 2 from 30 to 25. Wider or narrower
+ * address spaces are not supported. As a result, level 3 cannot be used as
+ * initial lookup level with 4 KB granularity. [2]
+ *
+ * For example, for a 35-bit address space (i.e. ADDR_SPACE_SIZE == 1 << 35),
+ * TCR.TxSZ will be programmed to (64 - 35) = 29. According to Table D4-11 in
+ * the ARM ARM, the initial lookup level for such an address space is 1.
+ *
+ * See the ARMv8-A Architecture Reference Manual (DDI 0487A.j) for more
+ * information:
+ * [1] Page 1730: 'Input address size', 'For all translation stages'.
+ * [2] Section D4.2.5
  */
-CASSERT(ADDR_SPACE_SIZE >= (1ull << 31) && ADDR_SPACE_SIZE <= (1ull << 39) &&
-	IS_POWER_OF_TWO(ADDR_SPACE_SIZE), assert_valid_addr_space_size);
 
-#define NUM_L1_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT)
+#if ADDR_SPACE_SIZE > (1ULL << (64 - TCR_TxSZ_MIN))
 
-static uint64_t l1_xlation_table[NUM_L1_ENTRIES]
+# error "ADDR_SPACE_SIZE is too big."
-		__aligned(NUM_L1_ENTRIES * sizeof(uint64_t));
+
+#elif ADDR_SPACE_SIZE > (1ULL << L0_XLAT_ADDRESS_SHIFT)
+
+# define XLAT_TABLE_LEVEL_BASE	0
+# define NUM_BASE_LEVEL_ENTRIES	(ADDR_SPACE_SIZE >> L0_XLAT_ADDRESS_SHIFT)
+
+#elif ADDR_SPACE_SIZE > (1 << L1_XLAT_ADDRESS_SHIFT)
+
+# define XLAT_TABLE_LEVEL_BASE	1
+# define NUM_BASE_LEVEL_ENTRIES	(ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT)
+
+#elif ADDR_SPACE_SIZE >= (1 << (64 - TCR_TxSZ_MAX))
+
+# define XLAT_TABLE_LEVEL_BASE	2
+# define NUM_BASE_LEVEL_ENTRIES	(ADDR_SPACE_SIZE >> L2_XLAT_ADDRESS_SHIFT)
+
+#else
+
+# error "ADDR_SPACE_SIZE is too small."
+
+#endif
+
+static uint64_t base_xlation_table[NUM_BASE_LEVEL_ENTRIES]
+		__aligned(NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t));
 
 static unsigned long long tcr_ps_bits;
 
@@ -88,7 +124,8 @@ void init_xlat_tables(void)
 	unsigned long long max_pa;
 	uintptr_t max_va;
 	print_mmap();
-	init_xlation_table(0, l1_xlation_table, 1, &max_va, &max_pa);
+	init_xlation_table(0, base_xlation_table, XLAT_TABLE_LEVEL_BASE,
+			   &max_va, &max_pa);
 	tcr_ps_bits = calc_physical_addr_size_bits(max_pa);
 	assert(max_va < ADDR_SPACE_SIZE);
 }
@@ -124,7 +161,8 @@ void init_xlat_tables(void)
 		_tlbi_fct();						\
 									\
 		/* Set TCR bits as well. */				\
-		/* Inner & outer WBWA & shareable + T0SZ = 32 */	\
+		/* Inner & outer WBWA & shareable. */			\
+		/* Set T0SZ to (64 - width of virtual address space) */	\
 		tcr = TCR_SH_INNER_SHAREABLE | TCR_RGN_OUTER_WBA |	\
 			TCR_RGN_INNER_WBA |				\
 			(64 - __builtin_ctzl(ADDR_SPACE_SIZE));		\
@@ -132,7 +170,7 @@ void init_xlat_tables(void)
 		write_tcr_el##_el(tcr);					\
 									\
 		/* Set TTBR bits as well */				\
-		ttbr = (uint64_t) l1_xlation_table;			\
+		ttbr = (uint64_t) base_xlation_table;			\
 		write_ttbr0_el##_el(ttbr);				\
 									\
 		/* Ensure all translation table writes have drained */	\

lib/xlat_tables/xlat_tables_common.c

@@ -199,7 +199,11 @@ static uint64_t mmap_desc(unsigned attr, unsigned long long addr_pa,
 	int mem_type;
 
 	desc = addr_pa;
-	desc |= (level == 3) ? TABLE_DESC : BLOCK_DESC;
+	/*
+	 * There are different translation table descriptors for level 3 and the
+	 * rest.
+	 */
+	desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC;
 	desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0;
 	desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO);
 	desc |= LOWER_ATTRS(ACCESS_FLAG);
@@ -311,14 +315,13 @@ static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm,
 					uint64_t *table,
 					int level)
 {
-	unsigned level_size_shift = L1_XLAT_ADDRESS_SHIFT - (level - 1) *
+	assert(level >= XLAT_TABLE_LEVEL_MIN && level <= XLAT_TABLE_LEVEL_MAX);
-						XLAT_TABLE_ENTRIES_SHIFT;
-	unsigned level_size = 1 << level_size_shift;
-	u_register_t level_index_mask =
-		(u_register_t)(((u_register_t) XLAT_TABLE_ENTRIES_MASK)
-		<< level_size_shift);
 
-	assert(level > 0 && level <= 3);
+	unsigned int level_size_shift =
+		       L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT;
+	u_register_t level_size = (u_register_t)1 << level_size_shift;
+	u_register_t level_index_mask =
+		((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift;
 
 	debug_print("New xlat table:\n");
 
@@ -334,8 +337,8 @@ static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm,
 			continue;
 		}
 
-		debug_print("%s VA:%p size:0x%x ", get_level_spacer(level),
+		debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level),
-				(void *)base_va, level_size);
+			(void *)base_va, (unsigned long long)level_size);
 
 		if (mm->base_va > base_va + level_size - 1) {
 			/* Next region is after this area. Nothing to map yet */

lib/xlat_tables/xlat_tables_private.h

@@ -31,6 +31,12 @@
 #ifndef __XLAT_TABLES_PRIVATE_H__
 #define __XLAT_TABLES_PRIVATE_H__
 
+#include <cassert.h>
+#include <utils.h>
+
+/* The virtual address space size must be a power of two. */
+CASSERT(IS_POWER_OF_TWO(ADDR_SPACE_SIZE), assert_valid_addr_space_size);
+
 void print_mmap(void);
 void init_xlation_table(uintptr_t base_va, uint64_t *table,
 			int level, uintptr_t *max_va,