Neoverse N1 Errata Workaround 1542419

Coherent I-cache is causing a prefetch violation where when the core
executes an instruction that has recently been modified, the core might
fetch a stale instruction which violates the ordering of instruction
fetches.

The workaround includes an instruction sequence to implementation
defined registers to trap all EL0 IC IVAU instructions to EL3 and a trap
handler to execute a TLB inner-shareable invalidation to an arbitrary
address followed by a DSB.

Signed-off-by: Lauren Wehrmeister <lauren.wehrmeister@arm.com>
Change-Id: Ic3b7cbb11cf2eaf9005523ef5578a372593ae4d6

bl31/aarch64/ea_delegate.S

@@ -11,7 +11,8 @@
 #include <bl31/ea_handle.h>
 #include <context.h>
 #include <lib/extensions/ras_arch.h>
-
+#include <cpu_macros.S>
+#include <context.h>
 
 	.globl	handle_lower_el_ea_esb
 	.globl	enter_lower_el_sync_ea
@@ -35,9 +36,9 @@ endfunc handle_lower_el_ea_esb
 
 /*
  * This function forms the tail end of Synchronous Exception entry from lower
- * EL, and expects to handle only Synchronous External Aborts from lower EL. If
- * any other kind of exception is detected, then this function reports unhandled
- * exception.
+ * EL, and expects to handle Synchronous External Aborts from lower EL and CPU
+ * Implementation Defined Exceptions. If any other kind of exception is detected,
+ * then this function reports unhandled exception.
  *
  * Since it's part of exception vector, this function doesn't expect any GP
  * registers to have been saved. It delegates the handling of the EA to platform
@@ -58,12 +59,33 @@ func enter_lower_el_sync_ea
 	b.eq	1f
 
 	cmp	x30, #EC_DABORT_LOWER_EL
-	b.ne	2f
+	b.eq	1f
+
+	/* Save GP registers */
+	stp	x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
+	stp	x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
+	stp	x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
+
+	/* Get the cpu_ops pointer */
+	bl	get_cpu_ops_ptr
+
+	/* Get the cpu_ops exception handler */
+	ldr	x0, [x0, #CPU_E_HANDLER_FUNC]
+
+	/*
+	 * If the reserved function pointer is NULL, this CPU does not have an
+	 * implementation defined exception handler function
+	 */
+	cbz	x0, 2f
+	mrs	x1, esr_el3
+	ubfx	x1, x1, #ESR_EC_SHIFT, #ESR_EC_LENGTH
+	blr	x0
+	b	2f
 
 1:
 	/* Test for EA bit in the instruction syndrome */
 	mrs	x30, esr_el3
-	tbz	x30, #ESR_ISS_EABORT_EA_BIT, 2f
+	tbz	x30, #ESR_ISS_EABORT_EA_BIT, 3f
 
 	/*
 	 * Save general purpose and ARMv8.3-PAuth registers (if enabled).
@@ -84,6 +106,11 @@ func enter_lower_el_sync_ea
 	b	delegate_sync_ea
 
 2:
+	ldp	x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
+	ldp	x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
+	ldp	x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
+
+3:
 	/* Synchronous exceptions other than the above are assumed to be EA */
 	ldr	x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
 	no_ret	report_unhandled_exception

docs/design/cpu-specific-build-macros.rst

@@ -258,6 +258,9 @@ For Neoverse N1, the following errata build flags are defined :
 -  ``ERRATA_N1_1315703``: This applies errata 1315703 workaround to Neoverse-N1
    CPU. This needs to be enabled only for revision <= r3p0 of the CPU.
 
+-  ``ERRATA_N1_1542419``: This applies errata 1542419 workaround to Neoverse-N1
+   CPU. This needs to be enabled only for revisions r3p0 - r4p0 of the CPU.
+
 DSU Errata Workarounds
 ----------------------
 

include/lib/cpus/aarch64/cpu_macros.S

@@ -43,6 +43,7 @@
 	.equ	CPU_MIDR_SIZE, CPU_WORD_SIZE
 	.equ	CPU_EXTRA1_FUNC_SIZE, CPU_WORD_SIZE
 	.equ	CPU_EXTRA2_FUNC_SIZE, CPU_WORD_SIZE
+	.equ	CPU_E_HANDLER_FUNC_SIZE, CPU_WORD_SIZE
 	.equ	CPU_RESET_FUNC_SIZE, CPU_WORD_SIZE
 	.equ	CPU_PWR_DWN_OPS_SIZE, CPU_WORD_SIZE * CPU_MAX_PWR_DWN_OPS
 	.equ	CPU_ERRATA_FUNC_SIZE, CPU_WORD_SIZE
@@ -83,7 +84,8 @@
 	.equ	CPU_RESET_FUNC, CPU_MIDR + CPU_MIDR_SIZE
 	.equ	CPU_EXTRA1_FUNC, CPU_RESET_FUNC + CPU_RESET_FUNC_SIZE
 	.equ	CPU_EXTRA2_FUNC, CPU_EXTRA1_FUNC + CPU_EXTRA1_FUNC_SIZE
-	.equ	CPU_PWR_DWN_OPS, CPU_EXTRA2_FUNC + CPU_EXTRA2_FUNC_SIZE
+	.equ	CPU_E_HANDLER_FUNC, CPU_EXTRA2_FUNC + CPU_EXTRA2_FUNC_SIZE
+	.equ	CPU_PWR_DWN_OPS, CPU_E_HANDLER_FUNC + CPU_E_HANDLER_FUNC_SIZE
 	.equ	CPU_ERRATA_FUNC, CPU_PWR_DWN_OPS + CPU_PWR_DWN_OPS_SIZE
 	.equ	CPU_ERRATA_LOCK, CPU_ERRATA_FUNC + CPU_ERRATA_FUNC_SIZE
 	.equ	CPU_ERRATA_PRINTED, CPU_ERRATA_LOCK + CPU_ERRATA_LOCK_SIZE
@@ -139,6 +141,8 @@
 	 *	This is a placeholder for future per CPU operations.  Currently
 	 *	some CPUs use this entry to set a function to disable the
 	 *	workaround for CVE-2018-3639.
+	 * _e_handler:
+	 *	This is a placeholder for future per CPU exception handlers.
 	 * _power_down_ops:
 	 *	Comma-separated list of functions to perform power-down
 	 *	operatios on the CPU. At least one, and up to
@@ -149,7 +153,7 @@
 	 *	used to handle power down at subsequent levels
 	 */
 	.macro declare_cpu_ops_base _name:req, _midr:req, _resetfunc:req, \
-		_extra1:req, _extra2:req, _power_down_ops:vararg
+		_extra1:req, _extra2:req, _e_handler:req, _power_down_ops:vararg
 	.section cpu_ops, "a"
 	.align 3
 	.type cpu_ops_\_name, %object
@@ -159,6 +163,7 @@
 #endif
 	.quad \_extra1
 	.quad \_extra2
+	.quad \_e_handler
 #ifdef IMAGE_BL31
 	/* Insert list of functions */
 	fill_constants CPU_MAX_PWR_DWN_OPS, \_power_down_ops
@@ -203,15 +208,21 @@
 
 	.macro declare_cpu_ops _name:req, _midr:req, _resetfunc:req, \
 		_power_down_ops:vararg
-		declare_cpu_ops_base \_name, \_midr, \_resetfunc, 0, 0, \
+		declare_cpu_ops_base \_name, \_midr, \_resetfunc, 0, 0, 0, \
 			\_power_down_ops
 	.endm
 
+	.macro declare_cpu_ops_eh _name:req, _midr:req, _resetfunc:req, \
+		_e_handler:req, _power_down_ops:vararg
+		declare_cpu_ops_base \_name, \_midr, \_resetfunc, \
+			0, 0, \_e_handler, \_power_down_ops
+	.endm
+
 	.macro declare_cpu_ops_wa _name:req, _midr:req, \
 		_resetfunc:req, _extra1:req, _extra2:req, \
 		_power_down_ops:vararg
 		declare_cpu_ops_base \_name, \_midr, \_resetfunc, \
-			\_extra1, \_extra2, \_power_down_ops
+			\_extra1, \_extra2, 0, \_power_down_ops
 	.endm
 
 #if REPORT_ERRATA

include/lib/cpus/aarch64/neoverse_n1.h

@@ -12,6 +12,9 @@
 /* Neoverse N1 MIDR for revision 0 */
 #define NEOVERSE_N1_MIDR		U(0x410fd0c0)
 
+/* Exception Syndrome register EC code for IC Trap */
+#define NEOVERSE_N1_EC_IC_TRAP		U(0x1f)
+
 /*******************************************************************************
  * CPU Power Control register specific definitions.
  ******************************************************************************/

lib/cpus/aarch64/cpu_helpers.S

@@ -227,6 +227,27 @@ func cpu_rev_var_hs
 	ret
 endfunc cpu_rev_var_hs
 
+/*
+ * Compare the CPU's revision-variant (x0) with a given range (x1 - x2), for errata
+ * application purposes. If the revision-variant is between or includes the given
+ * values, this indicates that errata applies; otherwise not.
+ *
+ * Shall clobber: x0-x4
+ */
+	.globl	cpu_rev_var_range
+func cpu_rev_var_range
+	mov	x3, #ERRATA_APPLIES
+	mov	x4, #ERRATA_NOT_APPLIES
+	cmp	x0, x1
+	csel	x1, x3, x4, hs
+	cbz	x1, 1f
+	cmp	x0, x2
+	csel	x1, x3, x4, ls
+1:
+	mov	x0, x1
+	ret
+endfunc cpu_rev_var_range
+
 #if REPORT_ERRATA
 /*
  * void print_errata_status(void);

lib/cpus/aarch64/neoverse_n1.S

@@ -9,6 +9,7 @@
 #include <neoverse_n1.h>
 #include <cpuamu.h>
 #include <cpu_macros.S>
+#include <context.h>
 
 /* Hardware handled coherency */
 #if HW_ASSISTED_COHERENCY == 0
@@ -20,6 +21,10 @@
 #error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0"
 #endif
 
+#if ERRATA_N1_IC_TRAP
+	.global neoverse_n1_errata_ic_trap_handler
+#endif
+
 /* --------------------------------------------------
  * Errata Workaround for Neoverse N1 Erratum 1043202.
  * This applies to revision r0p0 and r1p0 of Neoverse N1.
@@ -337,6 +342,41 @@ func check_errata_1315703
 	b	cpu_rev_var_ls
 endfunc check_errata_1315703
 
+/* --------------------------------------------------
+ * Errata Workaround for Neoverse N1 Erratum 1542419.
+ * This applies to revisions r3p0 - r4p0 of Neoverse N1
+ * Inputs:
+ * x0: variant[4:7] and revision[0:3] of current cpu.
+ * Shall clobber: x0-x17
+ * --------------------------------------------------
+ */
+func errata_n1_1542419_wa
+	/* Compare x0 against revision r3p0 and r4p0 */
+	mov	x17, x30
+	bl	check_errata_1542419
+	cbz	x0, 1f
+
+        /* Apply instruction patching sequence */
+	ldr	x0, =0x0
+	msr	CPUPSELR_EL3, x0
+	ldr	x0, =0xEE670D35
+	msr	CPUPOR_EL3, x0
+	ldr	x0, =0xFFFF0FFF
+	msr	CPUPMR_EL3, x0
+	ldr	x0, =0x08000020007D
+	msr	CPUPCR_EL3, x0
+	isb
+1:
+	ret	x17
+endfunc errata_n1_1542419_wa
+
+func check_errata_1542419
+	/* Applies to everything r3p0 - r4p0. */
+	mov	x1, #0x30
+	mov	x2, #0x40
+	b	cpu_rev_var_range
+endfunc check_errata_1542419
+
 func neoverse_n1_reset_func
 	mov	x19, x30
 
@@ -406,6 +446,11 @@ func neoverse_n1_reset_func
 	bl	errata_n1_1315703_wa
 #endif
 
+#if ERRATA_N1_1542419
+	mov	x0, x18
+	bl	errata_n1_1542419_wa
+#endif
+
 #if ENABLE_AMU
 	/* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */
 	mrs	x0, actlr_el3
@@ -471,6 +516,7 @@ func neoverse_n1_errata_report
 	report_errata ERRATA_N1_1262888, neoverse_n1, 1262888
 	report_errata ERRATA_N1_1275112, neoverse_n1, 1275112
 	report_errata ERRATA_N1_1315703, neoverse_n1, 1315703
+	report_errata ERRATA_N1_1542419, neoverse_n1, 1542419
 	report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184
 
 	ldp	x8, x30, [sp], #16
@@ -478,6 +524,42 @@ func neoverse_n1_errata_report
 endfunc neoverse_n1_errata_report
 #endif
 
+/*
+ * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB
+ * inner-shareable invalidation to an arbitrary address followed by a DSB.
+ *
+ * x1: Exception Syndrome
+ */
+func neoverse_n1_errata_ic_trap_handler
+	cmp	x1, #NEOVERSE_N1_EC_IC_TRAP
+	b.ne	1f
+	tlbi	vae3is, xzr
+	dsb	sy
+
+        # Skip the IC instruction itself
+        mrs     x3, elr_el3
+        add     x3, x3, #4
+        msr     elr_el3, x3
+
+	ldp	x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
+	ldp	x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
+	ldp	x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4]
+	ldr	x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR]
+
+#if IMAGE_BL31 && RAS_EXTENSION
+	/*
+	 * Issue Error Synchronization Barrier to synchronize SErrors before
+	 * exiting EL3. We're running with EAs unmasked, so any synchronized
+	 * errors would be taken immediately; therefore no need to inspect
+	 * DISR_EL1 register.
+	 */
+	esb
+#endif
+	eret
+1:
+	ret
+endfunc neoverse_n1_errata_ic_trap_handler
+
 	/* ---------------------------------------------
 	 * This function provides neoverse_n1 specific
 	 * register information for crash reporting.
@@ -497,6 +579,7 @@ func neoverse_n1_cpu_reg_dump
 	ret
 endfunc neoverse_n1_cpu_reg_dump
 
-declare_cpu_ops neoverse_n1, NEOVERSE_N1_MIDR, \
+declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \
 	neoverse_n1_reset_func, \
+	neoverse_n1_errata_ic_trap_handler, \
 	neoverse_n1_core_pwr_dwn

lib/cpus/cpu-ops.mk

@@ -278,6 +278,10 @@ ERRATA_N1_1275112	?=0
 # to revisions before r3p1 of the Neoverse N1 cpu.
 ERRATA_N1_1315703	?=1
 
+# Flag to apply erratum 1542419 workaround during reset. This erratum applies
+# to revisions r3p0 - r4p0 of the Neoverse N1 cpu.
+ERRATA_N1_1542419	?=0
+
 # Flag to apply DSU erratum 798953. This erratum applies to DSUs revision r0p0.
 # Applying the workaround results in higher DSU power consumption on idle.
 ERRATA_DSU_798953	?=0
@@ -507,6 +511,10 @@ $(eval $(call add_define,ERRATA_N1_1275112))
 $(eval $(call assert_boolean,ERRATA_N1_1315703))
 $(eval $(call add_define,ERRATA_N1_1315703))
 
+# Process ERRATA_N1_1542419 flag
+$(eval $(call assert_boolean,ERRATA_N1_1542419))
+$(eval $(call add_define,ERRATA_N1_1542419))
+
 # Process ERRATA_DSU_798953 flag
 $(eval $(call assert_boolean,ERRATA_DSU_798953))
 $(eval $(call add_define,ERRATA_DSU_798953))