Merge "fix(gpt): use DC CIGDPAPA when MTE2 is implemented" into integration

include/arch/aarch64/arch_helpers.h

@@ -241,6 +241,7 @@ DEFINE_SYSOP_PARAM_FUNC(xpaci)
 
 void flush_dcache_range(uintptr_t addr, size_t size);
 void flush_dcache_to_popa_range(uintptr_t addr, size_t size);
+void flush_dcache_to_popa_range_mte2(uintptr_t addr, size_t size);
 void clean_dcache_range(uintptr_t addr, size_t size);
 void inv_dcache_range(uintptr_t addr, size_t size);
 bool is_dcache_enabled(void);

lib/aarch64/cache_helpers.S

@@ -9,6 +9,7 @@
 
 	.globl	flush_dcache_range
 	.globl	flush_dcache_to_popa_range
+	.globl	flush_dcache_to_popa_range_mte2
 	.globl	clean_dcache_range
 	.globl	inv_dcache_range
 	.globl	dcsw_op_louis
@@ -17,6 +18,20 @@
 	.globl	dcsw_op_level2
 	.globl	dcsw_op_level3
 
+/* Opcodes for data cache maintenance by PA instructions. */
+
+/*
+ * sys  #6, c7, c14, #1, x0
+ * DC CIPAPA, X0
+ */
+#define dc_cipapa_x0	0xd50e7e20
+
+/*
+ * sys #6, c7, c14, #3, x0
+ * DC CIDGPAPA, X0
+  */
+#define dc_cigdpapa_x0	0xd50e7ea0
+
 /*
  * This macro can be used for implementing various data cache operations `op`
  */
@@ -37,6 +52,24 @@ exit_loop_\op:
 	ret
 .endm
 
+/* op: the hexadecimal instruction opcode for the cache operation */
+.macro do_dcache_maintenance_instr op
+	/* Exit early if size is zero */
+	cbz	x1, exit_loop_\op
+	dcache_line_size x2, x3
+	sub	x3, x2, #1
+	bic	x0, x0, x3
+	add	x1, x1, x0
+loop_\op:
+	.inst	\op
+	add	x0, x0, x2
+	cmp	x0, x1
+	b.lo	loop_\op
+	dsb	osh
+exit_loop_\op:
+	ret
+.endm
+
 .macro check_plat_can_cmo
 #if CONDITIONAL_CMO
 	mov	x3, x30
@@ -49,10 +82,11 @@ exit_loop_\op:
 	mov	 x0, x2
 #endif
 .endm
-	/* ------------------------------------------
+
-	 * Clean+Invalidate from base address till
+	/* -------------------------------------------
-	 * size. 'x0' = addr, 'x1' = size
+	 * DCache Clean+Invalidate by MVA from base
-	 * ------------------------------------------
+	 * address till size. 'x0' = addr, 'x1' = size
+	 * -------------------------------------------
 	 */
 func flush_dcache_range
 	check_plat_can_cmo
@@ -60,8 +94,8 @@ func flush_dcache_range
 endfunc flush_dcache_range
 
 	/* ------------------------------------------
-	 * Clean from base address till size.
+	 * DCache Clean by MVA from base address till
-	 * 'x0' = addr, 'x1' = size
+	 * size. 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 */
 func clean_dcache_range
@@ -70,8 +104,8 @@ func clean_dcache_range
 endfunc clean_dcache_range
 
 	/* ------------------------------------------
-	 * Invalidate from base address till
+	 * DCache Invalidate by MVA from base address
-	 * size. 'x0' = addr, 'x1' = size
+	 * till size. 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 */
 func inv_dcache_range
@@ -79,37 +113,36 @@ func inv_dcache_range
 	do_dcache_maintenance_by_mva ivac
 endfunc inv_dcache_range
 
-
 	/*
-	 * On implementations with FEAT_MTE2,
-	 * Root firmware must issue DC_CIGDPAPA instead of DC_CIPAPA ,
-	 * in order to additionally clean and invalidate Allocation Tags
-	 * associated with the affected locations.
-	 *
 	 * ------------------------------------------
-	 * Clean+Invalidate by PA to POPA
+	 * DCache Clean+Invalidate by PA to POPA from
-	 * from base address till size.
+	 * base address till size.
 	 * 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 */
 func flush_dcache_to_popa_range
-	/* Exit early if size is zero */
-	cbz	x1, exit_loop_dc_cipapa
 	check_plat_can_cmo
-	dcache_line_size x2, x3
+	/* dc cipapa, x0 */
-	sub	x3, x2, #1
+	do_dcache_maintenance_instr dc_cipapa_x0
-	bic	x0, x0, x3
-	add	x1, x1, x0
-loop_dc_cipapa:
-	sys	#6, c7, c14, #1, x0 /* DC CIPAPA,<Xt> */
-	add	x0, x0, x2
-	cmp	x0, x1
-	b.lo	loop_dc_cipapa
-	dsb	osh
-exit_loop_dc_cipapa:
-	ret
 endfunc	flush_dcache_to_popa_range
 
+	/*
+	 * ------------------------------------------
+	 * Clean+Invalidate by PA to POPA (MTE2)
+	 * from base address till size.
+	 * 'x0' = addr, 'x1' = size
+	 * ------------------------------------------
+	 * On implementations with FEAT_MTE2, Root firmware must issue
+	 * DC_CIGDPAPA instead of DC_CIPAPA, in order to additionally
+	 * clean and invalidate Allocation Tags associated with the
+	 * affected locations.
+	 */
+func flush_dcache_to_popa_range_mte2
+	check_plat_can_cmo
+	/* dc cigdpapa, x0 */
+	do_dcache_maintenance_instr dc_cigdpapa_x0
+endfunc	flush_dcache_to_popa_range_mte2
+
 	/* ---------------------------------------------------------------
 	 * Data cache operations by set/way to the level specified
 	 *

lib/gpt_rme/gpt_rme.c

@@ -11,6 +11,7 @@
 #include <stdint.h>
 
 #include <arch.h>
+#include <arch_features.h>
 #include <arch_helpers.h>
 #include <common/debug.h>
 #include "gpt_rme_private.h"
@@ -1095,8 +1096,13 @@ int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
 	 * states, remove any data speculatively fetched into the target
 	 * physical address space. Issue DC CIPAPA over address range
 	 */
-	flush_dcache_to_popa_range(nse | base,
+	if (is_feat_mte2_supported()) {
-				   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+		flush_dcache_to_popa_range_mte2(nse | base,
+					GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	} else {
+		flush_dcache_to_popa_range(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	}
 
 	write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
 		  gpi_info.gpi_shift, gpi_info.idx, target_pas);
@@ -1107,8 +1113,13 @@ int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
 
 	nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT;
 
-	flush_dcache_to_popa_range(nse | base,
+	if (is_feat_mte2_supported()) {
-				   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+		flush_dcache_to_popa_range_mte2(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	} else {
+		flush_dcache_to_popa_range(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	}
 
 	/* Unlock access to the L1 tables. */
 	spin_unlock(&gpt_lock);
@@ -1225,8 +1236,13 @@ int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
 	}
 
 	/* Ensure that the scrubbed data has made it past the PoPA */
-	flush_dcache_to_popa_range(nse | base,
+	if (is_feat_mte2_supported()) {
-				   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+		flush_dcache_to_popa_range_mte2(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	} else {
+		flush_dcache_to_popa_range(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	}
 
 	/*
 	 * Remove any data loaded speculatively
@@ -1234,8 +1250,13 @@ int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
 	 */
 	nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT;
 
-	flush_dcache_to_popa_range(nse | base,
+	if (is_feat_mte2_supported()) {
-				   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+		flush_dcache_to_popa_range_mte2(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	} else {
+		flush_dcache_to_popa_range(nse | base,
+					   GPT_PGS_ACTUAL_SIZE(gpt_config.p));
+	}
 
 	/* Clear existing GPI encoding and transition granule. */
 	write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,