Merge pull request #1178 from davidcunado-arm/dc/enable_sve

Enable SVE for Non-secure world

Makefile

@@ -468,6 +468,7 @@ $(eval $(call assert_boolean,ENABLE_PMF))
 $(eval $(call assert_boolean,ENABLE_PSCI_STAT))
 $(eval $(call assert_boolean,ENABLE_RUNTIME_INSTRUMENTATION))
 $(eval $(call assert_boolean,ENABLE_SPE_FOR_LOWER_ELS))
+$(eval $(call assert_boolean,ENABLE_SVE_FOR_NS))
 $(eval $(call assert_boolean,ERROR_DEPRECATED))
 $(eval $(call assert_boolean,GENERATE_COT))
 $(eval $(call assert_boolean,GICV2_G0_FOR_EL3))
@@ -508,6 +509,7 @@ $(eval $(call add_define,ENABLE_PMF))
 $(eval $(call add_define,ENABLE_PSCI_STAT))
 $(eval $(call add_define,ENABLE_RUNTIME_INSTRUMENTATION))
 $(eval $(call add_define,ENABLE_SPE_FOR_LOWER_ELS))
+$(eval $(call add_define,ENABLE_SVE_FOR_NS))
 $(eval $(call add_define,ERROR_DEPRECATED))
 $(eval $(call add_define,GICV2_G0_FOR_EL3))
 $(eval $(call add_define,HW_ASSISTED_COHERENCY))

bl31/bl31.mk

@@ -54,6 +54,10 @@ ifeq (${ENABLE_AMU},1)
 BL31_SOURCES		+=	lib/extensions/amu/aarch64/amu.c
 endif
 
+ifeq (${ENABLE_SVE_FOR_NS},1)
+BL31_SOURCES		+=	lib/extensions/sve/sve.c
+endif
+
 BL31_LINKERFILE		:=	bl31/bl31.ld.S
 
 # Flag used to indicate if Crash reporting via console should be included

docs/user-guide.rst

@@ -354,6 +354,17 @@ Common build options
    The default is 1 but is automatically disabled when the target architecture
    is AArch32.
 
+-  ``ENABLE_SVE_FOR_NS``: Boolean option to enable Scalable Vector Extension
+   (SVE) for the Non-secure world only. SVE is an optional architectural feature
+   for AArch64. Note that when SVE is enabled for the Non-secure world, access
+   to SIMD and floating-point functionality from the Secure world is disabled.
+   This is to avoid corruption of the Non-secure world data in the Z-registers
+   which are aliased by the SIMD and FP registers. The build option is not
+   compatible with the ``CTX_INCLUDE_FPREGS`` build option, and will raise an
+   assert on platforms where SVE is implemented and ``ENABLE_SVE_FOR_NS`` set to
+   1. The default is 1 but is automatically disabled when the target
+   architecture is AArch32.
+
 -  ``ENABLE_STACK_PROTECTOR``: String option to enable the stack protection
    checks in GCC. Allowed values are "all", "strong" and "0" (default).
    "strong" is the recommended stack protection level if this feature is

include/common/aarch64/el3_common_macros.S

@@ -127,9 +127,9 @@
 	 * CPTR_EL3.TTA: Set to zero so that System register accesses to the
 	 *  trace registers do not trap to EL3.
 	 *
-	 * CPTR_EL3.TFP: Set to zero so that accesses to Advanced SIMD and
-	 *  floating-point functionality do not trap to EL3.
-	 * ---------------------------------------------------------------------
+	 * CPTR_EL3.TFP: Set to zero so that accesses to the V- or Z- registers
+	 *  by Advanced SIMD, floating-point or SVE instructions (if implemented)
+	 *  do not trap to EL3.
 	 */
 	mov_imm x0, (CPTR_EL3_RESET_VAL & ~(TCPAC_BIT | TTA_BIT | TFP_BIT))
 	msr	cptr_el3, x0

include/lib/aarch64/arch.h

@@ -114,6 +114,9 @@
 #define ID_AA64PFR0_AMU_LENGTH	U(4)
 #define ID_AA64PFR0_AMU_MASK	U(0xf)
 #define ID_AA64PFR0_ELX_MASK	U(0xf)
+#define ID_AA64PFR0_SVE_SHIFT	U(32)
+#define ID_AA64PFR0_SVE_MASK	U(0xf)
+#define ID_AA64PFR0_SVE_LENGTH	U(4)
 
 /* ID_AA64DFR0_EL1.PMS definitions (for ARMv8.2+) */
 #define ID_AA64DFR0_PMS_SHIFT	U(32)
@@ -301,6 +304,7 @@
 #define TAM_BIT			(U(1) << 30)
 #define TTA_BIT			(U(1) << 20)
 #define TFP_BIT			(U(1) << 10)
+#define CPTR_EZ_BIT		(U(1) << 8)
 #define CPTR_EL3_RESET_VAL	U(0x0)
 
 /* CPTR_EL2 definitions */
@@ -309,6 +313,7 @@
 #define CPTR_EL2_TAM_BIT	(U(1) << 30)
 #define CPTR_EL2_TTA_BIT	(U(1) << 20)
 #define CPTR_EL2_TFP_BIT	(U(1) << 10)
+#define CPTR_EL2_TZ_BIT		(U(1) << 8)
 #define CPTR_EL2_RESET_VAL	CPTR_EL2_RES1
 
 /* CPSR/SPSR definitions */
@@ -555,6 +560,18 @@
 #define PMCR_EL0_X_BIT		(U(1) << 4)
 #define PMCR_EL0_D_BIT		(U(1) << 3)
 
+/*******************************************************************************
+ * Definitions for system register interface to SVE
+ ******************************************************************************/
+#define ZCR_EL3			S3_6_C1_C2_0
+#define ZCR_EL2			S3_4_C1_C2_0
+
+/* ZCR_EL3 definitions */
+#define ZCR_EL3_LEN_MASK	U(0xf)
+
+/* ZCR_EL2 definitions */
+#define ZCR_EL2_LEN_MASK	U(0xf)
+
 /*******************************************************************************
  * Definitions of MAIR encodings for device and normal memory
  ******************************************************************************/

include/lib/aarch64/arch_helpers.h

@@ -329,6 +329,9 @@ DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenset1_el0, AMCNTENSET1_EL0)
 
 DEFINE_RENAME_SYSREG_RW_FUNCS(pmblimitr_el1, PMBLIMITR_EL1)
 
+DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el3, ZCR_EL3)
+DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el2, ZCR_EL2)
+
 #define IS_IN_EL(x) \
 	(GET_EL(read_CurrentEl()) == MODE_EL##x)
 

include/lib/extensions/sve.h

@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __SVE_H__
+#define __SVE_H__
+
+void sve_enable(int el2_unused);
+
+#endif /* __SVE_H__ */

lib/el3_runtime/aarch64/context_mgmt.c

@@ -18,6 +18,7 @@
 #include <smcc_helpers.h>
 #include <spe.h>
 #include <string.h>
+#include <sve.h>
 #include <utils.h>
 
 
@@ -225,6 +226,10 @@ static void enable_extensions_nonsecure(int el2_unused)
 #if ENABLE_AMU
 	amu_enable(el2_unused);
 #endif
+
+#if ENABLE_SVE_FOR_NS
+	sve_enable(el2_unused);
+#endif
 #endif
 }
 

lib/extensions/sve/sve.c

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <arch.h>
+#include <arch_helpers.h>
+#include <pubsub.h>
+#include <sve.h>
+
+static void *disable_sve_hook(const void *arg)
+{
+	uint64_t features;
+
+	features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT;
+	if ((features & ID_AA64PFR0_SVE_MASK) == 1) {
+		uint64_t cptr;
+
+		/*
+		 * Disable SVE, SIMD and FP access for the Secure world.
+		 * As the SIMD/FP registers are part of the SVE Z-registers, any
+		 * use of SIMD/FP functionality will corrupt the SVE registers.
+		 * Therefore it is necessary to prevent use of SIMD/FP support
+		 * in the Secure world as well as SVE functionality.
+		 */
+		cptr = read_cptr_el3();
+		cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT);
+		write_cptr_el3(cptr);
+
+		/*
+		 * No explicit ISB required here as ERET to switch to Secure
+		 * world covers it
+		 */
+	}
+	return 0;
+}
+
+static void *enable_sve_hook(const void *arg)
+{
+	uint64_t features;
+
+	features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT;
+	if ((features & ID_AA64PFR0_SVE_MASK) == 1) {
+		uint64_t cptr;
+
+		/*
+		 * Enable SVE, SIMD and FP access for the Non-secure world.
+		 */
+		cptr = read_cptr_el3();
+		cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT);
+		write_cptr_el3(cptr);
+
+		/*
+		 * No explicit ISB required here as ERET to switch to Non-secure
+		 * world covers it
+		 */
+	}
+	return 0;
+}
+
+void sve_enable(int el2_unused)
+{
+	uint64_t features;
+
+	features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT;
+	if ((features & ID_AA64PFR0_SVE_MASK) == 1) {
+		uint64_t cptr;
+#if CTX_INCLUDE_FPREGS
+		/*
+		 * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems.
+		 */
+		assert(0);
+#endif
+		/*
+		 * Update CPTR_EL3 to enable access to SVE functionality for the
+		 * Non-secure world.
+		 * NOTE - assumed that CPTR_EL3.TFP is set to allow access to
+		 * the SIMD, floating-point and SVE support.
+		 *
+		 * CPTR_EL3.EZ: Set to 1 to enable access to SVE  functionality
+		 *  in the Non-secure world.
+		 */
+		cptr = read_cptr_el3();
+		cptr |= CPTR_EZ_BIT;
+		write_cptr_el3(cptr);
+
+		/*
+		 * Need explicit ISB here to guarantee that update to ZCR_ELx
+		 * and CPTR_EL2.TZ do not result in trap to EL3.
+		 */
+		isb();
+
+		/*
+		 * Ensure lower ELs have access to full vector length.
+		 */
+		write_zcr_el3(ZCR_EL3_LEN_MASK);
+
+		if (el2_unused) {
+			/*
+			 * Update CPTR_EL2 to enable access to SVE functionality
+			 * for Non-secure world, EL2 and Non-secure EL1 and EL0.
+			 * NOTE - assumed that CPTR_EL2.TFP is set to allow
+			 * access to the SIMD, floating-point and SVE support.
+			 *
+			 * CPTR_EL2.TZ: Set to 0 to enable access to SVE support
+			 *  for EL2 and Non-secure EL1 and EL0.
+			 */
+			cptr = read_cptr_el2();
+			cptr &= ~(CPTR_EL2_TZ_BIT);
+			write_cptr_el2(cptr);
+
+			/*
+			 * Ensure lower ELs have access to full vector length.
+			 */
+			write_zcr_el2(ZCR_EL2_LEN_MASK);
+		}
+		/*
+		 * No explicit ISB required here as ERET to switch to
+		 * Non-secure world covers it.
+		 */
+	}
+}
+
+SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook);
+SUBSCRIBE_TO_EVENT(cm_entering_normal_world, enable_sve_hook);

make_helpers/defaults.mk

@@ -155,3 +155,12 @@ ifeq (${ARCH},aarch32)
 endif
 
 ENABLE_AMU			:= 0
+
+# By default, enable Scalable Vector Extension if implemented for Non-secure
+# lower ELs
+# Note SVE is only supported on AArch64 - therefore do not enable in AArch32
+ifneq (${ARCH},aarch32)
+    ENABLE_SVE_FOR_NS		:= 1
+else
+    override ENABLE_SVE_FOR_NS	:= 0
+endif

plat/arm/board/juno/platform.mk

@@ -99,6 +99,9 @@ ENABLE_PLAT_COMPAT		:= 	0
 # Enable memory map related constants optimisation
 ARM_BOARD_OPTIMISE_MEM		:=	1
 
+# Do not enable SVE
+ENABLE_SVE_FOR_NS		:=	0
+
 include plat/arm/board/common/board_css.mk
 include plat/arm/common/arm_common.mk
 include plat/arm/soc/common/soc_css.mk

plat/compat/plat_compat.mk

@@ -18,3 +18,6 @@ PLAT_BL_COMMON_SOURCES	+=	plat/compat/aarch64/plat_helpers_compat.S
 BL31_SOURCES		+=	plat/common/plat_psci_common.c		\
 				plat/compat/plat_pm_compat.c		\
 				plat/compat/plat_topology_compat.c
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0

plat/hisilicon/hikey/platform.mk

@@ -24,6 +24,7 @@ PLAT_PARTITION_MAX_ENTRIES	:=	12
 PLAT_PL061_MAX_GPIOS		:=	160
 COLD_BOOT_SINGLE_CPU		:=	1
 PROGRAMMABLE_RESET_ADDRESS	:=	1
+ENABLE_SVE_FOR_NS		:=	0
 
 # Process flags
 $(eval $(call add_define,HIKEY_TSP_RAM_LOCATION_ID))

plat/hisilicon/hikey960/platform.mk

@@ -20,6 +20,7 @@ endif
 CRASH_CONSOLE_BASE		:=	PL011_UART6_BASE
 COLD_BOOT_SINGLE_CPU		:=	1
 PROGRAMMABLE_RESET_ADDRESS	:=	1
+ENABLE_SVE_FOR_NS		:=	0
 
 # Process flags
 $(eval $(call add_define,HIKEY960_TSP_RAM_LOCATION_ID))

plat/hisilicon/poplar/platform.mk

@@ -13,6 +13,7 @@ ENABLE_PLAT_COMPAT		:= 0
 ERRATA_A53_855873		:= 1
 ERRATA_A53_835769		:= 1
 ERRATA_A53_843419		:= 1
+ENABLE_SVE_FOR_NS		:= 0
 
 ARM_GIC_ARCH			:= 2
 $(eval $(call add_define,ARM_GIC_ARCH))
@@ -69,4 +70,3 @@ BL31_SOURCES	+=							\
 		plat/hisilicon/poplar/bl31_plat_setup.c			\
 		plat/hisilicon/poplar/plat_topology.c			\
 		plat/hisilicon/poplar/plat_pm.c
-

plat/mediatek/mt6795/platform.mk

@@ -66,3 +66,5 @@ PROGRAMMABLE_RESET_ADDRESS	:=	1
 
 $(eval $(call add_define,MTK_SIP_KERNEL_BOOT_ENABLE))
 
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0

plat/mediatek/mt8173/platform.mk

@@ -70,3 +70,6 @@ ERRATA_A53_855873	:=	1
 PROGRAMMABLE_RESET_ADDRESS	:=	1
 
 $(eval $(call add_define,MTK_SIP_SET_AUTHORIZED_SECURE_REG_ENABLE))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS		:=	0

plat/nvidia/tegra/platform.mk

@@ -29,6 +29,9 @@ SEPARATE_CODE_AND_RODATA :=	1
 # do not use coherent memory
 USE_COHERENT_MEM	:=	0
 
+# do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0
+
 include plat/nvidia/tegra/common/tegra_common.mk
 include ${SOC_DIR}/platform_${TARGET_SOC}.mk
 

plat/qemu/platform.mk

@@ -153,3 +153,6 @@ endif
 
 # Process flags
 $(eval $(call add_define,BL32_RAM_LOCATION_ID))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0

plat/rockchip/rk3328/platform.mk

@@ -52,3 +52,6 @@ ENABLE_PLAT_COMPAT 	:=      0
 
 $(eval $(call add_define,PLAT_EXTRA_LD_SCRIPT))
 $(eval $(call add_define,PLAT_SKIP_OPTEE_S_EL1_INT_REGISTER))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0

plat/rockchip/rk3368/platform.mk

@@ -51,3 +51,6 @@ BL31_SOURCES		+=	${RK_GIC_SOURCES}				\
 ENABLE_PLAT_COMPAT	:=	0
 
 $(eval $(call add_define,PLAT_EXTRA_LD_SCRIPT))
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0

plat/rockchip/rk3399/platform.mk

@@ -92,3 +92,6 @@ $(eval $(call MAKE_PREREQ_DIR,${BUILD_M0},${BUILD_PLAT}))
 .PHONY: $(RK3399M0FW)
 $(RK3399M0FW): | ${BUILD_M0}
 	$(MAKE) -C ${RK_PLAT_SOC}/drivers/m0 BUILD=$(abspath ${BUILD_PLAT}/m0)
+
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:=	0

plat/socionext/uniphier/platform.mk

@@ -10,6 +10,7 @@ override ERROR_DEPRECATED	:= 1
 override LOAD_IMAGE_V2		:= 1
 override USE_COHERENT_MEM	:= 1
 override USE_TBBR_DEFS		:= 1
+override ENABLE_SVE_FOR_NS	:= 0
 
 # Cortex-A53 revision r0p4-51rel0
 # needed for LD20, unneeded for LD11, PXs3 (no ACE)

plat/xilinx/zynqmp/platform.mk

@@ -11,6 +11,9 @@ A53_DISABLE_NON_TEMPORAL_HINT := 0
 SEPARATE_CODE_AND_RODATA := 1
 override RESET_TO_BL31 := 1
 
+# Do not enable SVE
+ENABLE_SVE_FOR_NS	:= 0
+
 ifdef ZYNQMP_ATF_MEM_BASE
     $(eval $(call add_define,ZYNQMP_ATF_MEM_BASE))