Merge patch series "test: Improvements to ut command and test-suite running"

Simon Glass <sjg@chromium.org> says:

The current method of running unit tests relies on subcommands of the
ut command. Only the code in each subcommand knows how to find the tests
related to that subcomand.

This is not ideal and we now have quite a few subcommands which do
nothing but locate the relevant tests in a linker list, then call a
common function to run them.

This series adds a list of test suites, so that these subcommands can be
removed.

An issue with 'ut all' is that it doesn't record how many tests failed
overall, so it is necessary to examine copious amounts of output to look
for failures. This series adds a new 'total' feature allow recording the
total number of failed tests.

To help with 'ut all' a new pytest is created which runs it (as well as
'ut info') and makes sure that all is well. Due to the 'ut all' failures
this does not pass, so the test is disabled for now. It is here because
it provides security against misnaming a test suite and causing it not
to run.

Future work may:
- get 'ut all' passing
- enable test_suite() in CL, to ensure that 'ut all' keeps passing
- record duration of each suite
- allow running the tests in random order to tease out dependencies
- tweak the output to remove common prefixes
- getting rid of bootstd, optee and seame 'ut' subcommands

Link: https://lore.kernel.org/r/20250120212613.516664-1-sjg@chromium.org

arch/sandbox/cpu/spl.c

@@ -147,10 +147,14 @@ void spl_board_init(void)
 	if (state->run_unittests) {
 		struct unit_test *tests = UNIT_TEST_ALL_START();
 		const int count = UNIT_TEST_ALL_COUNT();
+		struct unit_test_state uts;
 		int ret;
 
-		ret = ut_run_list("spl", NULL, tests, count,
+		ut_init_state(&uts);
+		ret = ut_run_list(&uts, "spl", NULL, tests, count,
 				  state->select_unittests, 1, false, NULL);
+		ut_report(&uts.cur, 1);
+		ut_uninit_state(&uts);
 		/* continue execution into U-Boot */
 	}
 }

doc/usage/cmd/ut.rst

@@ -11,7 +11,7 @@ Synopsis
 
 ::
 
-    ut [-r<runs>] [-f] [-I<n>:<one_test>] [<suite> [<test>]]
+    ut [-r<runs>] [-fs] [-I<n>:<one_test>] [<suite> [<test>]]
 
        <runs>      Number of times to run each test
        -f          Force 'manual' tests to run as well
@@ -30,7 +30,7 @@ includes a near-complete set of emulators, no code-size limits, many CONFIG
 options enabled and runs easily in CI without needing QEMU. It is also possible
 to run some tests on real boards.
 
-For a list of available test suites, type `ut` by itself.
+For a list of available test suites, type `ut info -s`.
 
 Each test is normally run once, although those marked with `UTF_DM` are
 run with livetree and flattree where possible. To run a test more than once,
@@ -56,36 +56,44 @@ write unit tests.
 Example
 -------
 
+Show information about tests::
+
+    => ut info
+    Test suites: 23
+    Total tests: 833
+
 List available unit-test suites::
 
-    => ut
+    => ut info -s
-    ut - unit tests
+    Test suites: 23
+    Total tests: 833
 
-    Usage:
+    Tests  Suite         Purpose
-    ut [-r] [-f] [<suite>] - run unit tests
+    -----  ------------  -------------------------
-    -r<runs>   Number of times to run each test
+        1  addrmap       very basic test of addrmap command
-    -f         Force 'manual' tests to run as well
+        4  bdinfo        bdinfo (board info) command
-    <suite>    Test suite to run, or all
+       14  bloblist      bloblist implementation
+        7  bootm         bootm command
+       66  bootstd       standard boot implementation
+        2  cmd           various commands
+       14  common        tests for common/ directory
+       502 dm            driver model
+        6  env           environment
+        1  exit          shell exit and variables
+       19  fdt           fdt command
+       10  fdt_overlay   device tree overlays
+        1  font          font command
+       20  hush          hush behaviour
+      115  lib           library functions
+        2  loadm         loadm command parameters and loading memory blob
+       18  log           logging functions
+        1  mbr           mbr command
+        1  measurement   TPM-based measured boot
+       13  mem           memory-related commands
+        1  pci_mps       PCI Express Maximum Payload Size
+       11  setexpr       setexpr command
+        4  upl           Universal payload support
 
-    Suites:
-    all - execute all enabled tests
-    addrmap - very basic test of addrmap command
-    bloblist - bloblist implementation
-    bootstd - standard boot implementation
-    compression - compressors and bootm decompression
-    dm - driver model
-    env - environment
-    fdt - fdt command
-    loadm - loadm command parameters and loading memory blob
-    lib - library functions
-    log - logging functions
-    mem - memory-related commands
-    overlay - device tree overlays
-    print  - printing things to the console
-    setexpr - setexpr command
-    str - basic test of string functions
-    time - very basic test of time functions
-    unicode - Unicode functions
 
 Run one of the suites::
 
@@ -112,9 +120,3 @@ Run just a single test in a suite::
     => ut bloblist bloblist_test_grow
     Test: bloblist_test_grow: bloblist.c
     Failures: 0
-
-Show information about tests::
-
-    => ut info
-    Test suites: 21
-    Total tests: 642

include/dm/test.h

@@ -143,7 +143,7 @@ extern struct unit_test_state global_dm_test_state;
 
 /* Declare a new driver model test */
 #define DM_TEST(_name, _flags) \
-	UNIT_TEST(_name, UTF_DM | UTF_CONSOLE | (_flags), dm_test)
+	UNIT_TEST(_name, UTF_DM | UTF_CONSOLE | (_flags), dm)
 
 /*
  * struct sandbox_sdl_plat - Platform data for the SDL video driver

include/linker_lists.h

@@ -186,6 +186,63 @@
 		_ll_result;						\
 	})
 
+/**
+ * Declares a symbol that points to the start/end of the list.
+ *
+ * @_sym:	Arbitrary name for the symbol (to use later in the file)
+ * @_type:	Data type of the entry
+ * @_list:	Name of the list in which this entry is placed
+ *
+ * The name of the (new) symbol is arbitrary and can be anything that is not
+ * already declared in the file where it appears. It is provided in _sym and
+ * can then be used (later in the same file) within a data structure.
+ * The _type and _list arguments must match those passed to ll_entry_start/end()
+ *
+ * Example:
+ *
+ * Here we want to record the start of each sub-command in a list. We have two
+ * sub-commands, 'bob' and 'mary'.
+ *
+ * In bob.c::
+ *
+ *   ll_entry_declare(struct my_sub_cmd, bob_cmd, cmd_sub) = {...};
+ *
+ * In mary.c::
+ *
+ *   ll_entry_declare(struct my_sub_cmd, mary_cmd, cmd_sub) = {...};
+ *
+ * In a different file where we want a list the start of all sub-commands.
+ * It is not possible to use ll_entry_start() in a data structure, due to its
+ * use of code inside expressions - ({ ... }) - so this fails to compile:
+ *
+ * In sub_cmds.c::
+ *
+ *    struct cmd_sub *my_list[] = {
+ *      ll_entry_start(cmd_sub, bob),
+ *      ll_entry_start(cmd_sub, bob),
+ *    };
+ *
+ * Instead, we can use::
+ *
+ *    ll_start_decl(bob, struct my_sub_cmd, cmd_sub);
+ *    ll_start_decl(mary, struct my_sub_cmd, cmd_sub);
+ *
+ *     struct cmd_sub *my_list[] = {
+ *       bob,
+ *       mary,
+ *     };
+ *
+ * So 'bob' is declared as symbol, a struct my_list * which points to the
+ * start of the bob sub-commands. It is then used in my_list[]
+ */
+#define ll_start_decl(_sym, _type, _list)					\
+	static _type _sym[0] __aligned(CONFIG_LINKER_LIST_ALIGN)	\
+		__maybe_unused __section("__u_boot_list_2_" #_list "_1")
+
+#define ll_end_decl(_sym, _type, _list)					\
+	static _type _sym[0] __aligned(CONFIG_LINKER_LIST_ALIGN)	\
+		__maybe_unused __section("__u_boot_list_2_" #_list "_3")
+
 /**
  * ll_entry_get() - Retrieve entry from linker-generated array by name
  * @_type:	Data type of the entry

include/test/cmd.h

@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new command test */
-#define CMD_TEST(_name, _flags) UNIT_TEST(_name, _flags, cmd_test)
+#define CMD_TEST(_name, _flags) UNIT_TEST(_name, _flags, cmd)
 
 #endif /* __TEST_CMD_H__ */

include/test/common.h

@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new common function test */
-#define COMMON_TEST(_name, _flags) UNIT_TEST(_name, _flags, common_test)
+#define COMMON_TEST(_name, _flags) UNIT_TEST(_name, _flags, common)
 
 #endif /* __TEST_COMMON_H__ */

include/test/env.h

@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define ENV_TEST(_name, _flags)	UNIT_TEST(_name, _flags, env_test)
+#define ENV_TEST(_name, _flags)	UNIT_TEST(_name, _flags, env)
 
 #endif /* __TEST_ENV_H__ */

include/test/hush.h

@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define HUSH_TEST(_name, _flags)	UNIT_TEST(_name, _flags, hush_test)
+#define HUSH_TEST(_name, _flags)	UNIT_TEST(_name, _flags, hush)
 
 #endif /* __TEST_HUSH_H__ */

include/test/lib.h

@@ -9,6 +9,6 @@
 #include <test/test.h>
 
 /* Declare a new library function test */
-#define LIB_TEST(_name, _flags)	UNIT_TEST(_name, _flags, lib_test)
+#define LIB_TEST(_name, _flags)	UNIT_TEST(_name, _flags, lib)
 
 #endif /* __TEST_LIB_H__ */

include/test/log.h

@@ -13,8 +13,8 @@
 #define LOGF_TEST (BIT(LOGF_FUNC) | BIT(LOGF_MSG))
 
 /* Declare a new logging test */
-#define LOG_TEST(_name) UNIT_TEST(_name, UTF_CONSOLE, log_test)
+#define LOG_TEST(_name) UNIT_TEST(_name, UTF_CONSOLE, log)
 #define LOG_TEST_FLAGS(_name, _flags) \
-		UNIT_TEST(_name, _flags | UTF_CONSOLE, log_test)
+		UNIT_TEST(_name, _flags | UTF_CONSOLE, log)
 
 #endif /* __TEST_LOG_H__ */

include/test/optee.h

@@ -9,6 +9,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define OPTEE_TEST(_name, _flags)	UNIT_TEST(_name, _flags, optee_test)
+#define OPTEE_TEST(_name, _flags)	UNIT_TEST(_name, _flags, optee)
 
 #endif /* __TEST_OPTEE_H__ */

include/test/overlay.h

@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define OVERLAY_TEST(_name, _flags)	UNIT_TEST(_name, _flags, overlay_test)
+#define OVERLAY_TEST(_name, _flags)	UNIT_TEST(_name, _flags, overlay)
 
 #endif /* __TEST_OVERLAY_H__ */

include/test/suites.h

@@ -9,10 +9,18 @@
 
 struct cmd_tbl;
 struct unit_test;
+struct unit_test_state;
+
+/* 'command' functions normally called do_xxx where xxx is the command name */
+typedef int (*ut_cmd_func)(struct unit_test_state *uts, struct cmd_tbl *cmd,
+			   int flags, int argc, char *const argv[]);
 
 /**
  * cmd_ut_category() - Run a category of unit tests
  *
+ * @uts: Unit-test state, which must be ready for use, i.e. ut_init_state()
+ *	has been called. The caller is responsible for calling
+ *	ut_uninit_state() after this function returns
  * @name:	Category name
  * @prefix:	Prefix of test name
  * @tests:	List of tests to run
@@ -22,47 +30,14 @@ struct unit_test;
  * @argv:	Arguments: argv[1] is the test to run (if @argc >= 2)
  * Return: 0 if OK, CMD_RET_FAILURE on failure
  */
-int cmd_ut_category(const char *name, const char *prefix,
+int cmd_ut_category(struct unit_test_state *uts, const char *name,
-		    struct unit_test *tests, int n_ents,
+		    const char *prefix, struct unit_test *tests, int n_ents,
 		    int argc, char *const argv[]);
 
-int do_ut_addrmap(struct cmd_tbl *cmdtp, int flag, int argc,
+int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
-		  char *const argv[]);
+		  int argc, char *const argv[]);
-int do_ut_bdinfo(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_bootm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc,
-		   char *const argv[]);
-int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_common(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_compression(struct cmd_tbl *cmdtp, int flag, int argc,
-		      char *const argv[]);
-int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_lib(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_log(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]);
-int do_ut_mbr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]);
-int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
 int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc,
+int do_ut_overlay(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
-		  char *const argv[]);
+		  int argc, char *const argv[]);
-int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_print(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_seama(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_setexpr(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_str(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_time(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_unicode(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
 
 #endif /* __TEST_SUITES_H__ */

include/test/test.h

@@ -9,11 +9,26 @@
 #include <malloc.h>
 #include <linux/bitops.h>
 
-/*
+/**
- * struct unit_test_state - Entire state of test system
+ * struct ut_stats - Statistics about tests run
  *
  * @fail_count: Number of tests that failed
  * @skip_count: Number of tests that were skipped
+ * @test_count: Number of tests run. If a test is run muiltiple times, only one
+ *	is counted
+ */
+struct ut_stats {
+	int fail_count;
+	int skip_count;
+	int test_count;
+};
+
+/*
+ * struct unit_test_state - Entire state of test system
+ *
+ * @cur: Statistics for the current run
+ * @total: Statistics for all test runs
+ * @run_count: Number of times ut_run_list() has been called
  * @start: Store the starting mallinfo when doing leak test
  * @of_live: true to use livetree if available, false to use flattree
  * @of_root: Record of the livetree root node (used for setting up tests)
@@ -34,8 +49,9 @@
  * @actual_str: Temporary string used to hold actual string value
  */
 struct unit_test_state {
-	int fail_count;
+	struct ut_stats cur;
-	int skip_count;
+	struct ut_stats total;
+	int run_count;
 	struct mallinfo start;
 	struct device_node *of_root;
 	bool of_live;

include/test/ut.h

@@ -466,18 +466,37 @@ void ut_unsilence_console(struct unit_test_state *uts);
 void ut_set_skip_delays(struct unit_test_state *uts, bool skip_delays);
 
 /**
- * test_get_state() - Get the active test state
+ * ut_state_get() - Get the active test state
  *
  * Return: the currently active test state, or NULL if none
  */
-struct unit_test_state *test_get_state(void);
+struct unit_test_state *ut_get_state(void);
 
 /**
- * test_set_state() - Set the active test state
+ * ut_set_state() - Set the active test state
  *
  * @uts: Test state to use as currently active test state, or NULL if none
  */
-void test_set_state(struct unit_test_state *uts);
+void ut_set_state(struct unit_test_state *uts);
+
+/**
+ * ut_init_state() - Set up a new test state
+ *
+ * This must be called before using the test state with ut_run_tests()
+ *
+ * @uts: Test state to init
+ */
+void ut_init_state(struct unit_test_state *uts);
+
+/**
+ * ut_uninit_state() - Free memory used by test state
+ *
+ * This must be called before after the test state with ut_run_tests(). To later
+ * reuse the test state to run more tests, call test_state_init() first
+ *
+ * @uts: Test state to uninit
+ */
+void ut_uninit_state(struct unit_test_state *uts);
 
 /**
  * ut_run_tests() - Run a set of tests
@@ -485,6 +504,9 @@ void test_set_state(struct unit_test_state *uts);
  * This runs the test, handling any preparation and clean-up needed. It prints
  * the name of each test before running it.
  *
+ * @uts: Unit-test state, which must be ready for use, i.e. ut_init_state()
+ *	has been called. The caller is responsible for calling
+ *	ut_uninit_state() after this function returns
  * @category: Category of these tests. This is a string printed at the start to
  *	announce the the number of tests
  * @prefix: String prefix for the tests. Any tests that have this prefix will be
@@ -503,8 +525,17 @@ void test_set_state(struct unit_test_state *uts);
  * Pass NULL to disable this
  * Return: 0 if all tests passed, -1 if any failed
  */
-int ut_run_list(const char *name, const char *prefix, struct unit_test *tests,
+int ut_run_list(struct unit_test_state *uts, const char *category,
-		int count, const char *select_name, int runs_per_test,
+		const char *prefix, struct unit_test *tests, int count,
-		bool force_run, const char *test_insert);
+		const char *select_name, int runs_per_test, bool force_run,
+		const char *test_insert);
+
+/**
+ * ut_report() - Report stats on a test run
+ *
+ * @stats: Stats to show
+ * @run_count: Number of suites that were run
+ */
+void ut_report(struct ut_stats *stats, int run_count);
 
 #endif

test/boot/bootm.c

@@ -13,7 +13,7 @@
 
 DECLARE_GLOBAL_DATA_PTR;
 
-#define BOOTM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bootm_test)
+#define BOOTM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bootm)
 
 enum {
 	BUF_SIZE	= 1024,
@@ -249,12 +249,3 @@ static int bootm_test_subst_both(struct unit_test_state *uts)
 	return 0;
 }
 BOOTM_TEST(bootm_test_subst_both, 0);
-
-int do_ut_bootm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bootm_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bootm_test);
-
-	return cmd_ut_category("bootm", "bootm_test_", tests, n_ents,
-			       argc, argv);
-}

test/boot/bootstd_common.c

@@ -94,10 +94,11 @@ void bootstd_reset_usb(void)
 	usb_started = false;
 }
 
-int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
+int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
+		  int argc, char *const argv[])
 {
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bootstd_test);
+	struct unit_test *tests = UNIT_TEST_SUITE_START(bootstd);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bootstd_test);
+	const int n_ents = UNIT_TEST_SUITE_COUNT(bootstd);
 	int ret;
 
 	ret = bootstd_setup_for_tests();
@@ -106,6 +107,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 		return CMD_RET_FAILURE;
 	}
 
-	return cmd_ut_category("bootstd", "bootstd_test_",
+	return cmd_ut_category(uts, "bootstd", "bootstd_",
 			       tests, n_ents, argc, argv);
 }

test/boot/bootstd_common.h

@@ -12,8 +12,7 @@
 #include <version_string.h>
 
 /* Declare a new bootdev test */
-#define BOOTSTD_TEST(_name, _flags) \
+#define BOOTSTD_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bootstd)
-		UNIT_TEST(_name, _flags, bootstd_test)
 
 #define NVDATA_START_BLK	((0x400 + 0x400) / MMC_MAX_BLOCK_LEN)
 #define VERSION_START_BLK	((0x400 + 0x800) / MMC_MAX_BLOCK_LEN)

test/boot/measurement.c

@@ -14,7 +14,7 @@
 #include <asm/io.h>
 
 #define MEASUREMENT_TEST(_name, _flags)	\
-	UNIT_TEST(_name, _flags, measurement_test)
+	UNIT_TEST(_name, _flags, measurement)
 
 static int measure(struct unit_test_state *uts)
 {
@@ -53,13 +53,3 @@ static int measure(struct unit_test_state *uts)
 	return 0;
 }
 MEASUREMENT_TEST(measure, 0);
-
-int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc,
-		      char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(measurement_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(measurement_test);
-
-	return cmd_ut_category("measurement", "measurement_test_", tests,
-			       n_ents, argc, argv);
-}

test/boot/upl.c

@@ -16,7 +16,7 @@
 #include "bootstd_common.h"
 
 /* Declare a new upl test */
-#define UPL_TEST(_name, _flags)	UNIT_TEST(_name, _flags, upl_test)
+#define UPL_TEST(_name, _flags)	UNIT_TEST(_name, _flags, upl)
 
 static int add_region(struct unit_test_state *uts, struct alist *lst,
 		      ulong base, ulong size)
@@ -426,12 +426,3 @@ static int upl_test_info_norun(struct unit_test_state *uts)
 	return 0;
 }
 UPL_TEST(upl_test_info_norun, UTF_CONSOLE | UTF_MANUAL);
-
-int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(upl_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(upl_test);
-
-	return cmd_ut_category("cmd_upl", "cmd_upl_", tests, n_ents, argc,
-			       argv);
-}

test/cmd/Makefile

@@ -3,8 +3,6 @@
 # Copyright (c) 2013 Google, Inc
 # Copyright 2022-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
 
-obj-y += cmd_ut_cmd.o
-
 obj-$(CONFIG_$(XPL_)CMDLINE) += command.o
 ifdef CONFIG_HUSH_PARSER
 obj-$(CONFIG_CONSOLE_RECORD) += test_echo.o
@@ -12,7 +10,7 @@ endif
 ifdef CONFIG_CONSOLE_RECORD
 obj-$(CONFIG_CMD_PAUSE) += test_pause.o
 endif
-obj-y += exit.o mem.o
+obj-y += exit.o
 obj-$(CONFIG_X86) += cpuid.o msr.o
 obj-$(CONFIG_CMD_ADDRMAP) += addrmap.o
 obj-$(CONFIG_CMD_BDI) += bdinfo.o

test/cmd/addrmap.c

@@ -10,7 +10,7 @@
 #include <test/ut.h>
 
 /* Declare a new addrmap test */
-#define ADDRMAP_TEST(_name, _flags)	UNIT_TEST(_name, _flags, addrmap_test)
+#define ADDRMAP_TEST(_name, _flags)	UNIT_TEST(_name, _flags, addrmap)
 
 /* Test 'addrmap' command output */
 static int addrmap_test_basic(struct unit_test_state *uts)
@@ -24,12 +24,3 @@ static int addrmap_test_basic(struct unit_test_state *uts)
 	return 0;
 }
 ADDRMAP_TEST(addrmap_test_basic, UTF_CONSOLE);
-
-int do_ut_addrmap(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(addrmap_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(addrmap_test);
-
-	return cmd_ut_category("cmd_addrmap", "cmd_addrmap_", tests, n_ents,
-			       argc, argv);
-}

test/cmd/bdinfo.c

@@ -26,7 +26,7 @@
 DECLARE_GLOBAL_DATA_PTR;
 
 /* Declare a new bdinfo test */
-#define BDINFO_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bdinfo_test)
+#define BDINFO_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bdinfo)
 
 static int test_num_l(struct unit_test_state *uts, const char *name,
 		      ulong value)
@@ -282,11 +282,3 @@ static int bdinfo_test_eth(struct unit_test_state *uts)
 	return 0;
 }
 BDINFO_TEST(bdinfo_test_eth, UTF_CONSOLE);
-
-int do_ut_bdinfo(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bdinfo_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bdinfo_test);
-
-	return cmd_ut_category("bdinfo", "bdinfo_test_", tests, n_ents, argc, argv);
-}

test/cmd/cmd_ut_cmd.c

@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright 2023 Google LLC
- * Written by Simon Glass <sjg@chromium.org>
- *
- * Unit tests for command functions
- */
-
-#include <command.h>
-#include <test/cmd.h>
-#include <test/suites.h>
-#include <test/ut.h>
-
-int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(cmd_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(cmd_test);
-
-	return cmd_ut_category("cmd", "cmd_test_", tests, n_ents, argc, argv);
-}

test/cmd/exit.c

@@ -14,7 +14,7 @@
 DECLARE_GLOBAL_DATA_PTR;
 
 /* Declare a new exit test */
-#define EXIT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, exit_test)
+#define EXIT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, exit)
 
 /* Test 'exit addr' getting/setting address */
 static int cmd_exit_test(struct unit_test_state *uts)
@@ -110,12 +110,3 @@ static int cmd_exit_test(struct unit_test_state *uts)
 	return 0;
 }
 EXIT_TEST(cmd_exit_test, UTF_CONSOLE);
-
-int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(exit_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(exit_test);
-
-	return cmd_ut_category("cmd_exit", "exit_test_", tests, n_ents,
-			       argc, argv);
-}

test/cmd/fdt.c

@@ -23,7 +23,7 @@ DECLARE_GLOBAL_DATA_PTR;
  */
 
 /* Declare a new fdt test */
-#define FDT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, fdt_test)
+#define FDT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, fdt)
 
 /**
  * make_test_fdt() - Create an FDT with just a root node
@@ -1462,11 +1462,3 @@ static int fdt_test_apply(struct unit_test_state *uts)
 	return 0;
 }
 FDT_TEST(fdt_test_apply, UTF_CONSOLE);
-
-int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(fdt_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(fdt_test);
-
-	return cmd_ut_category("fdt", "fdt_test_", tests, n_ents, argc, argv);
-}

test/cmd/font.c

@@ -12,7 +12,7 @@
 #include <test/ut.h>
 
 /* Declare a new fdt test */
-#define FONT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, font_test)
+#define FONT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, font)
 
 /* Test 'fdt addr' resizing an fdt */
 static int font_test_base(struct unit_test_state *uts)
@@ -85,11 +85,3 @@ static int font_test_base(struct unit_test_state *uts)
 }
 FONT_TEST(font_test_base, UTF_SCAN_PDATA | UTF_SCAN_FDT | UTF_CONSOLE |
 	  UTF_DM);
-
-int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(font_Test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(font_test);
-
-	return cmd_ut_category("font", "font_test_", tests, n_ents, argc, argv);
-}

test/cmd/loadm.c

@@ -19,7 +19,7 @@
 
 #define BUF_SIZE 0x100
 
-#define LOADM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, loadm_test)
+#define LOADM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, loadm)
 
 static int loadm_test_params(struct unit_test_state *uts)
 {
@@ -58,12 +58,3 @@ static int loadm_test_load (struct unit_test_state *uts)
 	return 0;
 }
 LOADM_TEST(loadm_test_load, UTF_CONSOLE);
-
-int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(loadm_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(loadm_test);
-
-	return cmd_ut_category("loadm", "loadm_test_", tests, n_ents, argc,
-			       argv);
-}

test/cmd/mbr.c

@@ -470,12 +470,4 @@ static int mbr_test_run(struct unit_test_state *uts)
 }
 
 /* Declare mbr test */
-UNIT_TEST(mbr_test_run, UTF_CONSOLE, mbr_test);
+UNIT_TEST(mbr_test_run, UTF_CONSOLE, mbr);
-
-int do_ut_mbr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(mbr_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(mbr_test);
-
-	return cmd_ut_category("mbr", "mbr_test_", tests, n_ents, argc, argv);
-}

test/cmd/mem.c

@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Executes tests for memory-related commands
- *
- * Copyright 2020 Google LLC
- */
-
-#include <command.h>
-#include <test/suites.h>
-#include <test/test.h>
-
-int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(mem_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(mem_test);
-
-	return cmd_ut_category("cmd_mem", "mem_test_", tests, n_ents, argc,
-			       argv);
-}

test/cmd/mem_copy.c

@@ -12,7 +12,7 @@
 #define BUF_SIZE	256
 
 /* Declare a new mem test */
-#define MEM_TEST(_name)	UNIT_TEST(_name, 0, mem_test)
+#define MEM_TEST(_name)	UNIT_TEST(_name, 0, mem)
 
 struct param {
 	int d, s, count;

test/cmd/mem_search.c

@@ -14,7 +14,7 @@
 #define BUF_SIZE	0x100
 
 /* Declare a new mem test */
-#define MEM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, mem_test)
+#define MEM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, mem)
 
 /* Test 'ms' command with bytes */
 static int mem_test_ms_b(struct unit_test_state *uts)

test/cmd/pci_mps.c

@@ -11,7 +11,7 @@
 #include <test/suites.h>
 #include <test/ut.h>
 
-#define PCI_MPS_TEST(_name, _flags) UNIT_TEST(_name, _flags, pci_mps_test)
+#define PCI_MPS_TEST(_name, _flags) UNIT_TEST(_name, _flags, pci_mps)
 
 /* Test "pci_mps" command in safe "s" mode */
 static int test_pci_mps_safe(struct unit_test_state *uts)
@@ -28,13 +28,3 @@ static int test_pci_mps_safe(struct unit_test_state *uts)
 	return 0;
 }
 PCI_MPS_TEST(test_pci_mps_safe, UTF_CONSOLE);
-
-int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char * const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(pci_mps_test);
-	const int n = UNIT_TEST_SUITE_COUNT(pci_mps_test);
-
-	return cmd_ut_category("cmd_pci_mps", "pci_mps_test_", tests, n,
-			       argc, argv);
-}

test/cmd/seama.c

@@ -11,7 +11,7 @@
 #include <test/test.h>
 #include <test/ut.h>
 
-#define SEAMA_TEST(_name, _flags)	UNIT_TEST(_name, _flags, seama_test)
+#define SEAMA_TEST(_name, _flags)	UNIT_TEST(_name, _flags, seama)
 
 static int seama_test_noargs(struct unit_test_state *uts)
 {
@@ -56,12 +56,3 @@ static int seama_test_index(struct unit_test_state *uts)
 	return 0;
 }
 SEAMA_TEST(seama_test_index, UTF_CONSOLE);
-
-int do_ut_seama(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(seama_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(seama_test);
-
-	return cmd_ut_category("seama", "seama_test_", tests, n_ents, argc,
-			       argv);
-}

test/cmd/setexpr.c

@@ -15,7 +15,7 @@
 #define BUF_SIZE	0x100
 
 /* Declare a new setexpr test */
-#define SETEXPR_TEST(_name, _flags)	UNIT_TEST(_name, _flags, setexpr_test)
+#define SETEXPR_TEST(_name, _flags)	UNIT_TEST(_name, _flags, setexpr)
 
 /* Test 'setexpr' command with simply setting integers */
 static int setexpr_test_int(struct unit_test_state *uts)
@@ -479,12 +479,3 @@ static int setexpr_test_fmt(struct unit_test_state *uts)
 }
 SETEXPR_TEST(setexpr_test_fmt, UTF_CONSOLE);
 #endif
-
-int do_ut_setexpr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(setexpr_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(setexpr_test);
-
-	return cmd_ut_category("cmd_setexpr", "setexpr_test_", tests, n_ents,
-			       argc, argv);
-}

test/cmd_ut.c

@@ -11,14 +11,34 @@
 #include <test/test.h>
 #include <test/ut.h>
 
-static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc,
+/**
-		     char *const argv[]);
+ * struct suite - A set of tests for a certain topic
+ *
+ * All tests end up in a single 'struct unit_test' linker-list array, in order
+ * of the suite they are in
+ *
+ * @name: Name of suite
+ * @start: First test in suite
+ * @end: End test in suite (points to the first test in the next suite)
+ * @cmd: Command to use to run the suite
+ * @help: Help-string to show for this suite
+ */
+struct suite {
+	const char *name;
+	struct unit_test *start;
+	struct unit_test *end;
+	ut_cmd_func cmd;
+	const char *help;
+};
+
+static int do_ut_all(struct unit_test_state *uts, struct cmd_tbl *cmdtp,
+		     int flag, int argc, char *const argv[]);
 
 static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc,
 		      char *const argv[]);
 
-int cmd_ut_category(const char *name, const char *prefix,
+int cmd_ut_category(struct unit_test_state *uts, const char *name,
-		    struct unit_test *tests, int n_ents,
+		    const char *prefix, struct unit_test *tests, int n_ents,
 		    int argc, char *const argv[])
 {
 	const char *test_insert = NULL;
@@ -44,105 +64,153 @@ int cmd_ut_category(const char *name, const char *prefix,
 		argc--;
 	}
 
-	ret = ut_run_list(name, prefix, tests, n_ents,
+	ret = ut_run_list(uts, name, prefix, tests, n_ents,
 			  cmd_arg1(argc, argv), runs_per_text, force_run,
 			  test_insert);
 
 	return ret ? CMD_RET_FAILURE : 0;
 }
 
-static struct cmd_tbl cmd_ut_sub[] = {
+/* declare linker-list symbols for the start and end of a suite */
-	U_BOOT_CMD_MKENT(all, CONFIG_SYS_MAXARGS, 1, do_ut_all, "", ""),
+#define SUITE_DECL(_name) \
-	U_BOOT_CMD_MKENT(info, 1, 1, do_ut_info, "", ""),
+	ll_start_decl(suite_start_ ## _name, struct unit_test, ut_ ## _name); \
-#ifdef CONFIG_CMD_BDI
+	ll_end_decl(suite_end_ ## _name, struct unit_test, ut_ ## _name)
-	U_BOOT_CMD_MKENT(bdinfo, CONFIG_SYS_MAXARGS, 1, do_ut_bdinfo, "", ""),
+
-#endif
+/* declare a test suite which uses a subcommand to run */
+#define SUITE_CMD(_name, _cmd_func, _help) { \
+	#_name, \
+	suite_start_ ## _name, \
+	suite_end_ ## _name, \
+	_cmd_func, \
+	_help, \
+	}
+
+/* declare a test suite which can be run directly without a subcommand */
+#define SUITE(_name, _help) { \
+	#_name, \
+	suite_start_ ## _name, \
+	suite_end_ ## _name, \
+	NULL, \
+	_help, \
+	}
+
+SUITE_DECL(addrmap);
+SUITE_DECL(bdinfo);
+SUITE_DECL(bloblist);
+SUITE_DECL(bootm);
+SUITE_DECL(bootstd);
+SUITE_DECL(cmd);
+SUITE_DECL(common);
+SUITE_DECL(dm);
+SUITE_DECL(env);
+SUITE_DECL(exit);
+SUITE_DECL(fdt);
+SUITE_DECL(font);
+SUITE_DECL(hush);
+SUITE_DECL(lib);
+SUITE_DECL(loadm);
+SUITE_DECL(log);
+SUITE_DECL(mbr);
+SUITE_DECL(measurement);
+SUITE_DECL(mem);
+SUITE_DECL(optee);
+SUITE_DECL(overlay);
+SUITE_DECL(pci_mps);
+SUITE_DECL(seama);
+SUITE_DECL(setexpr);
+SUITE_DECL(upl);
+
+static struct suite suites[] = {
+	SUITE(addrmap, "very basic test of addrmap command"),
+	SUITE(bdinfo, "bdinfo (board info) command"),
+	SUITE(bloblist, "bloblist implementation"),
+	SUITE(bootm, "bootm command"),
 #ifdef CONFIG_UT_BOOTSTD
-	U_BOOT_CMD_MKENT(bootstd, CONFIG_SYS_MAXARGS, 1, do_ut_bootstd,
+	SUITE_CMD(bootstd, do_ut_bootstd, "standard boot implementation"),
-			 "", ""),
-#endif
-#ifdef CONFIG_CMDLINE
-	U_BOOT_CMD_MKENT(cmd, CONFIG_SYS_MAXARGS, 1, do_ut_cmd, "", ""),
-#endif
-	U_BOOT_CMD_MKENT(common, CONFIG_SYS_MAXARGS, 1, do_ut_common, "", ""),
-#if defined(CONFIG_UT_DM)
-	U_BOOT_CMD_MKENT(dm, CONFIG_SYS_MAXARGS, 1, do_ut_dm, "", ""),
-#endif
-#if defined(CONFIG_UT_ENV)
-	U_BOOT_CMD_MKENT(env, CONFIG_SYS_MAXARGS, 1, do_ut_env, "", ""),
-#endif
-	U_BOOT_CMD_MKENT(exit, CONFIG_SYS_MAXARGS, 1, do_ut_exit, "", ""),
-#ifdef CONFIG_CMD_FDT
-	U_BOOT_CMD_MKENT(fdt, CONFIG_SYS_MAXARGS, 1, do_ut_fdt, "", ""),
-#endif
-#ifdef CONFIG_CONSOLE_TRUETYPE
-	U_BOOT_CMD_MKENT(font, CONFIG_SYS_MAXARGS, 1, do_ut_font, "", ""),
 #endif
+	SUITE(cmd, "various commands"),
+	SUITE(common, "tests for common/ directory"),
+	SUITE(dm, "driver model"),
+	SUITE(env, "environment"),
+	SUITE(exit, "shell exit and variables"),
+	SUITE(fdt, "fdt command"),
+	SUITE(font, "font command"),
+	SUITE(hush, "hush behaviour"),
+	SUITE(lib, "library functions"),
+	SUITE(loadm, "loadm command parameters and loading memory blob"),
+	SUITE(log, "logging functions"),
+	SUITE(mbr, "mbr command"),
+	SUITE(measurement, "TPM-based measured boot"),
+	SUITE(mem, "memory-related commands"),
 #ifdef CONFIG_UT_OPTEE
-	U_BOOT_CMD_MKENT(optee, CONFIG_SYS_MAXARGS, 1, do_ut_optee, "", ""),
+	SUITE_CMD(optee, do_ut_optee, "OP-TEE"),
 #endif
 #ifdef CONFIG_UT_OVERLAY
-	U_BOOT_CMD_MKENT(overlay, CONFIG_SYS_MAXARGS, 1, do_ut_overlay, "", ""),
+	SUITE_CMD(overlay, do_ut_overlay, "device tree overlays"),
-#endif
-#ifdef CONFIG_UT_LIB
-	U_BOOT_CMD_MKENT(lib, CONFIG_SYS_MAXARGS, 1, do_ut_lib, "", ""),
-#endif
-#ifdef CONFIG_UT_LOG
-	U_BOOT_CMD_MKENT(log, CONFIG_SYS_MAXARGS, 1, do_ut_log, "", ""),
-#endif
-#if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_MBR) && defined(CONFIG_CMD_MMC) \
-        && defined(CONFIG_MMC_SANDBOX) && defined(CONFIG_MMC_WRITE)
-	U_BOOT_CMD_MKENT(mbr, CONFIG_SYS_MAXARGS, 1, do_ut_mbr, "", ""),
-#endif
-	U_BOOT_CMD_MKENT(mem, CONFIG_SYS_MAXARGS, 1, do_ut_mem, "", ""),
-#if defined(CONFIG_SANDBOX) && defined(CONFIG_CMD_SETEXPR)
-	U_BOOT_CMD_MKENT(setexpr, CONFIG_SYS_MAXARGS, 1, do_ut_setexpr, "",
-			 ""),
-#endif
-#ifdef CONFIG_MEASURED_BOOT
-	U_BOOT_CMD_MKENT(measurement, CONFIG_SYS_MAXARGS, 1, do_ut_measurement,
-			 "", ""),
-#endif
-#ifdef CONFIG_SANDBOX
-#if CONFIG_IS_ENABLED(BLOBLIST)
-	U_BOOT_CMD_MKENT(bloblist, CONFIG_SYS_MAXARGS, 1, do_ut_bloblist,
-			 "", ""),
-	U_BOOT_CMD_MKENT(bootm, CONFIG_SYS_MAXARGS, 1, do_ut_bootm, "", ""),
-#endif
-#endif
-#ifdef CONFIG_CMD_ADDRMAP
-	U_BOOT_CMD_MKENT(addrmap, CONFIG_SYS_MAXARGS, 1, do_ut_addrmap, "", ""),
-#endif
-#if CONFIG_IS_ENABLED(HUSH_PARSER)
-	U_BOOT_CMD_MKENT(hush, CONFIG_SYS_MAXARGS, 1, do_ut_hush, "", ""),
-#endif
-#ifdef CONFIG_CMD_LOADM
-	U_BOOT_CMD_MKENT(loadm, CONFIG_SYS_MAXARGS, 1, do_ut_loadm, "", ""),
-#endif
-#ifdef CONFIG_CMD_PCI_MPS
-	U_BOOT_CMD_MKENT(pci_mps, CONFIG_SYS_MAXARGS, 1, do_ut_pci_mps, "", ""),
-#endif
-#ifdef CONFIG_CMD_SEAMA
-	U_BOOT_CMD_MKENT(seama, CONFIG_SYS_MAXARGS, 1, do_ut_seama, "", ""),
-#endif
-#ifdef CONFIG_CMD_UPL
-	U_BOOT_CMD_MKENT(upl, CONFIG_SYS_MAXARGS, 1, do_ut_upl, "", ""),
 #endif
+	SUITE(pci_mps, "PCI Express Maximum Payload Size"),
+	SUITE(seama, "seama command parameters loading and decoding"),
+	SUITE(setexpr, "setexpr command"),
+	SUITE(upl, "Universal payload support"),
 };
 
-static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc,
+/**
+ * has_tests() - Check if a suite has tests, i.e. is supported in this build
+ *
+ * If the suite is run using a command, we have to assume that tests may be
+ * present, since we have no visibility
+ *
+ * @ste: Suite to check
+ * Return: true if supported, false if not
+ */
+static bool has_tests(struct suite *ste)
+{
+	int n_ents = ste->end - ste->start;
+
+	return n_ents || ste->cmd;
+}
+
+/** run_suite() - Run a suite of tests */
+static int run_suite(struct unit_test_state *uts, struct suite *ste,
+		     struct cmd_tbl *cmdtp, int flag, int argc,
 		     char *const argv[])
+{
+	int ret;
+
+	if (ste->cmd) {
+		ret = ste->cmd(uts, cmdtp, flag, argc, argv);
+	} else {
+		int n_ents = ste->end - ste->start;
+		char prefix[30];
+
+		/* use a standard prefix */
+		snprintf(prefix, sizeof(prefix), "%s_test", ste->name);
+		ret = cmd_ut_category(uts, ste->name, prefix, ste->start,
+				      n_ents, argc, argv);
+	}
+
+	return ret;
+}
+
+static int do_ut_all(struct unit_test_state *uts, struct cmd_tbl *cmdtp,
+		     int flag, int argc, char *const argv[])
 {
 	int i;
 	int retval;
 	int any_fail = 0;
 
-	for (i = 1; i < ARRAY_SIZE(cmd_ut_sub); i++) {
+	for (i = 0; i < ARRAY_SIZE(suites); i++) {
-		printf("----Running %s tests----\n", cmd_ut_sub[i].name);
+		struct suite *ste = &suites[i];
-		retval = cmd_ut_sub[i].cmd(cmdtp, flag, 1, &cmd_ut_sub[i].name);
+		char *const argv[] = {(char *)ste->name, NULL};
-		if (!any_fail)
+
-			any_fail = retval;
+		if (has_tests(ste)) {
+			printf("----Running %s tests----\n", ste->name);
+			retval = run_suite(uts, ste, cmdtp, flag, 1, argv);
+			if (!any_fail)
+				any_fail = retval;
+		}
 	}
+	ut_report(&uts->total, uts->run_count);
 
 	return any_fail;
 }
@@ -150,15 +218,61 @@ static int do_ut_all(struct cmd_tbl *cmdtp, int flag, int argc,
 static int do_ut_info(struct cmd_tbl *cmdtp, int flag, int argc,
 		      char *const argv[])
 {
-	printf("Test suites: %d\n", (int)ARRAY_SIZE(cmd_ut_sub));
+	int suite_count, i;
+	const char *flags;
+
+	for (suite_count = 0, i = 0; i < ARRAY_SIZE(suites); i++) {
+		struct suite *ste = &suites[i];
+
+		if (has_tests(ste))
+			suite_count++;
+	}
+
+	printf("Test suites: %d\n", suite_count);
 	printf("Total tests: %d\n", (int)UNIT_TEST_ALL_COUNT());
 
+	flags = cmd_arg1(argc, argv);
+	if (flags && !strcmp("-s", flags)) {
+		int i;
+
+		puts("\nTests  Suite         Purpose");
+		puts("\n-----  ------------  -------------------------\n");
+		for (i = 0; i < ARRAY_SIZE(suites); i++) {
+			struct suite *ste = &suites[i];
+			long n_ent = ste->end - ste->start;
+
+			if (n_ent)
+				printf("%5ld", n_ent);
+			else if (ste->cmd)
+				printf("%5s", "?");
+			else  /* suite is not present */
+				continue;
+			printf("  %-13.13s %s\n", ste->name, ste->help);
+		}
+	}
+
 	return 0;
 }
 
+static struct suite *find_suite(const char *name)
+{
+	struct suite *ste;
+	int i;
+
+	for (i = 0, ste = suites; i < ARRAY_SIZE(suites); i++, ste++) {
+		if (!strcmp(ste->name, name))
+			return ste;
+	}
+
+	return NULL;
+}
+
 static int do_ut(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 {
-	struct cmd_tbl *cp;
+	struct unit_test_state uts;
+	struct suite *ste;
+	const char *name;
+	int ret;
 
 	if (argc < 2)
 		return CMD_RET_USAGE;
@@ -167,12 +281,30 @@ static int do_ut(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 	argc--;
 	argv++;
 
-	cp = find_cmd_tbl(argv[0], cmd_ut_sub, ARRAY_SIZE(cmd_ut_sub));
+	ut_init_state(&uts);
+	name = argv[0];
+	if (!strcmp(name, "all")) {
+		ret = do_ut_all(&uts, cmdtp, flag, argc, argv);
+	} else if (!strcmp(name, "info")) {
+		ret = do_ut_info(cmdtp, flag, argc, argv);
+	} else {
+		ste = find_suite(argv[0]);
+		if (!ste) {
+			printf("Suite '%s' not found\n", argv[0]);
+			return CMD_RET_FAILURE;
+		} else if (!has_tests(ste)) {
+			/* perhaps a Kconfig option needs to be set? */
+			printf("Suite '%s' is not enabled\n", argv[0]);
+			return CMD_RET_FAILURE;
+		}
 
-	if (cp)
+		ret = run_suite(&uts, ste, cmdtp, flag, argc, argv);
-		return cp->cmd(cmdtp, flag, argc, argv);
+	}
+	if (ret)
+		return ret;
+	ut_uninit_state(&uts);
 
-	return CMD_RET_USAGE;
+	return 0;
 }
 
 U_BOOT_LONGHELP(ut,
@@ -183,61 +315,7 @@ U_BOOT_LONGHELP(ut,
 	"\n"
 	"\nOptions for <suite>:"
 	"\nall - execute all enabled tests"
-	"\ninfo - show info about tests"
+	"\ninfo [-s] - show info about tests [and suites]"
-#ifdef CONFIG_CMD_ADDRMAP
-	"\naddrmap - very basic test of addrmap command"
-#endif
-#ifdef CONFIG_CMD_BDI
-	"\nbdinfo - bdinfo command"
-#endif
-#ifdef CONFIG_SANDBOX
-	"\nbloblist - bloblist implementation"
-#endif
-#ifdef CONFIG_BOOTSTD
-	"\nbootstd - standard boot implementation"
-#endif
-#ifdef CONFIG_CMDLINE
-	"\ncmd - test various commands"
-#endif
-	"\ncommon   - tests for common/ directory"
-#ifdef CONFIG_UT_DM
-	"\ndm - driver model"
-#endif
-#ifdef CONFIG_UT_ENV
-	"\nenv - environment"
-#endif
-#ifdef CONFIG_CMD_FDT
-	"\nfdt - fdt command"
-#endif
-#ifdef CONFIG_CONSOLE_TRUETYPE
-	"\nfont - font command"
-#endif
-#if CONFIG_IS_ENABLED(HUSH_PARSER)
-	"\nhush - Test hush behavior"
-#endif
-#ifdef CONFIG_CMD_LOADM
-	"\nloadm - loadm command parameters and loading memory blob"
-#endif
-#ifdef CONFIG_UT_LIB
-	"\nlib - library functions"
-#endif
-#ifdef CONFIG_UT_LOG
-	"\nlog - logging functions"
-#endif
-	"\nmem - memory-related commands"
-#ifdef CONFIG_UT_OPTEE
-	"\noptee - test OP-TEE"
-#endif
-#ifdef CONFIG_UT_OVERLAY
-	"\noverlay - device tree overlays"
-#endif
-#ifdef CONFIG_CMD_PCI_MPS
-	"\npci_mps - PCI Express Maximum Payload Size"
-#endif
-	"\nsetexpr - setexpr command"
-#ifdef CONFIG_CMD_SEAMA
-	"\nseama - seama command parameters loading and decoding"
-#endif
 	);
 
 U_BOOT_CMD(

test/common/Makefile

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0+
-obj-y += cmd_ut_common.o
 obj-$(CONFIG_AUTOBOOT) += test_autoboot.o
 
 ifneq ($(CONFIG_$(XPL_)BLOBLIST),)

test/common/bloblist.c

@@ -12,7 +12,7 @@
 
 /* Declare a new bloblist test */
 #define BLOBLIST_TEST(_name, _flags) \
-		UNIT_TEST(_name, _flags, bloblist_test)
+		UNIT_TEST(_name, _flags, bloblist)
 
 enum {
 	TEST_TAG		= BLOBLISTT_U_BOOT_SPL_HANDOFF,
@@ -602,13 +602,3 @@ static int bloblist_test_blob_maxsize(struct unit_test_state *uts)
 	return 0;
 }
 BLOBLIST_TEST(bloblist_test_blob_maxsize, UFT_BLOBLIST);
-
-int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc,
-		   char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bloblist_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bloblist_test);
-
-	return cmd_ut_category("bloblist", "bloblist_test_",
-			       tests, n_ents, argc, argv);
-}

test/common/cmd_ut_common.c

@@ -1,21 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (c) 2019 Heinrich Schuchardt <xypron.glpk@gmx.de>
- * Copyright (c) 2021 Steffen Jaeckel <jaeckel-floss@eyet-services.de>
- *
- * Unit tests for common functions
- */
-
-#include <command.h>
-#include <test/common.h>
-#include <test/suites.h>
-#include <test/ut.h>
-
-int do_ut_common(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(common_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(common_test);
-
-	return cmd_ut_category("common", "common_test_", tests, n_ents, argc,
-			       argv);
-}

test/dm/Makefile

@@ -3,8 +3,6 @@
 # Copyright (c) 2013 Google, Inc
 # Copyright 2022-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
 
-obj-$(CONFIG_UT_DM) += test-dm.o
-
 # Tests for particular subsystems - when enabling driver model for a new
 # subsystem you must add sandbox tests here.
 ifeq ($(CONFIG_XPL_BUILD),y)

test/dm/test-dm.c

@@ -1,16 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Copyright (c) 2013 Google, Inc
- */
-
-#include <test/suites.h>
-#include <test/test.h>
-
-int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(dm_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(dm_test);
-
-	return cmd_ut_category("driver model", "dm_test_", tests, n_ents, argc,
-			       argv);
-}

test/dm/test-driver.c

@@ -35,7 +35,7 @@ static const struct test_ops test_ops = {
 
 static int test_bind(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 
 	/* Private data should not be allocated */
 	ut_assert(!dev_get_priv(dev));
@@ -46,7 +46,7 @@ static int test_bind(struct udevice *dev)
 
 static int test_probe(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 	struct dm_test_priv *priv = dev_get_priv(dev);
 
 	/* Private data should be allocated */
@@ -59,7 +59,7 @@ static int test_probe(struct udevice *dev)
 
 static int test_remove(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 
 	/* Private data should still be allocated */
 	ut_assert(dev_get_priv(dev));
@@ -70,7 +70,7 @@ static int test_remove(struct udevice *dev)
 
 static int test_unbind(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 
 	/* Private data should not be allocated */
 	ut_assert(!dev_get_priv(dev));
@@ -121,7 +121,7 @@ static int test_manual_bind(struct udevice *dev)
 
 static int test_manual_probe(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 
 	dm_testdrv_op_count[DM_TEST_OP_PROBE]++;
 	if (!uts->force_fail_alloc)

test/dm/test-uclass.c

@@ -28,7 +28,7 @@ int test_ping(struct udevice *dev, int pingval, int *pingret)
 
 static int test_post_bind(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 	struct dm_test_perdev_uc_pdata *uc_pdata;
 
 	dm_testdrv_op_count[DM_TEST_OP_POST_BIND]++;
@@ -54,7 +54,7 @@ static int test_pre_unbind(struct udevice *dev)
 static int test_pre_probe(struct udevice *dev)
 {
 	struct dm_test_uclass_perdev_priv *priv = dev_get_uclass_priv(dev);
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 
 	dm_testdrv_op_count[DM_TEST_OP_PRE_PROBE]++;
 	ut_assert(priv);
@@ -65,7 +65,7 @@ static int test_pre_probe(struct udevice *dev)
 
 static int test_post_probe(struct udevice *dev)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 	struct udevice *prev = list_entry(dev->uclass_node.prev,
 					    struct udevice, uclass_node);
 
@@ -100,7 +100,7 @@ static int test_pre_remove(struct udevice *dev)
 
 static int test_init(struct uclass *uc)
 {
-	struct unit_test_state *uts = test_get_state();
+	struct unit_test_state *uts = ut_get_state();
 
 	dm_testdrv_op_count[DM_TEST_OP_INIT]++;
 	ut_assert(uclass_get_priv(uc));

test/env/cmd_ut_env.c

@@ -4,7 +4,6 @@
  * Joe Hershberger, National Instruments, joe.hershberger@ni.com
  */
 
-#include <command.h>
 #include <test/env.h>
 #include <test/suites.h>
 #include <test/ut.h>
@@ -74,12 +73,3 @@ static int env_test_env_cmd(struct unit_test_state *uts)
 	return 0;
 }
 ENV_TEST(env_test_env_cmd, UTF_CONSOLE);
-
-int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(env_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(env_test);
-
-	return cmd_ut_category("environment", "env_test_",
-			       tests, n_ents, argc, argv);
-}

test/hush/Makefile

@@ -3,7 +3,6 @@
 # (C) Copyright 2021
 # Francis Laniel, Amarula Solutions, francis.laniel@amarulasolutions.com
 
-obj-y += cmd_ut_hush.o
 obj-y += if.o
 ifdef CONFIG_CONSOLE_RECORD
 obj-y += dollar.o

test/hush/cmd_ut_hush.c

@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * (C) Copyright 2021
- * Francis Laniel, Amarula Solutions, francis.laniel@amarulasolutions.com
- */
-
-#include <command.h>
-#include <test/hush.h>
-#include <test/suites.h>
-#include <test/ut.h>
-
-int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(hush_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(hush_test);
-
-	return cmd_ut_category("hush", "hush_test_",
-			       tests, n_ents, argc, argv);
-}

test/hush/dollar.c

@@ -108,7 +108,7 @@ static int hush_test_simple_dollar(struct unit_test_state *uts)
 		/* Reset local variable. */
 		ut_assertok(run_command("dollar_foo=", 0));
 	} else if (gd->flags & GD_FLG_HUSH_OLD_PARSER) {
-		puts("Beware: this test set local variable dollar_foo and it cannot be unset!");
+		puts("Beware: this test set local variable dollar_foo and it cannot be unset!\n");
 	}
 
 	return 0;
@@ -140,7 +140,7 @@ static int hush_test_env_dollar(struct unit_test_state *uts)
 		/* Reset local variable. */
 		ut_assertok(run_command("env_foo=", 0));
 	} else if (gd->flags & GD_FLG_HUSH_OLD_PARSER) {
-		puts("Beware: this test set local variable env_foo and it cannot be unset!");
+		puts("Beware: this test set local variable env_foo and it cannot be unset!\n");
 	}
 
 	return 0;
@@ -206,7 +206,8 @@ static int hush_test_command_dollar(struct unit_test_state *uts)
 		ut_assertok(run_command("dollar_bar=", 0));
 		ut_assertok(run_command("dollar_quux=", 0));
 	} else if (gd->flags & GD_FLG_HUSH_OLD_PARSER) {
-		puts("Beware: this test sets local variable dollar_bar and dollar_quux and they cannot be unset!");
+		puts("Beware: this test sets local variable dollar_bar and "
+		     "dollar_quux and they cannot be unset!\n");
 	}
 
 	return 0;

test/hush/loop.c

@@ -25,7 +25,7 @@ static int hush_test_for(struct unit_test_state *uts)
 		/* Reset local variable. */
 		ut_assertok(run_command("loop_i=", 0));
 	} else if (gd->flags & GD_FLG_HUSH_OLD_PARSER) {
-		puts("Beware: this test set local variable loop_i and it cannot be unset!");
+		puts("Beware: this test set local variable loop_i and it cannot be unset!\n");
 	}
 
 	return 0;
@@ -56,7 +56,7 @@ static int hush_test_while(struct unit_test_state *uts)
 		/* Reset local variable. */
 		ut_assertok(run_command("loop_foo=", 0));
 	} else if (gd->flags & GD_FLG_HUSH_OLD_PARSER) {
-		puts("Beware: this test set local variable loop_foo and it cannot be unset!");
+		puts("Beware: this test set local variable loop_foo and it cannot be unset!\n");
 	}
 
 	return 0;

test/lib/Makefile

@@ -6,7 +6,6 @@
 obj-$(CONFIG_$(XPL_)UT_COMPRESSION) += compression.o
 
 ifeq ($(CONFIG_XPL_BUILD),)
-obj-y += cmd_ut_lib.o
 obj-y += abuf.o
 obj-y += alist.o
 obj-$(CONFIG_EFI_LOADER) += efi_device_path.o

test/lib/cmd_ut_lib.c

@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (c) 2019 Heinrich Schuchardt <xypron.glpk@gmx.de>
- *
- * Unit tests for library functions
- */
-
-#include <command.h>
-#include <test/lib.h>
-#include <test/suites.h>
-#include <test/ut.h>
-
-int do_ut_lib(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(lib_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(lib_test);
-
-	return cmd_ut_category("lib", "lib_test_", tests, n_ents, argc, argv);
-}

test/log/Makefile

@@ -7,8 +7,6 @@ obj-$(CONFIG_CMD_LOG) += log_filter.o
 
 ifdef CONFIG_UT_LOG
 
-obj-y += log_ut.o
-
 ifdef CONFIG_SANDBOX
 obj-$(CONFIG_LOG_SYSLOG) += syslog_test.o
 obj-$(CONFIG_LOG_SYSLOG) += syslog_test_ndebug.o

test/log/log_ut.c

@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Copyright (c) 2020, Heinrich Schuchardt <xypron.glpk@gmx.de>
- *
- * Logging function tests.
- */
-
-#include <console.h>
-#include <log.h>
-#include <test/log.h>
-#include <test/suites.h>
-
-int do_ut_log(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(log_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(log_test);
-
-	return cmd_ut_category("log", "log_test_",
-			       tests, n_ents, argc, argv);
-}

test/optee/cmd_ut_optee.c

@@ -93,8 +93,8 @@ OPTEE_TEST(optee_fdt_protected_memory, 0);
 
 int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 {
-	struct unit_test *tests = UNIT_TEST_SUITE_START(optee_test);
+	struct unit_test *tests = UNIT_TEST_SUITE_START(optee);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(optee_test);
+	const int n_ents = UNIT_TEST_SUITE_COUNT(optee);
 	struct unit_test_state *uts;
 	void *fdt_optee = &__dtb_test_optee_optee_begin;
 	void *fdt_no_optee = &__dtb_test_optee_no_optee_begin;

test/overlay/cmd_ut_overlay.c

@@ -210,21 +210,17 @@ static int fdt_overlay_stacked(struct unit_test_state *uts)
 }
 OVERLAY_TEST(fdt_overlay_stacked, 0);
 
-int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
+int do_ut_overlay(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
+		  int argc, char *const argv[])
 {
-	struct unit_test *tests = UNIT_TEST_SUITE_START(overlay_test);
+	struct unit_test *tests = UNIT_TEST_SUITE_START(overlay);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(overlay_test);
+	const int n_ents = UNIT_TEST_SUITE_COUNT(overlay);
-	struct unit_test_state *uts;
 	void *fdt_base = &__dtb_test_fdt_base_begin;
 	void *fdt_overlay = &__dtbo_test_fdt_overlay_begin;
 	void *fdt_overlay_stacked = &__dtbo_test_fdt_overlay_stacked_begin;
 	void *fdt_overlay_copy, *fdt_overlay_stacked_copy;
 	int ret = -ENOMEM;
 
-	uts = calloc(1, sizeof(*uts));
-	if (!uts)
-		return -ENOMEM;
-
 	ut_assertok(fdt_check_header(fdt_base));
 	ut_assertok(fdt_check_header(fdt_overlay));
 
@@ -272,7 +268,7 @@ int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 	/* Apply the stacked overlay */
 	ut_assertok(fdt_overlay_apply(fdt, fdt_overlay_stacked_copy));
 
-	ret = cmd_ut_category("overlay", "", tests, n_ents, argc, argv);
+	ret = cmd_ut_category(uts, "overlay", "", tests, n_ents, argc, argv);
 
 	free(fdt_overlay_stacked_copy);
 err3:

test/py/conftest.py

@@ -33,6 +33,9 @@ console = None
 
 TEST_PY_DIR = os.path.dirname(os.path.abspath(__file__))
 
+# Regex for test-function symbols
+RE_UT_TEST_LIST = re.compile(r'[^a-zA-Z0-9_]_u_boot_list_2_ut_(.*)_2_(.*)\s*$')
+
 def mkdir_p(path):
     """Create a directory path.
 
@@ -336,7 +339,7 @@ def pytest_configure(config):
         import u_boot_console_exec_attach
         console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig)
 
-re_ut_test_list = re.compile(r'[^a-zA-Z0-9_]_u_boot_list_2_ut_(.*)_test_2_(.*)\s*$')
+
 def generate_ut_subtest(metafunc, fixture_name, sym_path):
     """Provide parametrization for a ut_subtest fixture.
 
@@ -363,7 +366,7 @@ def generate_ut_subtest(metafunc, fixture_name, sym_path):
 
     vals = []
     for l in lines:
-        m = re_ut_test_list.search(l)
+        m = RE_UT_TEST_LIST.search(l)
         if not m:
             continue
         suite, name = m.groups()

test/py/tests/test_spl.py

@@ -36,7 +36,7 @@ def test_spl(u_boot_console, ut_spl_subtest):
         cons = u_boot_console
         cons.restart_uboot_with_flags(['-u', '-k', ut_spl_subtest.split()[1]])
         output = cons.get_spawn_output().replace('\r', '')
-        assert 'Failures: 0' in output
+        assert 'failures: 0' in output
     finally:
         # Restart afterward in case a non-SPL test is run next. This should not
         # happen since SPL tests are run in their own invocation of test.py, but

test/py/tests/test_suite.py

@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright 2024 Google LLC
+
+import pytest
+import re
+
+# List of test suites we expect to find with 'ut info' and 'ut all'
+EXPECTED_SUITES = [
+    'addrmap', 'bdinfo', 'bloblist', 'bootm', 'bootstd',
+    'cmd', 'common', 'dm', 'env', 'exit',
+    'fdt', 'font', 'hush', 'lib',
+    'loadm', 'log', 'mbr', 'measurement', 'mem',
+    'overlay', 'pci_mps', 'setexpr', 'upl',
+    ]
+
+
+# Set this to True to aid debugging of tests
+DEBUG_ME = False
+
+
+def collect_info(cons, output):
+    """Process the output from 'ut all'
+
+    Args:
+        cons: U-Boot console object
+        output: Output from running 'ut all'
+
+    Returns:
+        tuple:
+            set: suite names that were found in output
+            set: test names that were found in output
+            dict: test count for each suite:
+                key: suite name
+                value: number of tests for the suite found in output
+            set: missing suites (compared to EXPECTED_SUITES)
+            set: extra suites (compared to EXPECTED_SUITES)
+    """
+    suites = set()
+    tests = set()
+    cur_suite = None
+    test_count = None
+    exp_test_count = {}
+
+    # Collect suites{}
+    for line in output.splitlines():
+        line = line.rstrip()
+        if DEBUG_ME:
+            cons.log.info(f'line: {line}')
+        m = re.search('----Running ([^ ]*) tests----', line)
+        if m:
+            if DEBUG_ME and cur_suite and cur_suite != 'info':
+                cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}')
+
+            cur_suite = m.group(1)
+            if DEBUG_ME:
+                cons.log.info(f'cur_suite: {cur_suite}')
+            suites.add(cur_suite)
+
+            test_count = 0
+        m = re.match(rf'Running (\d+) {cur_suite} tests', line)
+        if m:
+            exp_test_count[cur_suite] = int(m.group(1))
+        m = re.search(r'Test: (\w*): ([-a-z0-9_]*\.c)?( .*)?', line)
+        if m:
+            test_name = m.group(1)
+            msg = m.group(3)
+            if DEBUG_ME:
+                cons.log.info(f"test_name {test_name} msg '{msg}'")
+            if msg == ' (flat tree)' and test_name not in tests:
+                tests.add(test_name)
+                test_count += 1
+            if not msg or 'skipped as it is manual' in msg:
+                tests.add(test_name)
+                test_count += 1
+        if DEBUG_ME:
+            cons.log.info(f'test_count {test_count}')
+    if DEBUG_ME:
+        cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}')
+        cons.log.info(f"Tests: {' '.join(sorted(list(tests)))}")
+
+    # Figure out what is missing, or extra
+    missing = set()
+    extra = set(suites)
+    for suite in EXPECTED_SUITES:
+        if suite in extra:
+            extra.remove(suite)
+        else:
+            missing.add(suite)
+
+    return suites, tests, exp_test_count, missing, extra
+
+
+def process_ut_info(cons, output):
+    """Process the output of the 'ut info' command
+
+    Args:
+        cons: U-Boot console object
+        output: Output from running 'ut all'
+
+    Returns:
+        tuple:
+            int: Number of suites reported
+            int: Number of tests reported
+            dict: test count for each suite:
+                key: suite name
+                value: number of tests reported for the suite
+
+    """
+    suite_count = None
+    total_test_count = None
+    test_count = {}
+    for line in output.splitlines():
+        line = line.rstrip()
+        if DEBUG_ME:
+            cons.log.info(f'line: {line}')
+        m = re.match(r'Test suites: (.*)', line)
+        if m:
+            suite_count = int(m.group(1))
+        m = re.match(r'Total tests: (.*)', line)
+        if m:
+            total_test_count = int(m.group(1))
+        m = re.match(r'  *([0-9?]*)  (\w*)', line)
+        if m:
+            test_count[m.group(2)] = m.group(1)
+    return suite_count, total_test_count, test_count
+
+
+@pytest.mark.buildconfigspec('sandbox')
+@pytest.mark.notbuildconfigspec('sandbox_spl')
+@pytest.mark.notbuildconfigspec('sandbox64')
+# This test is disabled since it fails; remove the leading 'x' to try it
+def xtest_suite(u_boot_console, u_boot_config):
+    """Perform various checks on the unit tests, including:
+
+       - The number of suites matches that reported by the 'ut info'
+       - Where available, the number of tests is each suite matches that
+         reported by 'ut info -s'
+       - The total number of tests adds up to the total that are actually run
+         with 'ut all'
+       - All suites are run with 'ut all'
+       - The expected set of suites is run (the list is hard-coded in this test)
+
+    """
+    cons = u_boot_console
+    buildconfig = u_boot_config.buildconfig
+    with cons.log.section('Run all unit tests'):
+        # ut hush hush_test_simple_dollar prints "Unknown command" on purpose.
+        with u_boot_console.disable_check('unknown_command'):
+            output = cons.run_command('ut all')
+
+    # Process the output from the run
+    with cons.log.section('Check output'):
+        suites, all_tests, exp_test_count, missing, extra = collect_info(cons,
+                                                                         output)
+    cons.log.info(f'missing {missing}')
+    cons.log.info(f'extra {extra}')
+
+    # Make sure we got a test count for each suite
+    assert not (suites - exp_test_count.keys())
+
+    # Deal with missing suites
+    with cons.log.section('Check missing suites'):
+        if 'config_cmd_seama' not in buildconfig:
+            cons.log.info("CMD_SEAMA not enabled: Ignoring suite 'seama'")
+            missing.discard('seama')
+
+    # Run 'ut info' and compare with the log results
+    with cons.log.section('Check suite test-counts'):
+        output = cons.run_command('ut info -s')
+
+        suite_count, total_test_count, test_count = process_ut_info(cons,
+                                                                    output)
+
+        if missing or extra:
+            cons.log.info(f"suites: {' '.join(sorted(list(suites)))}")
+            cons.log.error(f'missing: {sorted(list(missing))}')
+            cons.log.error(f'extra: {sorted(list(extra))}')
+
+        assert not missing, f'Missing suites {missing}'
+        assert not extra, f'Extra suites {extra}'
+
+        cons.log.info(str(exp_test_count))
+        for suite in EXPECTED_SUITES:
+            assert test_count[suite] in ['?', str(exp_test_count[suite])], \
+                f'suite {suite} expected {exp_test_count[suite]}'
+
+        assert suite_count == len(EXPECTED_SUITES)
+        assert total_test_count == len(all_tests)

test/py/tests/test_upl.py

@@ -35,4 +35,4 @@ def test_upl_handoff(u_boot_console):
 
     # Check the FIT offsets look correct
     output = cons.run_command('ut upl -f upl_test_info_norun')
-    assert 'Failures: 0' in output
+    assert 'failures: 0' in output

test/py/tests/test_ut.py

@@ -607,4 +607,4 @@ def test_ut(u_boot_console, ut_subtest):
         assert 'Unknown command \'quux\' - try \'help\'' in output
     else:
         output = u_boot_console.run_command('ut ' + ut_subtest)
-    assert output.endswith('Failures: 0')
+    assert output.endswith('failures: 0')

test/py/tests/test_vbe.py

@@ -117,4 +117,4 @@ def test_vbe(u_boot_console):
     with cons.log.section('Kernel load'):
         output = cons.run_command_list(cmd.splitlines())
 
-    assert 'Failures: 0' in output[-1]
+    assert 'failures: 0' in output[-1]

test/py/tests/test_vpl.py

@@ -26,7 +26,7 @@ def test_vpl(u_boot_console, ut_vpl_subtest):
         cons = u_boot_console
         cons.restart_uboot_with_flags(['-u', '-k', ut_vpl_subtest.split()[1]])
         output = cons.get_spawn_output().replace('\r', '')
-        assert 'Failures: 0' in output
+        assert 'failures: 0' in output
     finally:
         # Restart afterward in case a non-VPL test is run next. This should not
         # happen since VPL tests are run in their own invocation of test.py, but

test/test-main.c

@@ -63,16 +63,29 @@ static enum fdtchk_t fdt_action(void)
 /* This is valid when a test is running, NULL otherwise */
 static struct unit_test_state *cur_test_state;
 
-struct unit_test_state *test_get_state(void)
+struct unit_test_state *ut_get_state(void)
 {
 	return cur_test_state;
 }
 
-void test_set_state(struct unit_test_state *uts)
+void ut_set_state(struct unit_test_state *uts)
 {
 	cur_test_state = uts;
 }
 
+void ut_init_state(struct unit_test_state *uts)
+{
+	memset(uts, '\0', sizeof(*uts));
+}
+
+void ut_uninit_state(struct unit_test_state *uts)
+{
+	if (IS_ENABLED(CONFIG_SANDBOX)) {
+		os_free(uts->fdt_copy);
+		os_free(uts->other_fdt);
+	}
+}
+
 /**
  * dm_test_pre_run() - Get ready to run a driver model test
  *
@@ -435,7 +448,7 @@ static int test_post_run(struct unit_test_state *uts, struct unit_test *test)
  */
 static int skip_test(struct unit_test_state *uts)
 {
-	uts->skip_count++;
+	uts->cur.skip_count++;
 
 	return -EAGAIN;
 }
@@ -447,7 +460,7 @@ static int skip_test(struct unit_test_state *uts)
  * the name of each test before running it.
  *
  * @uts: Test state to update. The caller should ensure that this is zeroed for
- *	the first call to this function. On exit, @uts->fail_count is
+ *	the first call to this function. On exit, @uts->cur.fail_count is
  *	incremented by the number of failures (0, one hopes)
  * @test_name: Test to run
  * @name: Name of test, possibly skipping a prefix that should not be displayed
@@ -466,7 +479,7 @@ static int ut_run_test(struct unit_test_state *uts, struct unit_test *test,
 	printf("Test: %s: %s%s\n", test_name, fname, note);
 
 	/* Allow access to test state from drivers */
-	test_set_state(uts);
+	ut_set_state(uts);
 
 	ret = test_pre_run(uts, test);
 	if (ret == -EAGAIN)
@@ -482,7 +495,7 @@ static int ut_run_test(struct unit_test_state *uts, struct unit_test *test,
 	if (ret)
 		return ret;
 
-	test_set_state( NULL);
+	ut_set_state(NULL);
 
 	return 0;
 }
@@ -497,7 +510,7 @@ static int ut_run_test(struct unit_test_state *uts, struct unit_test *test,
  * SPL.
  *
  * @uts: Test state to update. The caller should ensure that this is zeroed for
- *	the first call to this function. On exit, @uts->fail_count is
+ *	the first call to this function. On exit, @uts->cur.fail_count is
  *	incremented by the number of failures (0, one hopes)
  * @test: Test to run
  * Return: 0 if all tests passed, -EAGAIN if the test should be skipped, -1 if
@@ -561,7 +574,7 @@ static int ut_run_test_live_flat(struct unit_test_state *uts,
  * the name of each test before running it.
  *
  * @uts: Test state to update. The caller should ensure that this is zeroed for
- *	the first call to this function. On exit, @uts->fail_count is
+ *	the first call to this function. On exit, @uts->cur.fail_count is
  *	incremented by the number of failures (0, one hopes)
  * @prefix: String prefix for the tests. Any tests that have this prefix will be
  *	printed without the prefix, so that it is easier to see the unique part
@@ -619,34 +632,34 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix,
 			if (len < 6 || strcmp(test_name + len - 6, "_norun")) {
 				printf("Test '%s' is manual so must have a name ending in _norun\n",
 				       test_name);
-				uts->fail_count++;
+				uts->cur.fail_count++;
 				return -EBADF;
 			}
 			if (!uts->force_run) {
-				if (select_name) {
+				printf("Test: %s: skipped as it is manual (use -f to run it)\n",
-					printf("Test '%s' skipped as it is manual (use -f to run it)\n",
+				       test_name);
-					       test_name);
-				}
 				continue;
 			}
 		}
-		old_fail_count = uts->fail_count;
+		old_fail_count = uts->cur.fail_count;
 
+		uts->cur.test_count++;
 		if (one && upto == pos) {
 			ret = ut_run_test_live_flat(uts, one);
-			if (uts->fail_count != old_fail_count) {
+			if (uts->cur.fail_count != old_fail_count) {
 				printf("Test '%s' failed %d times (position %d)\n",
 				       one->name,
-				       uts->fail_count - old_fail_count, pos);
+				       uts->cur.fail_count - old_fail_count,
+				       pos);
 			}
 			return -EBADF;
 		}
 
 		for (i = 0; i < uts->runs_per_test; i++)
 			ret = ut_run_test_live_flat(uts, test);
-		if (uts->fail_count != old_fail_count) {
+		if (uts->cur.fail_count != old_fail_count) {
 			printf("Test '%s' failed %d times\n", test_name,
-			       uts->fail_count - old_fail_count);
+			       uts->cur.fail_count - old_fail_count);
 		}
 		found++;
 		if (ret == -EAGAIN)
@@ -657,17 +670,32 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix,
 	if (select_name && !found)
 		return -ENOENT;
 
-	return uts->fail_count ? -EBADF : 0;
+	return uts->cur.fail_count ? -EBADF : 0;
 }
 
-int ut_run_list(const char *category, const char *prefix,
+void ut_report(struct ut_stats *stats, int run_count)
-		struct unit_test *tests, int count, const char *select_name,
-		int runs_per_test, bool force_run, const char *test_insert)
 {
-	struct unit_test_state uts = { .fail_count = 0 };
+	if (run_count > 1)
+		printf("Suites run: %d, total tests", run_count);
+	else
+		printf("Tests");
+	printf(" run: %d, ", stats->test_count);
+	if (stats->skip_count)
+		printf("skipped: %d, ", stats->skip_count);
+	printf("failures: %d\n", stats->fail_count);
+}
+
+int ut_run_list(struct unit_test_state *uts, const char *category,
+		const char *prefix, struct unit_test *tests, int count,
+		const char *select_name, int runs_per_test, bool force_run,
+		const char *test_insert)
+{
+	;
 	bool has_dm_tests = false;
 	int ret;
 
+	memset(&uts->cur, '\0', sizeof(struct ut_stats));
+
 	if (!CONFIG_IS_ENABLED(OF_PLATDATA) &&
 	    ut_list_has_dm_tests(tests, count, prefix, select_name)) {
 		has_dm_tests = true;
@@ -685,35 +713,33 @@ int ut_run_list(const char *category, const char *prefix,
 	if (!select_name)
 		printf("Running %d %s tests\n", count, category);
 
-	uts.of_root = gd_of_root();
+	uts->of_root = gd_of_root();
-	uts.runs_per_test = runs_per_test;
+	uts->runs_per_test = runs_per_test;
 	if (fdt_action() == FDTCHK_COPY && gd->fdt_blob) {
-		uts.fdt_size = fdt_totalsize(gd->fdt_blob);
+		uts->fdt_size = fdt_totalsize(gd->fdt_blob);
-		uts.fdt_copy = os_malloc(uts.fdt_size);
+		uts->fdt_copy = os_malloc(uts->fdt_size);
-		if (!uts.fdt_copy) {
+		if (!uts->fdt_copy) {
 			printf("Out of memory for device tree copy\n");
 			return -ENOMEM;
 		}
-		memcpy(uts.fdt_copy, gd->fdt_blob, uts.fdt_size);
+		memcpy(uts->fdt_copy, gd->fdt_blob, uts->fdt_size);
 	}
-	uts.force_run = force_run;
+	uts->force_run = force_run;
-	ret = ut_run_tests(&uts, prefix, tests, count, select_name,
+	ret = ut_run_tests(uts, prefix, tests, count, select_name,
 			   test_insert);
 
 	/* Best efforts only...ignore errors */
 	if (has_dm_tests)
-		dm_test_restore(uts.of_root);
+		dm_test_restore(uts->of_root);
-	if (IS_ENABLED(CONFIG_SANDBOX)) {
-		os_free(uts.fdt_copy);
-		os_free(uts.other_fdt);
-	}
 
-	if (uts.skip_count)
+	ut_report(&uts->cur, 1);
-		printf("Skipped: %d, ", uts.skip_count);
 	if (ret == -ENOENT)
 		printf("Test '%s' not found\n", select_name);
-	else
+
-		printf("Failures: %d\n", uts.fail_count);
+	uts->total.skip_count += uts->cur.skip_count;
+	uts->total.fail_count += uts->cur.fail_count;
+	uts->total.test_count += uts->cur.test_count;
+	uts->run_count++;
 
 	return ret;
 }

test/ut.c

@@ -21,7 +21,7 @@ void ut_fail(struct unit_test_state *uts, const char *fname, int line,
 {
 	gd->flags &= ~(GD_FLG_SILENT | GD_FLG_RECORD);
 	printf("%s:%d, %s(): %s\n", fname, line, func, cond);
-	uts->fail_count++;
+	uts->cur.fail_count++;
 }
 
 void ut_failf(struct unit_test_state *uts, const char *fname, int line,
@@ -35,7 +35,7 @@ void ut_failf(struct unit_test_state *uts, const char *fname, int line,
 	vprintf(fmt, args);
 	va_end(args);
 	putc('\n');
-	uts->fail_count++;
+	uts->cur.fail_count++;
 }
 
 ulong ut_check_free(void)