From 9109d6a4b4405b4c27103ee6a0ceebb6d7605838 Mon Sep 17 00:00:00 2001
From: Quentin Monnet <quentin@isovalent.com>
Date: Tue, 16 Nov 2021 10:34:58 +0000
Subject: [PATCH] ci: create summary for tests and account for bpftool checks
 result

The bpftool checks work as expected when the CI runs, except that they
do not set any error status code for the script on error, which means
that the failures are lost among the logs and not reported in a clear
way to the reviewers.

This commit aims at fixing the issue. We could simply exit with a
non-zero error status when the bpftool checks, but that would prevent
the other tests from running. Instead, we propose to store the result of
the bpftool checks in a bash array. This array can later be reused to
print a summary of the different groups of tests, at the end of the CI
run, to help the reviewers understand where the failure happened without
having to manually unfold all the sections on the GitHub interface.

Currently, there are only two groups: the bpftool checks and the "VM
tests". The latter may later be split into test_maps, test_progs,
test_progs-no_alu32, etc. by teaching each of them to append their exit
status code to the "exitstatus" file.

Fixes: 88649fe655b8 ("ci: run script to test bpftool types/options sync")
---
 travis-ci/vmtest/run.sh | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/travis-ci/vmtest/run.sh b/travis-ci/vmtest/run.sh
index 571ce36..93a1f90 100755
--- a/travis-ci/vmtest/run.sh
+++ b/travis-ci/vmtest/run.sh
@@ -410,10 +410,20 @@ LIBBPF_PATH="${REPO_ROOT}" \
 	REPO_PATH="${REPO_PATH}" \
 	VMLINUX_BTF=$(realpath ${source_vmlinux}) ${VMTEST_ROOT}/build_selftests.sh
 
+declare -A test_results
+
 travis_fold start bpftool_checks "Running bpftool checks..."
 if [[ "${KERNEL}" = 'LATEST' ]]; then
-	"${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf/test_bpftool_synctypes.py" && \
-		echo "Consistency checks passed successfully."
+	# "&& true" does not change the return code (it is not executed if the
+	# Python script fails), but it prevents the trap on ERR set at the top
+	# of this file to trigger on failure.
+	"${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf/test_bpftool_synctypes.py" && true
+	test_results["bpftool"]=$?
+	if [[ ${test_results["bpftool"]} -eq 0 ]]; then
+		echo "::notice title=bpftool_checks::bpftool checks passed successfully."
+	else
+		echo "::error title=bpftool_checks::bpftool checks returned ${test_results["bpftool"]}."
+	fi
 else
 	echo "Consistency checks skipped."
 fi
@@ -532,4 +542,21 @@ fi
 
 travis_fold end shutdown
 
+test_results["vm_tests"]=$exitstatus
+
+# Final summary - Don't use a fold, keep it visible
+echo -e "\033[1;33mTest Results:\033[0m"
+for testgroup in ${!test_results[@]}; do
+	# Print final result for each group of tests
+	if [[ ${test_results[$testgroup]} -eq 0 ]]; then
+		printf "%20s: \033[1;32mPASS\033[0m\n" $testgroup
+	else
+		printf "%20s: \033[1;31mFAIL\033[0m\n" $testgroup
+	fi
+	# Make exitstatus > 0 if at least one test group has failed
+	if [[ ${test_results[$testgroup]} -ne 0 ]]; then
+		exitstatus=1
+	fi
+done
+
 exit "$exitstatus"