Skip to content

Commit 65e5703

Browse files
Googlera-maurice
authored andcommitted
Re-write run_local_tests.sh to run the tests in parallel.
When I profiled this change locally, the runtime of the script was reduced from about 30 minutes down to 12 minutes when the blaze commands were run in parallel. This makes sense because a lot of the time is spent just waiting for Kokoro to perform the build. PiperOrigin-RevId: 330748434
1 parent 5889e73 commit 65e5703

File tree

1 file changed

+94
-4
lines changed

1 file changed

+94
-4
lines changed

firestore/run_local_tests.sh

Lines changed: 94 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,99 @@
33
# TAP or Guitar at the moment (see b/135205911). Execute this script before
44
# submitting.
55

6+
set -euo pipefail
7+
8+
# Executes blaze to run a Kokoro test.
9+
#
10+
# The first argument is the name of an associative array variable. A key/value
11+
# pair will be inserted into this associative array with information about the
12+
# launched blaze process. The key will be the PID of the blaze process and the
13+
# value will be the full blaze command that was executed.
14+
#
15+
# The second argument is the "name" of the test, and is used in the subdirectory
16+
# name of the directory specified to blaze's --output_base argument.
17+
#
18+
# The remaining arguments, if any, will be specified to the blaze command after
19+
# the "test" argument.
20+
function run_blaze {
21+
if [[ $# -lt 2 ]] ; then
22+
echo "INTERNAL ERROR: run_blaze invalid arguments: $*" >&2
23+
exit 1
24+
fi
25+
26+
local -n readonly blaze_process_map="$1"
27+
local readonly blaze_build_name="$2"
28+
shift 2
29+
30+
local readonly blaze_args=(
31+
"blaze"
32+
"--blazerc=/dev/null"
33+
"--output_base=/tmp/run_local_tests_blaze_output_bases/${blaze_build_name}"
34+
"test"
35+
"$@"
36+
"//firebase/firestore/client/cpp:kokoro_build_test"
37+
)
38+
39+
echo "${blaze_args[*]}"
40+
"${blaze_args[@]}" &
41+
blaze_process_map["$!"]="${blaze_args[*]}"
42+
}
43+
44+
# Waits for blaze commands started by `run_blaze` to complete.
45+
#
46+
# If all blaze commands complete successfully then a "success" message is
47+
# printed; otherwise, if one or more of the blaze commands fail, then an error
48+
# message is printed.
49+
#
50+
# The first (and only) argument is the name of an associative array variable.
51+
# This variable should be the same that was was specified to `run_blaze` and
52+
# specifies the blaze commands for which to wait.
53+
#
54+
# The return value is 0 if all blaze processes completed successfully or 1 if
55+
# one or more of the blaze processes failed.
56+
function wait_for_blazes_to_complete {
57+
if [[ $# -ne 1 ]] ; then
58+
echo "INTERNAL ERROR: wait_for_blazes_to_complete invalid arguments: $*" >&2
59+
exit 1
60+
fi
61+
62+
local -n blaze_process_map="$1"
63+
64+
local blaze_pid
65+
local -a blaze_failed_pids=()
66+
67+
for blaze_pid in "${!blaze_process_map[@]}" ; do
68+
if ! wait "${blaze_pid}" ; then
69+
blaze_failed_pids+=("${blaze_pid}")
70+
fi
71+
done
72+
73+
local readonly num_failed_blazes="${#blaze_failed_pids[@]}"
74+
if [[ ${num_failed_blazes} -eq 0 ]] ; then
75+
echo "All blaze commands completed successfully."
76+
else
77+
echo "ERROR: The following ${num_failed_blazes} blaze invocation(s) failed:"
78+
for blaze_pid in "${blaze_failed_pids[@]}" ; do
79+
echo "${blaze_process_map[${blaze_pid}]}"
80+
done
81+
echo "Go to http://sponge2 to see the results."
82+
return 1
83+
fi
84+
}
85+
86+
# The main program begins here.
87+
if [[ $# -gt 0 ]] ; then
88+
echo "ERROR: $0 does not accept any arguments, but got: $*" >&2
89+
exit 2
90+
fi
91+
92+
declare -A BLAZE_PROCESS_MAP
93+
694
# LINT.IfChange
7-
blaze test --config=android_arm //firebase/firestore/client/cpp:kokoro_build_test && \
8-
blaze test --config=darwin_x86_64 //firebase/firestore/client/cpp:kokoro_build_test && \
9-
blaze test --define=force_regular_grpc=1 //firebase/firestore/client/cpp:kokoro_build_test && \
10-
blaze test --config=msvc //firebase/firestore/client/cpp:kokoro_build_test
95+
run_blaze "BLAZE_PROCESS_MAP" "android_arm" "--config=android_arm"
96+
run_blaze "BLAZE_PROCESS_MAP" "darwin_x86_64" "--config=darwin_x86_64"
97+
run_blaze "BLAZE_PROCESS_MAP" "linux" "--define=force_regular_grpc=1"
98+
run_blaze "BLAZE_PROCESS_MAP" "msvc" "--config=msvc"
1199
# LINT.ThenChange(//depot_firebase_cpp/firestore/client/cpp/METADATA)
100+
101+
wait_for_blazes_to_complete "BLAZE_PROCESS_MAP"

0 commit comments

Comments
 (0)