Parallel Testing

Author: avik-pal

.buildkite/pipeline.yml

@@ -4,15 +4,15 @@ steps:
       - JuliaCI/julia#v1:
           version: "1"
       - JuliaCI/julia-test#v1:
-           coverage: false # 1000x slowdown
+           coverage: true
     agents:
       queue: "juliagpu"
       cuda: "*"
-    timeout_in_minutes: 30
+    timeout_in_minutes: 60
     # Don't run Buildkite if the commit message includes the text [skip tests]
     if: build.message !~ /\[skip tests\]/
 
 env:
   GROUP: GPU
   JULIA_PKG_SERVER: "" # it often struggles with our large artifacts
-  # SECRET_CODECOV_TOKEN: "..."
+  SECRET_CODECOV_TOKEN: "HC7K/ymhi62KUQ5OLU4DOl+11gaQt4JhXX/2nfTGlTsBB8mEMxQ8R+sHIp/2HjEup5eSXAN2IWQDQ7RDBuQvVp0T1UVtr2e4YNZFztKnsJXrFO15hXxYShJodI//X/8DzhlQd/lyTDOAOJu3eznsc3sC2CUgJzXZxLUtQN9YaZ1i3a+NoN1mO5UpkkHVhXigwF5gjy+0tei8fCdcP+SIhG0EanS5yd9q/SurtCpMHsHyUG97+ZVPglSKgdaqr31+PdmiPJ+ynp4+Hnc/esosxUSHSIL+ryRTO+28RNwPTiNf99J51RJLQmz1knWTR1ky6tiYIZ5218O6wvNil0SqNw==;U2FsdGVkX18nBY3t4LZYlEIz3EVKjpqCd994JNeJGt006up+sAjXEssI0tgCVXnfXsenVsP3NCCEoOS1GXc44g=="

.github/workflows/CI.yml

@@ -18,13 +18,9 @@ jobs:
       fail-fast: false
       matrix:
         group:
-          - RootFinding
-          - NLLSSolvers
-          - 23TestProblems
-          - Wrappers
-          - Miscellaneous
+          - Core
         version:
-          - '1.10'
+          - '1'
         os:
           - ubuntu-latest
           - macos-latest
@@ -48,7 +44,7 @@ jobs:
       - uses: julia-actions/julia-runtest@v1
         env:
           GROUP: ${{ matrix.group }}
-          JULIA_NUM_THREADS: 11
+          JULIA_NUM_THREADS: 8
       - uses: julia-actions/julia-processcoverage@v1
         with:
           directories: src,ext

Project.toml

@@ -101,6 +101,7 @@ Symbolics = "5.13"
 Test = "1.10"
 TimerOutputs = "0.5.23"
 Zygote = "0.6.67"
+XUnit = "1.1"
 julia = "1.10"
 
 [extras]
@@ -131,7 +132,8 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Sundials = "c3572dad-4567-51f8-b174-8c6c989267f4"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+XUnit = "3e3c03f2-1a94-11e9-2981-050a4ca824ab"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "Enzyme", "BenchmarkTools", "SafeTestsets", "Pkg", "Test", "ForwardDiff", "StaticArrays", "Symbolics", "LinearSolve", "Random", "LinearAlgebra", "Zygote", "SparseDiffTools", "NonlinearProblemLibrary", "LeastSquaresOptim", "FastLevenbergMarquardt", "NaNMath", "BandedMatrices", "DiffEqBase", "StableRNGs", "MINPACK", "NLsolve", "OrdinaryDiffEq", "SpeedMapping", "FixedPointAcceleration", "SIAMFANLEquations", "Sundials"]
+test = ["Aqua", "Enzyme", "BenchmarkTools", "SafeTestsets", "Pkg", "Test", "ForwardDiff", "StaticArrays", "Symbolics", "LinearSolve", "Random", "LinearAlgebra", "Zygote", "SparseDiffTools", "NonlinearProblemLibrary", "LeastSquaresOptim", "FastLevenbergMarquardt", "NaNMath", "BandedMatrices", "DiffEqBase", "StableRNGs", "MINPACK", "NLsolve", "OrdinaryDiffEq", "SpeedMapping", "FixedPointAcceleration", "SIAMFANLEquations", "Sundials", "XUnit"]

test/core/23_test_problems.jl

@@ -1,4 +1,4 @@
-using NonlinearSolve, LinearAlgebra, LinearSolve, NonlinearProblemLibrary, Test
+using NonlinearSolve, LinearAlgebra, LinearSolve, NonlinearProblemLibrary, XUnit
 
 problems = NonlinearProblemLibrary.problems
 dicts = NonlinearProblemLibrary.dicts
@@ -36,7 +36,7 @@ function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
     end
 end
 
-@testset "NewtonRaphson 23 Test Problems" begin
+@testcase "NewtonRaphson 23 Test Problems" begin
     alg_ops = (NewtonRaphson(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -45,26 +45,26 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-# @testset "TrustRegion 23 Test Problems" begin
-#     alg_ops = (TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Simple),
-#         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Fan),
-#         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Hei),
-#         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Yuan),
-#         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Bastin),
-#         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.NLsolve))
-
-#     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-#     broken_tests[alg_ops[1]] = [11, 21]
-#     broken_tests[alg_ops[2]] = [11, 21]
-#     broken_tests[alg_ops[3]] = [11, 21]
-#     broken_tests[alg_ops[4]] = [11, 21]
-#     broken_tests[alg_ops[5]] = [21]
-#     broken_tests[alg_ops[6]] = [21]
-
-#     test_on_library(problems, dicts, alg_ops, broken_tests)
-# end
-
-@testset "LevenbergMarquardt 23 Test Problems" begin
+@testcase "TrustRegion 23 Test Problems" begin
+    alg_ops = (TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Simple),
+        TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Fan),
+        TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Hei),
+        TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Yuan),
+        TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Bastin),
+        TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.NLsolve))
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [11, 21]
+    broken_tests[alg_ops[2]] = [11, 21]
+    broken_tests[alg_ops[3]] = [11, 21]
+    broken_tests[alg_ops[4]] = [8, 11, 21]
+    broken_tests[alg_ops[5]] = [21]
+    broken_tests[alg_ops[6]] = [11, 21]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testcase "LevenbergMarquardt 23 Test Problems" begin
     alg_ops = (LevenbergMarquardt(),
         LevenbergMarquardt(; α_geodesic = 0.1),
         LevenbergMarquardt(; linsolve = CholeskyFactorization()))
@@ -77,7 +77,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testset "DFSane 23 Test Problems" begin
+@testcase "DFSane 23 Test Problems" begin
     alg_ops = (DFSane(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -86,7 +86,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testset "Broyden 23 Test Problems" begin
+@testcase "Broyden 23 Test Problems" begin
     alg_ops = (Broyden(),
         Broyden(; init_jacobian = Val(:true_jacobian)),
         Broyden(; update_rule = Val(:bad_broyden)),
@@ -101,7 +101,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testset "Klement 23 Test Problems" begin
+@testcase "Klement 23 Test Problems" begin
     alg_ops = (Klement(), Klement(; init_jacobian = Val(:true_jacobian_diagonal)))
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -111,7 +111,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testset "PseudoTransient 23 Test Problems" begin
+@testcase "PseudoTransient 23 Test Problems" begin
     # PT relies on the root being a stable equilibrium for convergence, so it won't work on
     # most problems
     alg_ops = (PseudoTransient(),)

test/core/forward_ad.jl

@@ -1,5 +1,5 @@
 using ForwardDiff,
-    NonlinearSolve, MINPACK, NLsolve, StaticArrays, Sundials, Test, LinearAlgebra
+    NonlinearSolve, MINPACK, NLsolve, StaticArrays, Sundials, XUnit, LinearAlgebra
 
 test_f!(du, u, p) = (@. du = u^2 - p)
 test_f(u, p) = (@. u^2 - p)
@@ -58,9 +58,10 @@ __compatible(::NLsolveJL, ::Val{:oop_cache}) = false
 __compatible(::KINSOL, ::Val{:iip_cache}) = false
 __compatible(::KINSOL, ::Val{:oop_cache}) = false
 
-@testset "ForwardDiff.jl Integration: $(alg)" for alg in (NewtonRaphson(), TrustRegion(),
+@testcase "ForwardDiff.jl Integration: $(alg)" for alg in (NewtonRaphson(), TrustRegion(),
     LevenbergMarquardt(), PseudoTransient(; alpha_initial = 10.0), Broyden(), Klement(),
-    DFSane(), nothing, NLsolveJL(), CMINPACK(), KINSOL())
+    DFSane(), nothing, NLsolveJL(), CMINPACK(),
+    KINSOL(; globalization_strategy = :LineSearch))
     us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
 
     @testset "Scalar AD" begin

test/core/nlls.jl

@@ -1,5 +1,5 @@
 using NonlinearSolve,
-    LinearSolve, LinearAlgebra, Test, StableRNGs, Random, ForwardDiff, Zygote
+    LinearSolve, LinearAlgebra, XUnit, StableRNGs, Random, ForwardDiff, Zygote
 
 true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
 true_function(y, x, θ) = (@. y = θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4]))
@@ -52,10 +52,12 @@ for radius_update_scheme in [RadiusUpdateSchemes.Simple, RadiusUpdateSchemes.Noc
     push!(solvers, TrustRegion(; radius_update_scheme))
 end
 
-for prob in nlls_problems, solver in solvers
-    @time sol = solve(prob, solver; maxiters = 10000, abstol = 1e-8)
-    @test SciMLBase.successful_retcode(sol)
-    @test maximum(abs, sol.resid) < 1e-6
+@testcase "General NLLS Solvers" begin
+    for prob in nlls_problems, solver in solvers
+        sol = solve(prob, solver; maxiters = 10000, abstol = 1e-8)
+        @test SciMLBase.successful_retcode(sol)
+        @test maximum(abs, sol.resid) < 1e-6
+    end
 end
 
 # This is just for testing that we can use vjp provided by the user
@@ -79,7 +81,9 @@ probs = [
             resid_prototype = zero(y_target), vjp = vjp), θ_init, x),
 ]
 
-for prob in probs, solver in solvers
-    sol = solve(prob, solver; maxiters = 10000, abstol = 1e-8)
-    @test maximum(abs, sol.resid) < 1e-6
+@testcase "Custom VJP" begin
+    for prob in probs, solver in solvers
+        sol = solve(prob, solver; maxiters = 10000, abstol = 1e-8)
+        @test maximum(abs, sol.resid) < 1e-6
+    end
 end