Merge pull request #19 from TensorBFS/mrv/test-benchmark

Run benchmarks in CI and push results to TensorBFS/TensorInference-data

Author: GiggleLiu

.github/workflows/benchmark.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Install dependencies
         run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"'
       - name: Run benchmarks
-        run: julia -e 'using BenchmarkCI; BenchmarkCI.judge(baseline="origin/main")'
+        run: julia -e 'using BenchmarkCI; BenchmarkCI.judge(baseline = "origin/main")'
       - name: Post results
         run: julia -e 'using BenchmarkCI; BenchmarkCI.postjudge()'
         env:
@@ -29,4 +29,4 @@ jobs:
           '
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SSH_KEY: ${{ secrets.DATA_SSH_KEY }}
+          SSH_KEY: ${{ secrets.DOCUMENTER_KEY }}

benchmark/Artifacts.toml

@@ -0,0 +1,34 @@
+[MAR_prob]
+git-tree-sha1 = "4dd2447ab2b8eeb8fbfc35d0c7416b5f01ff9711"
+
+    [[MAR_prob.download]]
+    sha256 = "857f8b6706dad46698c6e90a1804e1b40633d87f6eb3d82b5f29929bb6f5155a"
+    url = "https://personal.utdallas.edu/~vibhav.gogate/uai14-competition/files/MAR_prob.tar.gz"
+
+[MAR_sol]
+git-tree-sha1 = "8a86826145efa6b7d7d9c837acd80e33d1368f37"
+
+    [[MAR_sol.download]]
+    sha256 = "d8616ebfc1ab538c2957919f8bb47bd9c1fb86392dd025585cde041bc4e0e934"
+    url = "https://personal.utdallas.edu/~vibhav.gogate/uai14-competition/files/MAR_sol.tar.gz"
+
+[MMAP_UAI2014]
+git-tree-sha1 = "869778bbf5214cbff2169ecc2f3e67e2cb1b6a7a"
+
+    [[MMAP_UAI2014.download]]
+    sha256 = "0080c4851e23c17a42262cdaf4cb1a7aa4b80674435756d0d1ade76d80b9f33f"
+    url = "https://personal.utdallas.edu/~vibhav.gogate/uai14-competition/files/MMAP_UAI2014.tar.gz"
+
+[PR_prob]
+git-tree-sha1 = "a9baff14cc5f64a5d95920928581e2b4eb82acd3"
+
+    [[PR_prob.download]]
+    sha256 = "20b38d43a0718ab555988f041069380a30bbb68881e60d07ebf004054eb9f14f"
+    url = "https://personal.utdallas.edu/~vibhav.gogate/uai14-competition/files/PR_prob.tar.gz"
+
+[PR_sol]
+git-tree-sha1 = "e6643155c4ee433d20ddb7e58441b33b35ecacb8"
+
+    [[PR_sol.download]]
+    sha256 = "86b37c0e7c83f6ff162c88d48250e0bf2b7dd73d44441074ae93a81986238436"
+    url = "https://personal.utdallas.edu/~vibhav.gogate/uai14-competition/files/PR_sol.tar.gz"

benchmark/Project.toml

@@ -1,4 +1,5 @@
 [deps]
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 TensorInference = "c2297e78-99bd-40ad-871d-f50e56b81012"

benchmark/bench_map.jl

@@ -0,0 +1,16 @@
+module BenchMap
+
+using BenchmarkTools
+using TensorInference
+using Artifacts
+
+const SUITE = BenchmarkGroup()
+
+problem = read_uai_problem("Promedus_14")
+
+optimizer = TreeSA(ntrials = 1, niters = 2, βs = 1:0.1:40)
+tn = TensorNetworkModel(problem; optimizer)
+SUITE["map"] = @benchmarkable most_probable_config(tn)
+
+end  # module
+BenchMap.SUITE

benchmark/bench_mar.jl

@@ -0,0 +1,22 @@
+module BenchMar
+
+using BenchmarkTools
+using TensorInference
+using Artifacts
+# using CUDA
+# CUDA.allowscalar(false)
+
+const SUITE = BenchmarkGroup()
+
+problem = read_uai_problem("Promedus_14")
+
+optimizer = TreeSA(ntrials = 1, niters = 5, βs = 0.1:0.1:100)
+tn1 = TensorNetworkModel(problem; optimizer)
+SUITE["mar"] = @benchmarkable marginals(tn1)
+
+# optimizer = TreeSA(ntrials = 1, niters = 2, βs = 1:0.1:40)
+# tn2 = TensorNetworkModel(problem; optimizer)
+# SUITE["mar-cuda"] = @benchmarkable marginals(tn2; usecuda = true)
+
+end  # module
+BenchMar.SUITE

benchmark/bench_marginals.jl

@@ -0,0 +1,25 @@
+module BenchMmap
+
+using BenchmarkTools
+using TensorInference
+using Artifacts
+
+const SUITE = BenchmarkGroup()
+
+problem = read_uai_problem("Promedus_14")
+optimizer = TreeSA(ntrials = 1, niters = 2, βs = 1:0.1:40)
+
+# Does not marginalize any var
+mmap1 = MMAPModel(problem; marginalizedvertices = Int[], optimizer)
+SUITE["mmap-1"] = @benchmarkable maximum_logp(mmap1)
+
+# Marginalizes all vars
+mmap2 = MMAPModel(problem; marginalizedvertices = collect(1:(problem.nvars)), optimizer)
+SUITE["mmap-2"] = @benchmarkable maximum_logp(mmap2)
+
+# Does not optimize over open vertices
+mmap3 = MMAPModel(problem; marginalizedvertices = [2, 4, 6], optimizer)
+SUITE["mmap-3"] = @benchmarkable most_probable_config(mmap3)
+
+end  # module
+BenchMmap.SUITE