From 8f1996cad63b5694252dedebfb5ea4254d27b784 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Tue, 23 Dec 2025 17:25:28 +0100 Subject: [PATCH 1/5] pull in JSON and HTML formatters, set them up correctly --- bench/scripts/macro/all_bench.exs | 6 +----- bench/scripts/macro/insert_bench.exs | 6 +----- bench/scripts/micro/load_bench.exs | 6 +----- bench/scripts/micro/to_sql_bench.exs | 6 +----- bench/support/helper.exs | 13 +++++++++++++ bench/support/setup.exs | 4 ++++ mix.exs | 6 ++++-- mix.lock | 2 ++ 8 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 bench/support/helper.exs diff --git a/bench/scripts/macro/all_bench.exs b/bench/scripts/macro/all_bench.exs index 7de9afb4..29ed6059 100644 --- a/bench/scripts/macro/all_bench.exs +++ b/bench/scripts/macro/all_bench.exs @@ -35,13 +35,9 @@ jobs = %{ "MyXQL Repo.all/2" => fn -> Ecto.Bench.MyXQLRepo.all(User, limit: limit) end } -path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results" -file = Path.join(path, "all.json") - Benchee.run( jobs, - formatters: [Benchee.Formatters.Console], - formatter_options: [json: [file: file]], + formatters: Ecto.Bench.Helper.formatters("all"), time: 10, after_each: fn results -> ^limit = length(results) diff --git a/bench/scripts/macro/insert_bench.exs b/bench/scripts/macro/insert_bench.exs index fbb6f12c..d4b0fbe8 100644 --- a/bench/scripts/macro/insert_bench.exs +++ b/bench/scripts/macro/insert_bench.exs @@ -29,14 +29,10 @@ jobs = %{ "MyXQL Insert" => fn entry -> Ecto.Bench.MyXQLRepo.insert!(entry) end } -path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results" -file = Path.join(path, "insert.json") - Benchee.run( jobs, inputs: inputs, - formatters: [Benchee.Formatters.Console], - formatter_options: [json: [file: file]] + formatters: Ecto.Bench.Helper.formatters("insert") ) # Clean inserted data diff --git a/bench/scripts/micro/load_bench.exs b/bench/scripts/micro/load_bench.exs index 7ef3dfb2..94a8296e 100644 --- a/bench/scripts/micro/load_bench.exs +++ b/bench/scripts/micro/load_bench.exs @@ -44,12 +44,8 @@ jobs = %{ "MyXQL Loader" => fn data -> Enum.map(data, &Ecto.Bench.MyXQLRepo.load(User, &1)) end } -path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results" -file = Path.join(path, "load.json") - Benchee.run( jobs, inputs: inputs, - formatters: [Benchee.Formatters.Console], - formatter_options: [json: [file: file]] + formatters: Ecto.Bench.Helper.formatters("load") ) diff --git a/bench/scripts/micro/to_sql_bench.exs b/bench/scripts/micro/to_sql_bench.exs index 20b115b0..f841ed38 100644 --- a/bench/scripts/micro/to_sql_bench.exs +++ b/bench/scripts/micro/to_sql_bench.exs @@ -53,12 +53,8 @@ jobs = %{ "MyXQL Query Builder" => fn {type, query} -> Ecto.Bench.MyXQLRepo.to_sql(type, query) end } -path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results" -file = Path.join(path, "to_sql.json") - Benchee.run( jobs, inputs: inputs, - formatters: [Benchee.Formatters.Console], - formatter_options: [json: [file: file]] + formatters: Ecto.Bench.Helper.formatters("to_sql") ) diff --git a/bench/support/helper.exs b/bench/support/helper.exs new file mode 100644 index 00000000..d222a50f --- /dev/null +++ b/bench/support/helper.exs @@ -0,0 +1,13 @@ +defmodule Ecto.Bench.Helper do + def report_dir, do: System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results" + + def formatters(test_name) do + timestamp = DateTime.now!("Etc/UTC") |> DateTime.to_unix(:second) + file = Path.join(report_dir(), "#{timestamp}_#{test_name}") + [ + Benchee.Formatters.Console, + {Benchee.Formatters.JSON, file: "#{file}.json"}, + {Benchee.Formatters.HTML, file: "#{file}.html", auto_open: false} + ] + end +end diff --git a/bench/support/setup.exs b/bench/support/setup.exs index a01c75d2..3ea40ec1 100644 --- a/bench/support/setup.exs +++ b/bench/support/setup.exs @@ -1,8 +1,12 @@ Code.require_file("repo.exs", __DIR__) Code.require_file("migrations.exs", __DIR__) Code.require_file("schemas.exs", __DIR__) +Code.require_file("helper.exs", __DIR__) + +File.mkdir_p(Ecto.Bench.Helper.report_dir()) alias Ecto.Bench.{PgRepo, MyXQLRepo, CreateUser} +alias Ecto.Bench.{PgRepo, CreateUser} {:ok, _} = Ecto.Adapters.Postgres.ensure_all_started(PgRepo.config(), :temporary) {:ok, _} = Ecto.Adapters.MyXQL.ensure_all_started(MyXQLRepo.config(), :temporary) diff --git a/mix.exs b/mix.exs index 93487adb..0013472e 100644 --- a/mix.exs +++ b/mix.exs @@ -62,13 +62,15 @@ defmodule EctoSQL.MixProject do tds_dep(), # Bring something in for JSON during tests - {:jason, ">= 0.0.0", only: [:test, :docs]}, + {:jason, "~> 1.0", only: [:test, :bench, :docs]}, # Docs {:ex_doc, "~> 0.21", only: :docs}, # Benchmarks - {:benchee, "~> 1.0", only: :bench} + {:benchee, "~> 1.0", only: :bench}, + {:benchee_html, "~> 1.0", only: :bench}, + {:benchee_json, "~> 1.0", only: :bench} ] end diff --git a/mix.lock b/mix.lock index 4e04cfc6..270dcdd1 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,7 @@ %{ "benchee": {:hex, :benchee, "1.2.0", "afd2f0caec06ce3a70d9c91c514c0b58114636db9d83c2dc6bfd416656618353", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "ee729e53217898b8fd30aaad3cce61973dab61574ae6f48229fe7ff42d5e4457"}, + "benchee_html": {:hex, :benchee_html, "1.0.1", "1e247c0886c3fdb0d3f4b184b653a8d6fb96e4ad0d0389267fe4f36968772e24", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "b00a181af7152431901e08f3fc9f7197ed43ff50421a8347b0c80bf45d5b3fef"}, + "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"}, "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, From d00dae1163bb9f01adda31d578f4eda1ce06aa0e Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Tue, 23 Dec 2025 17:26:24 +0100 Subject: [PATCH 2/5] number the inputs to keep order, use Stream, add a control test Stream removes some of the overhead jitter of Enum, and having a control test helps one understand what the overhead of the test itself is versus what sort of time is being spent in EctoSQL modules --- bench/scripts/micro/load_bench.exs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/bench/scripts/micro/load_bench.exs b/bench/scripts/micro/load_bench.exs index 94a8296e..eed7a8c3 100644 --- a/bench/scripts/micro/load_bench.exs +++ b/bench/scripts/micro/load_bench.exs @@ -22,25 +22,26 @@ Code.require_file("../../support/setup.exs", __DIR__) alias Ecto.Bench.User inputs = %{ - "Small 1 Thousand" => - 1..1_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), - "Medium 100 Thousand" => - 1..100_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), - "Big 1 Million" => - 1..1_000_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), - "Time attr" => - 1..100_000 |> Enum.map(fn _ -> %{name: "Alice", time_attr: ~T[21:25:04.361140]} end), - "Date attr" => 1..100_000 |> Enum.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end), - "NaiveDateTime attr" => + "1. Small 1 Thousand" => + 1..1_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), + "2. Medium 100 Thousand" => + 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), + "3. Big 1 Million" => + 1..1_000_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), + "4. Time attr" => + 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", time_attr: ~T[21:25:04.361140]} end), + "5. Date attr" => 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end), + "6. NaiveDateTime attr" => 1..100_000 - |> Enum.map(fn _ -> %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]} end), - "UUID attr" => + |> Stream.map(fn _ -> %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]} end), + "7. UUID attr" => 1..100_000 - |> Enum.map(fn _ -> %{name: "Alice", uuid: Ecto.UUID.bingenerate()} end) + |> Stream.map(fn _ -> %{name: "Alice", uuid: Ecto.UUID.bingenerate()} end) } jobs = %{ - "Pg Loader" => fn data -> Enum.map(data, &Ecto.Bench.PgRepo.load(User, &1)) end, + "Control" => fn stream -> stream |> Stream.map(fn _data -> true end) |> Stream.run() end, + "Pg Loader" => fn stream -> stream |> Stream.map(&Ecto.Bench.PgRepo.load(User, &1)) |> Stream.run() end, "MyXQL Loader" => fn data -> Enum.map(data, &Ecto.Bench.MyXQLRepo.load(User, &1)) end } From 485fa080e5440603db0a7d38cbce7d99727a5815 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Tue, 23 Dec 2025 17:28:28 +0100 Subject: [PATCH 3/5] add numbers to preserve order --- bench/scripts/micro/to_sql_bench.exs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bench/scripts/micro/to_sql_bench.exs b/bench/scripts/micro/to_sql_bench.exs index f841ed38..faac94b6 100644 --- a/bench/scripts/micro/to_sql_bench.exs +++ b/bench/scripts/micro/to_sql_bench.exs @@ -25,20 +25,20 @@ import Ecto.Query alias Ecto.Bench.{User, Game} inputs = %{ - "Ordinary Select All" => {:all, from(User)}, - "Ordinary Delete All" => {:delete_all, from(User)}, - "Ordinary Update All" => {:update_all, from(User, update: [set: [name: "Thor"]])}, - "Ordinary Where" => {:all, from(User, where: [name: "Thanos", email: "blah@blah"])}, - "Fetch First Registry" => {:all, first(User)}, - "Fetch Last Registry" => {:all, last(User)}, - "Ordinary Order By" => {:all, order_by(User, desc: :name)}, - "Complex Query 2 Joins" => + "1. Ordinary Select All" => {:all, from(User)}, + "2. Ordinary Delete All" => {:delete_all, from(User)}, + "3. Ordinary Update All" => {:update_all, from(User, update: [set: [name: "Thor"]])}, + "4. Ordinary Where" => {:all, from(User, where: [name: "Thanos", email: "blah@blah"])}, + "5. Fetch First Registry" => {:all, first(User)}, + "6. Fetch Last Registry" => {:all, last(User)}, + "7. Ordinary Order By" => {:all, order_by(User, desc: :name)}, + "8. Complex Query 2 Joins" => {:all, from(User, where: [name: "Thanos"]) |> join(:left, [u], ux in User, on: u.id == ux.id) |> join(:right, [j], uj in User, on: j.id == 1 and j.email == "email@email") |> select([u, ux], {u.name, ux.email})}, - "Complex Query 4 Joins" => + "9. Complex Query 4 Joins" => {:all, from(User) |> join(:left, [u], g in Game, on: g.name == u.name) From 584fe87dd20f83e8fc02cc5bcc905a6f4ee5f8ce Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Tue, 23 Dec 2025 17:28:50 +0100 Subject: [PATCH 4/5] remove a warning --- bench/scripts/micro/to_sql_bench.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/scripts/micro/to_sql_bench.exs b/bench/scripts/micro/to_sql_bench.exs index faac94b6..f3ae2a9a 100644 --- a/bench/scripts/micro/to_sql_bench.exs +++ b/bench/scripts/micro/to_sql_bench.exs @@ -43,8 +43,8 @@ inputs = %{ from(User) |> join(:left, [u], g in Game, on: g.name == u.name) |> join(:right, [g], u in User, on: g.id == 1 and u.email == "email@email") - |> join(:inner, [u], g in fragment("SELECT * from games where game.id = ?", u.id)) - |> join(:left, [g], u in fragment("SELECT * from users = ?", g.id)) + |> join(:inner, [u], g in fragment("SELECT * from games where game.id = ?", u.id), on: true) + |> join(:left, [g], u in fragment("SELECT * from users = ?", g.id), on: true) |> select([u, g], {u.name, g.price})} } From b98e6ac63d3987eccbb556606b767b0c8d18fe98 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Tue, 23 Dec 2025 19:13:28 +0100 Subject: [PATCH 5/5] use Enum.each; it is a bit faster than stream in this case --- bench/scripts/micro/load_bench.exs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bench/scripts/micro/load_bench.exs b/bench/scripts/micro/load_bench.exs index eed7a8c3..f6c611ee 100644 --- a/bench/scripts/micro/load_bench.exs +++ b/bench/scripts/micro/load_bench.exs @@ -30,19 +30,22 @@ inputs = %{ 1..1_000_000 |> Stream.map(fn _ -> %{name: "Alice", email: "email@email.com"} end), "4. Time attr" => 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", time_attr: ~T[21:25:04.361140]} end), - "5. Date attr" => 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end), + "5. Date attr" => + 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end), "6. NaiveDateTime attr" => 1..100_000 - |> Stream.map(fn _ -> %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]} end), + |> Stream.map(fn _ -> + %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]} + end), "7. UUID attr" => 1..100_000 |> Stream.map(fn _ -> %{name: "Alice", uuid: Ecto.UUID.bingenerate()} end) } jobs = %{ - "Control" => fn stream -> stream |> Stream.map(fn _data -> true end) |> Stream.run() end, - "Pg Loader" => fn stream -> stream |> Stream.map(&Ecto.Bench.PgRepo.load(User, &1)) |> Stream.run() end, - "MyXQL Loader" => fn data -> Enum.map(data, &Ecto.Bench.MyXQLRepo.load(User, &1)) end + "Control" => fn stream -> Enum.each(stream, fn _data -> true end) end, + "Pg Loader" => fn stream -> Enum.each(stream, &Ecto.Bench.PgRepo.load(User, &1)) end, + "MyXQL Loader" => fn stream -> Enum.each(stream, &Ecto.Bench.MyXQLRepo.load(User, &1)) end } Benchee.run(