From 2de6b420f369abd07f28f78ff901447827e9fddd Mon Sep 17 00:00:00 2001 From: Tyler Brinks Date: Sun, 1 Dec 2024 17:45:26 -0700 Subject: [PATCH 1/2] Explain plan with single column schema --- .../Prequel/Execution/ExecutionContext.cs | 2 + .../Prequel/Execution/ExplainExecution.cs | 25 +++++++++++ src/Engine/Prequel/Logical/Plans/Explain.cs | 13 ++++++ src/Engine/Prequel/Logical/Plans/Filter.cs | 2 +- src/Engine/Prequel/Logical/Plans/Limit.cs | 2 +- src/Engine/Prequel/Logical/Plans/Sort.cs | 5 ++- src/Engine/Prequel/Logical/Plans/TableScan.cs | 10 ++--- src/Engine/Prequel/Logical/Plans/Union.cs | 2 +- .../Prequel/Physical/PhysicalPlanner.cs | 2 + src/Tests/Prequel.Console/Program.cs | 42 ++++++++++++++++++- .../Physical/ExecutionPlanTests.cs | 18 ++++++++ 11 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 src/Engine/Prequel/Execution/ExplainExecution.cs create mode 100644 src/Engine/Prequel/Logical/Plans/Explain.cs diff --git a/src/Engine/Prequel/Execution/ExecutionContext.cs b/src/Engine/Prequel/Execution/ExecutionContext.cs index 6991fea..a1ec8a9 100644 --- a/src/Engine/Prequel/Execution/ExecutionContext.cs +++ b/src/Engine/Prequel/Execution/ExecutionContext.cs @@ -4,6 +4,7 @@ using Prequel.Metrics; using Prequel.Data; using Prequel.Logical; +using Prequel.Logical.Plans; namespace Prequel.Execution; @@ -76,6 +77,7 @@ internal ILogicalPlan BuildLogicalPlan(string sql) var plan = ast.First() switch { Statement.Select select => LogicalExtensions.CreateQuery(select.Query, new PlannerContext(_tables)), + Statement.Explain explain => new Explain(LogicalExtensions.CreateQuery(explain.Statement.AsSelect(), new PlannerContext(_tables))), _ => throw new NotImplementedException() }; diff --git a/src/Engine/Prequel/Execution/ExplainExecution.cs b/src/Engine/Prequel/Execution/ExplainExecution.cs new file mode 100644 index 0000000..bff0e7f --- /dev/null +++ b/src/Engine/Prequel/Execution/ExplainExecution.cs @@ -0,0 +1,25 @@ +using System.Runtime.CompilerServices; +using Prequel.Data; +using Prequel.Logical; +using Prequel.Logical.Plans; + +namespace Prequel.Execution; + +internal class ExplainExecution(Explain explain) : IExecutionPlan +{ + public Schema Schema => explain.Schema; + + public async IAsyncEnumerable ExecuteAsync(QueryContext queryContext, + [EnumeratorCancellation] CancellationToken cancellation = default) + { + var steps = explain.ToStringIndented(new Indentation()).Split(Environment.NewLine); + var batch = new RecordBatch(Schema); + + foreach (var step in steps) + { + batch.AddResult(0, step); + } + + yield return batch; + } +} diff --git a/src/Engine/Prequel/Logical/Plans/Explain.cs b/src/Engine/Prequel/Logical/Plans/Explain.cs new file mode 100644 index 0000000..63d2dbd --- /dev/null +++ b/src/Engine/Prequel/Logical/Plans/Explain.cs @@ -0,0 +1,13 @@ +using Prequel.Data; + +namespace Prequel.Logical.Plans; + +internal class Explain(ILogicalPlan plan) : ILogicalPlan +{ + public Schema Schema { get; } = new([new QualifiedField("plan", ColumnDataType.Utf8) ]); + + public string ToStringIndented(Indentation? indentation = null) + { + return plan.ToStringIndented(indentation ?? new Indentation()); + } +} diff --git a/src/Engine/Prequel/Logical/Plans/Filter.cs b/src/Engine/Prequel/Logical/Plans/Filter.cs index 1acfd1f..c03b731 100644 --- a/src/Engine/Prequel/Logical/Plans/Filter.cs +++ b/src/Engine/Prequel/Logical/Plans/Filter.cs @@ -23,6 +23,6 @@ public override string ToString() public string ToStringIndented(Indentation? indentation = null) { var indent = indentation ?? new Indentation(); - return $"Filter: {Predicate}{indent.Next(Plan)}"; + return $"{this}{indent.Next(Plan)}"; } } \ No newline at end of file diff --git a/src/Engine/Prequel/Logical/Plans/Limit.cs b/src/Engine/Prequel/Logical/Plans/Limit.cs index afa29c2..bdc9ed8 100644 --- a/src/Engine/Prequel/Logical/Plans/Limit.cs +++ b/src/Engine/Prequel/Logical/Plans/Limit.cs @@ -23,6 +23,6 @@ public override string ToString() public string ToStringIndented(Indentation? indentation = null) { var indent = indentation ?? new Indentation(); - return $"Limit: Skip {Skip}, Limit {Fetch}{indent.Next(Plan)}"; + return $"{this}{indent.Next(Plan)}"; } } \ No newline at end of file diff --git a/src/Engine/Prequel/Logical/Plans/Sort.cs b/src/Engine/Prequel/Logical/Plans/Sort.cs index 9d27a88..6a35dca 100644 --- a/src/Engine/Prequel/Logical/Plans/Sort.cs +++ b/src/Engine/Prequel/Logical/Plans/Sort.cs @@ -126,10 +126,11 @@ private static ILogicalExpression RewriteForProjection( }); } - public string ToStringIndented(Indentation? indentation = null) { var indent = indentation ?? new Indentation(); - return $"Sort: {indent.Next(Plan)}"; + var orders = string.Join(",", OrderByExpressions.Select(o => o.ToString()? + .Replace("Order By",string.Empty, StringComparison.InvariantCultureIgnoreCase))); + return $"Sort: {orders}{indent.Next(Plan)}"; } } \ No newline at end of file diff --git a/src/Engine/Prequel/Logical/Plans/TableScan.cs b/src/Engine/Prequel/Logical/Plans/TableScan.cs index 503fb3d..2525383 100644 --- a/src/Engine/Prequel/Logical/Plans/TableScan.cs +++ b/src/Engine/Prequel/Logical/Plans/TableScan.cs @@ -19,12 +19,10 @@ public string ToStringIndented(Indentation? indentation = null) public override string ToString() { - string? fields = null; - if (Projection != null) - { - fields = " projection=" + string.Join(",", Projection.Select(i => Table.Schema!.Fields[i].Name)); - } + var fields = string.Join(", ", Projection != null + ? Projection.Select(i => Table.Schema!.Fields[i].Name) + : Schema.Fields.Select(f => f.QualifiedName)); - return $"Table Scan: {Name}{fields}"; + return $"Table Scan: {Name} projection=({fields})"; } } \ No newline at end of file diff --git a/src/Engine/Prequel/Logical/Plans/Union.cs b/src/Engine/Prequel/Logical/Plans/Union.cs index 64383da..721b359 100644 --- a/src/Engine/Prequel/Logical/Plans/Union.cs +++ b/src/Engine/Prequel/Logical/Plans/Union.cs @@ -18,7 +18,7 @@ public string ToStringIndented(Indentation? indentation = null) { var indent = indentation ?? new Indentation(); - var children = string.Join("", Inputs.Select((input, index) => index == 0 ? indent.Next(input) : indent.Repeat(input))); + var children = string.Join(string.Empty, Inputs.Select((input, index) => index == 0 ? indent.Next(input) : indent.Repeat(input))); return $"{this} {children}"; } diff --git a/src/Engine/Prequel/Physical/PhysicalPlanner.cs b/src/Engine/Prequel/Physical/PhysicalPlanner.cs index 18e5e0c..83f0959 100644 --- a/src/Engine/Prequel/Physical/PhysicalPlanner.cs +++ b/src/Engine/Prequel/Physical/PhysicalPlanner.cs @@ -35,6 +35,8 @@ public IExecutionPlan CreateInitialPlan(ILogicalPlan plan) // Distinct should have been replaced by an // aggregate plan by this point. Distinct => throw new InvalidOperationException("Distinct plans must be replaced with aggregations"), + Explain explain => new ExplainExecution(explain), + _ => throw new NotImplementedException("The physical plan type has not been implemented") }; } diff --git a/src/Tests/Prequel.Console/Program.cs b/src/Tests/Prequel.Console/Program.cs index 39fb8e2..3682c44 100644 --- a/src/Tests/Prequel.Console/Program.cs +++ b/src/Tests/Prequel.Console/Program.cs @@ -179,6 +179,14 @@ SELECT 1 """ }); +execution.AddQuery(new Query +{ + Name = "explain_scalar_value", + Text = """ + EXPLAIN SELECT 1 + """ +}); + // Slightly more complex, scalar values can be calculated using basic // SQL-style arithmetic. The query returns a table with three columns // with the calculated values. @@ -212,7 +220,7 @@ SELECT 1 { Name = "all_colors", Text = """ - SELECT + EXPLAIN SELECT * FROM colors """ @@ -327,6 +335,38 @@ LEFT JOIN departments d """ }); +execution.AddQuery(new Query +{ + Name = "explain_simple_query", + Text = """ + EXPLAIN SELECT + max(c1) as max_c1, avg(c3) avg_c3 + FROM colors + WHERE c1 in (1,2,3) + ORDER BY max_c1, avg_c3 desc + LIMIT 10 + OFFSET 3 + """ +}); + +execution.AddQuery(new Query +{ + Name = "explain_complex_query", + Text = """ + EXPLAIN SELECT + m.employee_id as ManagerId, + e.employee_id as EmpId, + m.first_name ManagerFN, + m.last_name ManagerLN, + e.first_name EmployeeFN, + e.last_name EmployeeLN + FROM employees m + JOIN employees e + ON m.employee_id = e.manager_id + WHERE e.manager_id in (100, 101) + ORDER BY e.manager_id + """ +}); #endregion var result = await execution.ExecuteAllAsync(); diff --git a/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs b/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs index 1a37452..f5b1ed4 100644 --- a/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs +++ b/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs @@ -329,4 +329,22 @@ public void Planner_Rejects_Distinct_Plan() { Assert.Throws(() => new PhysicalPlanner().CreateInitialPlan(new Distinct(new TestPlan()))); } + + [Fact] + public async Task Context_Explains_Query_Plan() + { + const string sql = "EXPLAIN SELECT a, b FROM db order by a desc offset 5 limit 10"; + var explainPlan = _context.BuildLogicalPlan(sql); + var explain = (ExplainExecution)Exec.BuildPhysicalPlan(explainPlan); + + var batch = await explain.ExecuteAsync(new QueryContext()).FirstAsync(); + + var expectedSchema = new Schema([new QualifiedField("plan", ColumnDataType.Utf8)]); + Assert.Equal(expectedSchema, batch.Schema); + + Assert.Equal("Limit: Skip 5, Limit 10", batch.Results[0].Values[0]); + Assert.Equal(" Sort: db.a Desc", batch.Results[0].Values[1]); + Assert.Equal(" Projection: db.a, db.b ", batch.Results[0].Values[2]); + Assert.Equal(" Table Scan: db projection=(db.a, db.b, db.c)", batch.Results[0].Values[3]); + } } \ No newline at end of file From fef622d6381980de550a43913ca3aed2cffab3b9 Mon Sep 17 00:00:00 2001 From: Tyler Brinks Date: Sun, 1 Dec 2024 17:55:40 -0700 Subject: [PATCH 2/2] Explain plan tests --- README.md | 2 +- src/Tests/Prequel.Tests/Data/ModelTests.cs | 4 ++-- src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5a04300..aa58967 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ A query engine is software that interprets a query (Structured Query Language) a - [X] Aggregations - `min`/`max`/`stddev`/etc... - [X] Math opperations - [X] `CAST` functions + - [X] `EXPLAIN` plans - [ ] `EXCLUDE`/`EXCEPT` - Not yet implemented - - [ ] `EXPLAIN` - Not yet implemented - [ ] `INSERT`, `UPDATE`, `DROP` operations ## Motivation diff --git a/src/Tests/Prequel.Tests/Data/ModelTests.cs b/src/Tests/Prequel.Tests/Data/ModelTests.cs index 7ed0a63..2b7d8c2 100644 --- a/src/Tests/Prequel.Tests/Data/ModelTests.cs +++ b/src/Tests/Prequel.Tests/Data/ModelTests.cs @@ -163,8 +163,8 @@ public void TableScan_Overrides_ToString() new EmptyDataTable("", schema, []), []); - Assert.Equal("Table Scan: table projection=", scan.ToStringIndented()); - Assert.Equal("Table Scan: table projection=", scan.ToString()); + Assert.Equal("Table Scan: table projection=()", scan.ToStringIndented()); + Assert.Equal("Table Scan: table projection=()", scan.ToString()); } [Fact] diff --git a/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs b/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs index f5b1ed4..ae075a3 100644 --- a/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs +++ b/src/Tests/Prequel.Tests/Physical/ExecutionPlanTests.cs @@ -6,6 +6,7 @@ using Prequel.Physical.Expressions; using Prequel.Execution; using Prequel.Data; +using SqlParser; using Schema = Prequel.Data.Schema; using Exec = Prequel.Execution.ExecutionContext; @@ -342,9 +343,17 @@ public async Task Context_Explains_Query_Plan() var expectedSchema = new Schema([new QualifiedField("plan", ColumnDataType.Utf8)]); Assert.Equal(expectedSchema, batch.Schema); + Assert.Equal(4, batch.Results[0].Values.Count); Assert.Equal("Limit: Skip 5, Limit 10", batch.Results[0].Values[0]); Assert.Equal(" Sort: db.a Desc", batch.Results[0].Values[1]); Assert.Equal(" Projection: db.a, db.b ", batch.Results[0].Values[2]); Assert.Equal(" Table Scan: db projection=(db.a, db.b, db.c)", batch.Results[0].Values[3]); } + + [Fact] + public void Nested_Explain_Should_Fail() + { + const string sql = "EXPLAIN EXPLAIN SELECT 1"; + Assert.Throws(() => _context.BuildLogicalPlan(sql)); + } } \ No newline at end of file