From b6d22fc5122cfa7970312c083eee1ef0474fd733 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 4 Dec 2025 16:11:04 -0500 Subject: [PATCH 1/4] Use follower add/drop for permissions and quote follower metadata --- .../Changes/FollowerPermissionChangeTests.cs | 79 ++++++++++++++++ KustoSchemaTools/Changes/DatabaseChanges.cs | 13 +++ .../Changes/FollowerPermissionChange.cs | 94 +++++++++++++++++++ KustoSchemaTools/Model/FollowerPermissions.cs | 3 + .../Parser/KustoLoader/FollowerLoader.cs | 61 +++++++++++- .../KustoWriter/DefaultDatabaseWriter.cs | 31 +++++- 6 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 KustoSchemaTools.Tests/Changes/FollowerPermissionChangeTests.cs create mode 100644 KustoSchemaTools/Changes/FollowerPermissionChange.cs diff --git a/KustoSchemaTools.Tests/Changes/FollowerPermissionChangeTests.cs b/KustoSchemaTools.Tests/Changes/FollowerPermissionChangeTests.cs new file mode 100644 index 0000000..69849ef --- /dev/null +++ b/KustoSchemaTools.Tests/Changes/FollowerPermissionChangeTests.cs @@ -0,0 +1,79 @@ +using KustoSchemaTools.Changes; +using KustoSchemaTools.Model; +using Microsoft.Extensions.Logging; +using Moq; + +namespace KustoSchemaTools.Tests.Changes +{ + public class FollowerPermissionChangeTests + { + private readonly Mock _logger = new(); + + private static FollowerDatabase BuildFollower(params (string role, string id, string name)[] principals) + { + var follower = new FollowerDatabase + { + DatabaseName = "DDoSNeuralAnalysis", + Permissions = new FollowerPermissions { ModificationKind = FollowerModificationKind.Union } + }; + + foreach (var (role, id, name) in principals) + { + var obj = new AADObject { Id = id, Name = name }; + if (string.Equals(role, "admin", StringComparison.OrdinalIgnoreCase)) + { + follower.Permissions.Admins.Add(obj); + } + else + { + follower.Permissions.Viewers.Add(obj); + } + } + + return follower; + } + + [Fact] + public void GeneratesFollowerAdd_WithLeaderName() + { + var oldFollower = BuildFollower(); + var newFollower = BuildFollower(("viewer", "aadapp=64decea3-723a-4fbf-b2ec-9faaf852cfdc;398a6654-997b-47e9-b12b-9515b896b4de", "spn-dev-spam-slam")); + newFollower.Permissions.LeaderName = "leader-cluster"; + + var changes = DatabaseChanges.GenerateFollowerChanges(oldFollower, newFollower, _logger.Object); + + var permChange = Assert.Single(changes.OfType()); + var script = Assert.Single(permChange.Scripts).Script!.Text; + + Assert.Equal(".add follower database DDoSNeuralAnalysis viewers (\"aadapp=64decea3-723a-4fbf-b2ec-9faaf852cfdc;398a6654-997b-47e9-b12b-9515b896b4de\") 'leader-cluster'", script); + } + + [Fact] + public void GeneratesFollowerAdd_WithoutLeaderName() + { + var oldFollower = BuildFollower(); + var newFollower = BuildFollower(("admin", "aaduser=foo;tenant", "Foo")); + + var changes = DatabaseChanges.GenerateFollowerChanges(oldFollower, newFollower, _logger.Object); + + var permChange = Assert.Single(changes.OfType()); + var script = Assert.Single(permChange.Scripts).Script!.Text; + + Assert.Equal(".add follower database DDoSNeuralAnalysis admins (\"aaduser=foo;tenant\")", script); + } + + [Fact] + public void EmitsDrop_WhenRemovingPrincipals() + { + var oldFollower = BuildFollower(("admin", "aadapp=1;tenant", "v1")); + var newFollower = BuildFollower(); + + var changes = DatabaseChanges.GenerateFollowerChanges(oldFollower, newFollower, _logger.Object); + + var permChange = Assert.Single(changes.OfType()); + var script = Assert.Single(permChange.Scripts).Script!.Text; + + Assert.Equal(".drop follower database DDoSNeuralAnalysis admins (\"aadapp=1;tenant\")", script); + } + } +} diff --git a/KustoSchemaTools/Changes/DatabaseChanges.cs b/KustoSchemaTools/Changes/DatabaseChanges.cs index 59017ef..4644708 100644 --- a/KustoSchemaTools/Changes/DatabaseChanges.cs +++ b/KustoSchemaTools/Changes/DatabaseChanges.cs @@ -224,6 +224,19 @@ .. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews ]; + var permissionChanges = new List + { + new FollowerPermissionChange(newState.DatabaseName, "Admins", oldState.Permissions.Admins, newState.Permissions.Admins, newState.Permissions.LeaderName), + new FollowerPermissionChange(newState.DatabaseName, "Viewers", oldState.Permissions.Viewers, newState.Permissions.Viewers, newState.Permissions.LeaderName) + }.Where(itm => itm.Scripts.Any()).ToList(); + + if (permissionChanges.Any()) + { + log.LogInformation($"Detected {permissionChanges.Count} follower permission changes"); + permissionChanges.Insert(0, new Heading("Permissions (Follower)")); + result.AddRange(permissionChanges); + } + if (oldState.Permissions.ModificationKind != newState.Permissions.ModificationKind) { var kind = newState.Permissions.ModificationKind.ToString().ToLower(); diff --git a/KustoSchemaTools/Changes/FollowerPermissionChange.cs b/KustoSchemaTools/Changes/FollowerPermissionChange.cs new file mode 100644 index 0000000..13d15db --- /dev/null +++ b/KustoSchemaTools/Changes/FollowerPermissionChange.cs @@ -0,0 +1,94 @@ +using Kusto.Language; +using KustoSchemaTools.Model; +using KustoSchemaTools.Parser; +using System.Text; + +namespace KustoSchemaTools.Changes +{ + // Generates follower-safe permission commands (add/drop) for admins/viewers. + public class FollowerPermissionChange : BaseChange> + { + public FollowerPermissionChange(string db, string entity, List from, List to, string? leaderName) + : base("FollowerPermissions", entity, from ?? new List(), to) + { + Db = db; + LeaderName = leaderName; + Init(); + } + + public string Db { get; } + public string? LeaderName { get; } + + private static IEnumerable PrincipalStrings(IEnumerable principals) => + principals.OrderBy(p => p.Id, StringComparer.OrdinalIgnoreCase).Select(a => "\"" + a.Id + "\""); + + private string BuildAdd(IEnumerable principals) + { + var ids = string.Join(",", PrincipalStrings(principals)); + var leaderSuffix = string.IsNullOrWhiteSpace(LeaderName) ? string.Empty : $" '{LeaderName}'"; + return $".add follower database {Db.BracketIfIdentifier()} {Entity.ToLower()} ({ids}){leaderSuffix}"; + } + + private string BuildDrop(IEnumerable principals) + { + var ids = string.Join(",", PrincipalStrings(principals)); + return $".drop follower database {Db.BracketIfIdentifier()} {Entity.ToLower()} ({ids})"; + } + + private void Init() + { + var added = To.Where(itm => From.All(t => t.Id != itm.Id)).ToList(); + var removed = From.Where(itm => To.All(t => t.Id != itm.Id)).ToList(); + var changed = From.Join(To, f => f.Id, t => t.Id, (f, t) => new { f, t }) + .Where(x => x.f.Name != x.t.Name) + .ToList(); + + if (removed.Any()) + { + var script = new DatabaseScript { Text = BuildDrop(removed), Order = -1 }; + var container = new DatabaseScriptContainer(script, "FollowerPermissionChange"); + container.IsValid = !KustoCode.Parse(script.Text).GetDiagnostics().Any(); + Scripts.Add(container); + } + + if (added.Any()) + { + var script = new DatabaseScript { Text = BuildAdd(added), Order = 0 }; + var container = new DatabaseScriptContainer(script, "FollowerPermissionChange"); + container.IsValid = !KustoCode.Parse(script.Text).GetDiagnostics().Any(); + Scripts.Add(container); + } + + if (changed.Any()) + { + Scripts.Add(new DatabaseScriptContainer("FollowerPermissionRenamed", -1, "// No Database Change")); + } + + var sb = new StringBuilder(); + sb.AppendLine($"## {Entity} (Follower)"); + sb.AppendLine(); + sb.AppendLine(""); + sb.AppendLine(""); + + if (added.Any()) + { + sb.AppendLine(""); + } + if (removed.Any()) + { + sb.AppendLine(""); + } + if (changed.Any()) + { + sb.AppendLine(""); + } + + var logo = Scripts.Any() && Scripts.First().IsValid == false ? ":red_circle:" : ":green_circle:"; + var displayCmd = Scripts.FirstOrDefault()?.Script?.Text ?? "// No change"; + sb.AppendLine($""); + sb.AppendLine("
Added:" + string.Join("
", added.Select(t => $"{t.Name} ({t.Id})")) + "
Removed:" + string.Join("
", removed.Select(t => $"{t.Name} ({t.Id})")) + "
Changed:" + string.Join("
", changed.Select(t => $"{t.f.Name} => {t.t.Name} ({t.t.Id})")) + "
{logo}
{displayCmd.PrettifyKql()}
"); + + Markdown = sb.ToString(); + } + } +} diff --git a/KustoSchemaTools/Model/FollowerPermissions.cs b/KustoSchemaTools/Model/FollowerPermissions.cs index 193d887..91a618d 100644 --- a/KustoSchemaTools/Model/FollowerPermissions.cs +++ b/KustoSchemaTools/Model/FollowerPermissions.cs @@ -5,6 +5,9 @@ public class FollowerPermissions public FollowerModificationKind ModificationKind { get; set; } public List Viewers { get; set; } = new List(); public List Admins { get; set; } = new List(); + + // Optional leader name (as known to the follower) used by some follower commands + public string? LeaderName { get; set; } } } diff --git a/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs b/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs index d09989b..489147d 100644 --- a/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs +++ b/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs @@ -1,4 +1,5 @@ using KustoSchemaTools.Model; +using KustoSchemaTools.Parser; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; @@ -21,15 +22,24 @@ public class FollowerLoader | where isnotempty(Timespan) | limit 1 ) -| summarize CachingPolicies=make_bag(bag_pack(Table,Timespan)) -) -"; +| summarize CachingPolicies=make_bag(bag_pack(Table,Timespan)), + DatabaseName=any(DatabaseName), + LeaderClusterMetadataPath=any(LeaderClusterMetadataPath), + CachingPolicyOverride=any(CachingPolicyOverride), + AuthorizedPrincipalsOverride=any(AuthorizedPrincipalsOverride), + AuthorizedPrincipalsModificationKind=any(AuthorizedPrincipalsModificationKind), + CachingPoliciesModificationKind=any(CachingPoliciesModificationKind), + ChildEntities=any(ChildEntities), + OriginalDatabaseName=any(OriginalDatabaseName), + IsAutoPrefetchEnabled=any(IsAutoPrefetchEnabled), + LeaderName=any(LeaderName) +)"; public static FollowerDatabase LoadFollower(string databaseName, KustoClient client) { var follower = new FollowerDatabase { DatabaseName = databaseName }; // Execute the query and handle the case where no rows are returned (e.g., database is not a follower) - var queryResult = client.Client.ExecuteQuery(string.Format(FollowerMetadataQuery, databaseName)); + var queryResult = client.Client.ExecuteQuery(string.Format(FollowerMetadataQuery, databaseName.BracketIfIdentifier())); var metdaData = queryResult.As().FirstOrDefault(); if (metdaData == null) @@ -73,6 +83,48 @@ public static FollowerDatabase LoadFollower(string databaseName, KustoClient cli target.Add(key, kvp.Value.Days+"d"); } + follower.Permissions.LeaderName = metdaData.LeaderName; + + if (!string.IsNullOrWhiteSpace(metdaData.AuthorizedPrincipalsOverride)) + { + try + { + var arr = JArray.Parse(metdaData.AuthorizedPrincipalsOverride); + foreach (var principalObj in arr) + { + var role = principalObj["Role"]?.Value(); + var principal = principalObj["Principal"]?["FullyQualifiedName"]?.Value() + ?? principalObj["Principal"]?["Id"]?.Value(); + var displayName = principalObj["Principal"]?["DisplayName"]?.Value() + ?? principal; + + if (string.IsNullOrWhiteSpace(principal) || role == null) + { + continue; + } + + var aadObj = new AADObject + { + Id = principal, + Name = displayName + }; + + if (role == 0) + { + follower.Permissions.Admins.Add(aadObj); + } + else if (role == 2) + { + follower.Permissions.Viewers.Add(aadObj); + } + } + } + catch (Exception) + { + // Ignore parse errors; treat as empty and skip permission diffs + } + } + return follower; } } @@ -81,6 +133,7 @@ public class FollowerMetadata { public string? DatabaseName { get; set; } public string? LeaderClusterMetadataPath { get; set; } + public string? LeaderName { get; set; } public string? CachingPolicyOverride { get; set; } public string? AuthorizedPrincipalsOverride { get; set; } public string? AuthorizedPrincipalsModificationKind { get; set; } diff --git a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs index 52b8db6..2e4e2a7 100644 --- a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs +++ b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs @@ -12,7 +12,36 @@ public class DefaultDatabaseWriter : IDBEntityWriter { public async Task WriteAsync(Database sourceDb, Database targetDb, KustoClient client, ILogger logger) { - var changes = DatabaseChanges.GenerateChanges(targetDb, sourceDb, targetDb.Name, logger); + var followerMeta = FollowerLoader.LoadFollower(targetDb.Name, client); + var isFollower = followerMeta.Permissions.ModificationKind != FollowerModificationKind.None + || followerMeta.Cache.ModificationKind != FollowerModificationKind.None + || followerMeta.Cache.Tables.Any() + || followerMeta.Cache.MaterializedViews.Any(); + + List changes; + if (isFollower) + { + // Build desired follower from YAML/source DB + var desiredFollower = new FollowerDatabase + { + DatabaseName = targetDb.Name, + Permissions = new FollowerPermissions + { + ModificationKind = followerMeta.Permissions.ModificationKind, + Admins = sourceDb.Admins, + Viewers = sourceDb.Viewers, + LeaderName = followerMeta.Permissions.LeaderName + }, + Cache = followerMeta.Cache + }; + + changes = DatabaseChanges.GenerateFollowerChanges(followerMeta, desiredFollower, logger); + } + else + { + changes = DatabaseChanges.GenerateChanges(targetDb, sourceDb, targetDb.Name, logger); + } + var results = await ApplyChangesToDatabase(targetDb.Name, changes, client, logger); foreach (var result in results) From 96e3f891669faace060c9d08844abfc6c0c92b85 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 4 Dec 2025 16:25:37 -0500 Subject: [PATCH 2/4] Ensure follower drops execute before adds --- KustoSchemaTools/Changes/FollowerPermissionChange.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools/Changes/FollowerPermissionChange.cs b/KustoSchemaTools/Changes/FollowerPermissionChange.cs index 13d15db..41fda2a 100644 --- a/KustoSchemaTools/Changes/FollowerPermissionChange.cs +++ b/KustoSchemaTools/Changes/FollowerPermissionChange.cs @@ -45,7 +45,8 @@ private void Init() if (removed.Any()) { - var script = new DatabaseScript { Text = BuildDrop(removed), Order = -1 }; + // Execute drops before adds; keep non-negative so they aren't filtered out. + var script = new DatabaseScript { Text = BuildDrop(removed), Order = 0 }; var container = new DatabaseScriptContainer(script, "FollowerPermissionChange"); container.IsValid = !KustoCode.Parse(script.Text).GetDiagnostics().Any(); Scripts.Add(container); @@ -53,7 +54,7 @@ private void Init() if (added.Any()) { - var script = new DatabaseScript { Text = BuildAdd(added), Order = 0 }; + var script = new DatabaseScript { Text = BuildAdd(added), Order = removed.Any() ? 1 : 0 }; var container = new DatabaseScriptContainer(script, "FollowerPermissionChange"); container.IsValid = !KustoCode.Parse(script.Text).GetDiagnostics().Any(); Scripts.Add(container); From 3a998b741b079b454b2b701ba0fd7c75786577ee Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 4 Dec 2025 16:34:22 -0500 Subject: [PATCH 3/4] Detect follower DBs when metadata exists --- .../Parser/KustoWriter/DefaultDatabaseWriter.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs index 2e4e2a7..eda0f2d 100644 --- a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs +++ b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs @@ -13,10 +13,8 @@ public class DefaultDatabaseWriter : IDBEntityWriter public async Task WriteAsync(Database sourceDb, Database targetDb, KustoClient client, ILogger logger) { var followerMeta = FollowerLoader.LoadFollower(targetDb.Name, client); - var isFollower = followerMeta.Permissions.ModificationKind != FollowerModificationKind.None - || followerMeta.Cache.ModificationKind != FollowerModificationKind.None - || followerMeta.Cache.Tables.Any() - || followerMeta.Cache.MaterializedViews.Any(); + // Treat as follower if metadata came back (DatabaseName populated); followers may have no overrides yet. + var isFollower = !string.IsNullOrWhiteSpace(followerMeta.DatabaseName); List changes; if (isFollower) From de4d85bb1a9b265711453d7b35bb51c15c3a71b7 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 4 Dec 2025 16:40:01 -0500 Subject: [PATCH 4/4] Detect followers via metadata flag and keep leaders on normal path --- KustoSchemaTools/Model/FollowerDatabase.cs | 6 ++++++ KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs | 2 ++ .../Parser/KustoWriter/DefaultDatabaseWriter.cs | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools/Model/FollowerDatabase.cs b/KustoSchemaTools/Model/FollowerDatabase.cs index a834d54..1376372 100644 --- a/KustoSchemaTools/Model/FollowerDatabase.cs +++ b/KustoSchemaTools/Model/FollowerDatabase.cs @@ -6,6 +6,12 @@ public class FollowerDatabase public FollowerCache Cache { get; set; } = new FollowerCache(); // TODO: No logic to load data / roll out changes implemented yet! public FollowerPermissions Permissions { get; set; } = new FollowerPermissions(); + + // True when follower metadata was returned by .show follower database + public bool IsFollower { get; set; } + + // Populated when available from follower metadata (e.g., Data Share followers) + public string? LeaderClusterMetadataPath { get; set; } } } diff --git a/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs b/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs index 489147d..cf9286d 100644 --- a/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs +++ b/KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs @@ -48,6 +48,8 @@ public static FollowerDatabase LoadFollower(string databaseName, KustoClient cli return follower; } + follower.IsFollower = true; + switch (metdaData.AuthorizedPrincipalsModificationKind) { case "Union": diff --git a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs index eda0f2d..204b576 100644 --- a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs +++ b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs @@ -13,8 +13,8 @@ public class DefaultDatabaseWriter : IDBEntityWriter public async Task WriteAsync(Database sourceDb, Database targetDb, KustoClient client, ILogger logger) { var followerMeta = FollowerLoader.LoadFollower(targetDb.Name, client); - // Treat as follower if metadata came back (DatabaseName populated); followers may have no overrides yet. - var isFollower = !string.IsNullOrWhiteSpace(followerMeta.DatabaseName); + // Treat as follower only when metadata exists (.show follower database returned a row) + var isFollower = followerMeta.IsFollower; List changes; if (isFollower)