From 53fb0643d327c2c1ba190918659fb848f8cd36e3 Mon Sep 17 00:00:00 2001 From: Alekhya-Polavarapu Date: Thu, 4 Dec 2025 10:56:29 -0800 Subject: [PATCH 1/4] Fix the Serialization/Deserialization issue with $ prefix columns (#2944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Why make this change? Serialization and deserialization of metadata currently fail when column names are prefixed with the $ symbol. ### Root cause This issue occurs because we’ve enabled the ReferenceHandler flag in our System.Text.Json serialization settings. When this flag is active, the serializer treats $ as a reserved character used for special metadata (e.g., $id, $ref). As a result, any property name starting with $ is interpreted as metadata and cannot be deserialized properly. ### What is this change? This update introduces custom logic in the converter’s Write and Read methods to handle $-prefixed column names safely. - During serialization, columns beginning with $ are escaped as "_$". - During deserialization, this transformation is reversed to restore the original property names. ### How was this tested - [x] Unit tests --------- Co-authored-by: Aniruddh Munde --- .../Converters/DatabaseObjectConverter.cs | 78 +++++++++++++- .../SerializationDeserializationTests.cs | 101 ++++++++++++++++-- 2 files changed, 172 insertions(+), 7 deletions(-) diff --git a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs index c6e694a394..d1894c7c45 100644 --- a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs +++ b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs @@ -17,6 +17,11 @@ namespace Azure.DataApiBuilder.Core.Services.MetadataProviders.Converters public class DatabaseObjectConverter : JsonConverter { private const string TYPE_NAME = "TypeName"; + private const string DOLLAR_CHAR = "$"; + + // ``DAB_ESCAPE$`` is used to escape column names that start with `$` during serialization. + // It is chosen to be unique enough to avoid collisions with actual column names. + private const string ESCAPED_DOLLARCHAR = "DAB_ESCAPE$"; public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -29,6 +34,15 @@ public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConver DatabaseObject objA = (DatabaseObject)JsonSerializer.Deserialize(document, concreteType, options)!; + foreach (PropertyInfo prop in objA.GetType().GetProperties().Where(IsSourceDefinitionProperty)) + { + SourceDefinition? sourceDef = (SourceDefinition?)prop.GetValue(objA); + if (sourceDef is not null) + { + UnescapeDollaredColumns(sourceDef); + } + } + return objA; } } @@ -58,12 +72,74 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri } writer.WritePropertyName(prop.Name); - JsonSerializer.Serialize(writer, prop.GetValue(value), options); + object? propVal = prop.GetValue(value); + Type propType = prop.PropertyType; + + // Only escape columns for properties whose type is exactly SourceDefinition (not subclasses). + // This is because, we do not want unnecessary mutation of subclasses of SourceDefinition unless needed. + if (IsSourceDefinitionProperty(prop) && propVal is SourceDefinition sourceDef && propVal.GetType() == typeof(SourceDefinition)) + { + EscapeDollaredColumns(sourceDef); + } + + JsonSerializer.Serialize(writer, propVal, propType, options); } writer.WriteEndObject(); } + private static bool IsSourceDefinitionProperty(PropertyInfo prop) + { + // Only return true for properties whose type is exactly SourceDefinition (not subclasses) + return prop.PropertyType == typeof(SourceDefinition); + } + + /// + /// Escapes column keys that start with '$' to '_$' for serialization. + /// + private static void EscapeDollaredColumns(SourceDefinition sourceDef) + { + if (sourceDef.Columns is null || sourceDef.Columns.Count == 0) + { + return; + } + + List keysToEscape = sourceDef.Columns.Keys + .Where(k => k.StartsWith(DOLLAR_CHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysToEscape) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = ESCAPED_DOLLARCHAR + key[1..]; + sourceDef.Columns[newKey] = col; + } + } + + /// + /// Unescapes column keys that start with '_$' to '$' for deserialization. + /// + private static void UnescapeDollaredColumns(SourceDefinition sourceDef) + { + if (sourceDef.Columns is null || sourceDef.Columns.Count == 0) + { + return; + } + + List keysToUnescape = sourceDef.Columns.Keys + .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysToUnescape) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = DOLLAR_CHAR + key[11..]; + sourceDef.Columns[newKey] = col; + } + } + private static Type GetTypeFromName(string typeName) { Type? type = Type.GetType(typeName); diff --git a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs index 2b5e5bf3ba..8a22c5d85a 100644 --- a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs @@ -276,8 +276,96 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization() VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "FirstName"); } - private void InitializeObjects() + /// + /// Validates serialization and deserilization of Dictionary containing DatabaseTable + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDictionaryDatabaseObjectSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + }, + ReferenceHandler = ReferenceHandler.Preserve + }; + + Dictionary dict = new() { { "person", _databaseTable } }; + + string serializedDict = JsonSerializer.Serialize(dict, _options); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDict, _options)!; + + DatabaseTable deserializedDatabaseTable = (DatabaseTable)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseTable.SourceType, _databaseTable.SourceType); + Assert.AreEqual(deserializedDatabaseTable.FullName, _databaseTable.FullName); + deserializedDatabaseTable.Equals(_databaseTable); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.SourceDefinition, _databaseTable.SourceDefinition, "$FirstName"); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "$FirstName"); + } + + /// + /// Validates serialization and deserilization of Dictionary containing DatabaseView + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDatabaseViewSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + TestTypeNameChanges(_databaseView, "DatabaseView"); + + // Test to catch if there is change in number of properties/fields + // Note: On Addition of property make sure it is added in following object creation _databaseView and include in serialization + // and deserialization test. + int fields = typeof(DatabaseView).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; + Assert.AreEqual(fields, 6); + + string serializedDatabaseView = JsonSerializer.Serialize(_databaseView, _options); + DatabaseView deserializedDatabaseView = JsonSerializer.Deserialize(serializedDatabaseView, _options)!; + + Assert.AreEqual(deserializedDatabaseView.SourceType, _databaseView.SourceType); + deserializedDatabaseView.Equals(_databaseView); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.SourceDefinition, _databaseView.SourceDefinition, "$FirstName"); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.ViewDefinition, _databaseView.ViewDefinition, "$FirstName"); + } + + /// + /// Validates serialization and deserilization of Dictionary containing DatabaseStoredProcedure + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDatabaseStoredProcedureSerializationDeserialization_WithDollarColumn() { + InitializeObjects(generateDollaredColumn: true); + + TestTypeNameChanges(_databaseStoredProcedure, "DatabaseStoredProcedure"); + + // Test to catch if there is change in number of properties/fields + // Note: On Addition of property make sure it is added in following object creation _databaseStoredProcedure and include in serialization + // and deserialization test. + int fields = typeof(DatabaseStoredProcedure).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; + Assert.AreEqual(fields, 6); + + string serializedDatabaseSP = JsonSerializer.Serialize(_databaseStoredProcedure, _options); + DatabaseStoredProcedure deserializedDatabaseSP = JsonSerializer.Deserialize(serializedDatabaseSP, _options)!; + + Assert.AreEqual(deserializedDatabaseSP.SourceType, _databaseStoredProcedure.SourceType); + deserializedDatabaseSP.Equals(_databaseStoredProcedure); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.SourceDefinition, _databaseStoredProcedure.SourceDefinition, "$FirstName", true); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.StoredProcedureDefinition, _databaseStoredProcedure.StoredProcedureDefinition, "$FirstName", true); + } + + private void InitializeObjects(bool generateDollaredColumn = false) + { + string columnName = generateDollaredColumn ? "$FirstName" : "FirstName"; _options = new() { // ObjectConverter behavior different in .NET8 most likely due to @@ -289,10 +377,11 @@ private void InitializeObjects() new DatabaseObjectConverter(), new TypeConverter() } + }; _columnDefinition = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("John"), false); - _sourceDefinition = GetSourceDefinition(false, false, new List() { "FirstName" }, _columnDefinition); + _sourceDefinition = GetSourceDefinition(false, false, new List() { columnName }, _columnDefinition); _databaseTable = new DatabaseTable() { @@ -311,10 +400,10 @@ private void InitializeObjects() { IsInsertDMLTriggerEnabled = false, IsUpdateDMLTriggerEnabled = false, - PrimaryKey = new List() { "FirstName" }, + PrimaryKey = new List() { columnName }, }, }; - _databaseView.ViewDefinition.Columns.Add("FirstName", _columnDefinition); + _databaseView.ViewDefinition.Columns.Add(columnName, _columnDefinition); _parameterDefinition = new() { @@ -331,10 +420,10 @@ private void InitializeObjects() SourceType = EntitySourceType.StoredProcedure, StoredProcedureDefinition = new() { - PrimaryKey = new List() { "FirstName" }, + PrimaryKey = new List() { columnName }, } }; - _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add("FirstName", _columnDefinition); + _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add(columnName, _columnDefinition); _databaseStoredProcedure.StoredProcedureDefinition.Parameters.Add("Id", _parameterDefinition); } From e38566751642275d6f125f2c82aabe92a6d55ea0 Mon Sep 17 00:00:00 2001 From: Alekhya-Polavarapu Date: Tue, 13 Jan 2026 09:41:36 -0800 Subject: [PATCH 2/4] Fix serialization for StoredProcedureDefinition inheritance (#3045) ## Why make this change? - To apply correct serialization and deserialization logic for stored procedures. With the previous changes, serialization was not working correctly for the StoredProcedureDefinition type, which extends SourceDefinition. When the value type was passed explicitly for serialization, the parent type was used instead, causing some child-type properties to be omitted. ## What is this change? Instead of manually specifying the value type during serialization, this change allows the library to infer the type automatically and perform the correct serialization. ## How was this tested? - [x] Unit Tests --------- Co-authored-by: Aniruddh Munde --- .../Converters/DatabaseObjectConverter.cs | 16 +++++------ .../SerializationDeserializationTests.cs | 28 +++++++++++++++---- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs index d1894c7c45..6d625c7f9d 100644 --- a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs +++ b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs @@ -34,7 +34,7 @@ public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConver DatabaseObject objA = (DatabaseObject)JsonSerializer.Deserialize(document, concreteType, options)!; - foreach (PropertyInfo prop in objA.GetType().GetProperties().Where(IsSourceDefinitionProperty)) + foreach (PropertyInfo prop in objA.GetType().GetProperties().Where(IsSourceDefinitionOrDerivedClassProperty)) { SourceDefinition? sourceDef = (SourceDefinition?)prop.GetValue(objA); if (sourceDef is not null) @@ -73,25 +73,23 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri writer.WritePropertyName(prop.Name); object? propVal = prop.GetValue(value); - Type propType = prop.PropertyType; - // Only escape columns for properties whose type is exactly SourceDefinition (not subclasses). - // This is because, we do not want unnecessary mutation of subclasses of SourceDefinition unless needed. - if (IsSourceDefinitionProperty(prop) && propVal is SourceDefinition sourceDef && propVal.GetType() == typeof(SourceDefinition)) + // Only escape columns for properties whose type(derived type) is SourceDefinition. + if (IsSourceDefinitionOrDerivedClassProperty(prop) && propVal is SourceDefinition sourceDef) { EscapeDollaredColumns(sourceDef); } - JsonSerializer.Serialize(writer, propVal, propType, options); + JsonSerializer.Serialize(writer, propVal, options); } writer.WriteEndObject(); } - private static bool IsSourceDefinitionProperty(PropertyInfo prop) + private static bool IsSourceDefinitionOrDerivedClassProperty(PropertyInfo prop) { - // Only return true for properties whose type is exactly SourceDefinition (not subclasses) - return prop.PropertyType == typeof(SourceDefinition); + // Return true for properties whose type is SourceDefinition or any class derived from SourceDefinition + return typeof(SourceDefinition).IsAssignableFrom(prop.PropertyType); } /// diff --git a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs index 8a22c5d85a..29735231c2 100644 --- a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs @@ -298,8 +298,11 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization_WithDollarC Dictionary dict = new() { { "person", _databaseTable } }; string serializedDict = JsonSerializer.Serialize(dict, _options); - Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDict, _options)!; + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDict.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDict, _options)!; DatabaseTable deserializedDatabaseTable = (DatabaseTable)deserializedDict["person"]; Assert.AreEqual(deserializedDatabaseTable.SourceType, _databaseTable.SourceType); @@ -321,14 +324,22 @@ public void TestDatabaseViewSerializationDeserialization_WithDollarColumn() TestTypeNameChanges(_databaseView, "DatabaseView"); + Dictionary dict = new(); + dict.Add("person", _databaseView); + // Test to catch if there is change in number of properties/fields // Note: On Addition of property make sure it is added in following object creation _databaseView and include in serialization // and deserialization test. int fields = typeof(DatabaseView).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; Assert.AreEqual(fields, 6); - string serializedDatabaseView = JsonSerializer.Serialize(_databaseView, _options); - DatabaseView deserializedDatabaseView = JsonSerializer.Deserialize(serializedDatabaseView, _options)!; + string serializedDatabaseView = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDatabaseView.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDatabaseView, _options)!; + + DatabaseView deserializedDatabaseView = (DatabaseView)deserializedDict["person"]; Assert.AreEqual(deserializedDatabaseView.SourceType, _databaseView.SourceType); deserializedDatabaseView.Equals(_databaseView); @@ -348,14 +359,21 @@ public void TestDatabaseStoredProcedureSerializationDeserialization_WithDollarCo TestTypeNameChanges(_databaseStoredProcedure, "DatabaseStoredProcedure"); + Dictionary dict = new(); + dict.Add("person", _databaseStoredProcedure); + // Test to catch if there is change in number of properties/fields // Note: On Addition of property make sure it is added in following object creation _databaseStoredProcedure and include in serialization // and deserialization test. int fields = typeof(DatabaseStoredProcedure).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; Assert.AreEqual(fields, 6); - string serializedDatabaseSP = JsonSerializer.Serialize(_databaseStoredProcedure, _options); - DatabaseStoredProcedure deserializedDatabaseSP = JsonSerializer.Deserialize(serializedDatabaseSP, _options)!; + string serializedDatabaseSP = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDatabaseSP.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDatabaseSP, _options)!; + DatabaseStoredProcedure deserializedDatabaseSP = (DatabaseStoredProcedure)deserializedDict["person"]; Assert.AreEqual(deserializedDatabaseSP.SourceType, _databaseStoredProcedure.SourceType); deserializedDatabaseSP.Equals(_databaseStoredProcedure); From 2900d9024ea1db6d0c4175f18f440bc074ddaf2a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 13 Jan 2026 23:44:54 +0000 Subject: [PATCH 3/4] Initial plan From 2e12d18dc360474331d6f132b8aee451de4e8b90 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 14 Jan 2026 00:01:26 +0000 Subject: [PATCH 4/4] Implement robust double-encoding escaping mechanism with HashSet tracking Co-authored-by: Alekhya-Polavarapu <67075378+Alekhya-Polavarapu@users.noreply.github.com> --- .../Converters/DatabaseObjectConverter.cs | 55 ++++++- .../SerializationDeserializationTests.cs | 152 ++++++++++++++++++ 2 files changed, 203 insertions(+), 4 deletions(-) diff --git a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs index 6d625c7f9d..b24bfc19a6 100644 --- a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs +++ b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs @@ -62,6 +62,10 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri // "TypeName": "Azure.DataApiBuilder.Config.DatabasePrimitives.DatabaseTable, Azure.DataApiBuilder.Config", writer.WriteString(TYPE_NAME, GetTypeNameFromType(value.GetType())); + // Track SourceDefinition objects we've already escaped to avoid double-escaping + // (SourceDefinition and TableDefinition/ViewDefinition/StoredProcedureDefinition can reference the same object) + HashSet escapedSourceDefs = new HashSet(); + // Add other properties of DatabaseObject foreach (PropertyInfo prop in value.GetType().GetProperties()) { @@ -75,9 +79,14 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri object? propVal = prop.GetValue(value); // Only escape columns for properties whose type(derived type) is SourceDefinition. + // Use HashSet to avoid double-escaping when multiple properties reference the same SourceDefinition object. if (IsSourceDefinitionOrDerivedClassProperty(prop) && propVal is SourceDefinition sourceDef) { - EscapeDollaredColumns(sourceDef); + if (!escapedSourceDefs.Contains(sourceDef)) + { + EscapeDollaredColumns(sourceDef); + escapedSourceDefs.Add(sourceDef); + } } JsonSerializer.Serialize(writer, propVal, options); @@ -93,7 +102,11 @@ private static bool IsSourceDefinitionOrDerivedClassProperty(PropertyInfo prop) } /// - /// Escapes column keys that start with '$' to '_$' for serialization. + /// Escapes column keys that start with '$' or 'DAB_ESCAPE$' for serialization. + /// Uses a double-encoding approach to handle edge cases: + /// 1. First escapes columns starting with 'DAB_ESCAPE$' to 'DAB_ESCAPE$DAB_ESCAPE$...' + /// 2. Then escapes columns starting with '$' to 'DAB_ESCAPE$...' + /// This ensures that even if a column is named 'DAB_ESCAPE$xyz', it will be properly handled. /// private static void EscapeDollaredColumns(SourceDefinition sourceDef) { @@ -102,6 +115,21 @@ private static void EscapeDollaredColumns(SourceDefinition sourceDef) return; } + // Step 1: Escape columns that start with the escape sequence itself + // This prevents collision when a column name already contains 'DAB_ESCAPE$' + List keysStartingWithEscapeSequence = sourceDef.Columns.Keys + .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysStartingWithEscapeSequence) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = ESCAPED_DOLLARCHAR + key; + sourceDef.Columns[newKey] = col; + } + + // Step 2: Escape columns that start with '$' List keysToEscape = sourceDef.Columns.Keys .Where(k => k.StartsWith(DOLLAR_CHAR, StringComparison.Ordinal)) .ToList(); @@ -116,7 +144,10 @@ private static void EscapeDollaredColumns(SourceDefinition sourceDef) } /// - /// Unescapes column keys that start with '_$' to '$' for deserialization. + /// Unescapes column keys for deserialization using reverse double-encoding: + /// 1. First unescapes columns starting with 'DAB_ESCAPE$DAB_ESCAPE$' to 'DAB_ESCAPE$...' + /// 2. Then unescapes columns starting with 'DAB_ESCAPE$' to '$...' + /// This ensures proper reconstruction of original column names even in edge cases. /// private static void UnescapeDollaredColumns(SourceDefinition sourceDef) { @@ -125,6 +156,22 @@ private static void UnescapeDollaredColumns(SourceDefinition sourceDef) return; } + // Step 1: Unescape columns that were double-escaped (originally started with 'DAB_ESCAPE$') + string doubleEscapeSequence = ESCAPED_DOLLARCHAR + ESCAPED_DOLLARCHAR; + List doubleEscapedKeys = sourceDef.Columns.Keys + .Where(k => k.StartsWith(doubleEscapeSequence, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in doubleEscapedKeys) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + // Remove the first 'DAB_ESCAPE$' prefix + string newKey = key.Substring(ESCAPED_DOLLARCHAR.Length); + sourceDef.Columns[newKey] = col; + } + + // Step 2: Unescape columns that start with 'DAB_ESCAPE$' (originally started with '$') List keysToUnescape = sourceDef.Columns.Keys .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal)) .ToList(); @@ -133,7 +180,7 @@ private static void UnescapeDollaredColumns(SourceDefinition sourceDef) { ColumnDefinition col = sourceDef.Columns[key]; sourceDef.Columns.Remove(key); - string newKey = DOLLAR_CHAR + key[11..]; + string newKey = DOLLAR_CHAR + key[ESCAPED_DOLLARCHAR.Length..]; sourceDef.Columns[newKey] = col; } } diff --git a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs index 29735231c2..8a38f84ebf 100644 --- a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs @@ -596,5 +596,157 @@ private RelationShipPair GetRelationShipPair() }; return new(_databaseTable, table2); } + + /// + /// Tests the edge case where a column name starts with the escape sequence 'DAB_ESCAPE$'. + /// This validates the double-encoding mechanism: + /// - Original: 'DAB_ESCAPE$FirstName' -> Serialized: 'DAB_ESCAPE$DAB_ESCAPE$FirstName' -> Deserialized: 'DAB_ESCAPE$FirstName' + /// + [TestMethod] + public void TestDatabaseTableSerializationDeserialization_WithEscapeSequenceInColumnName() + { + // Create a column with the escape sequence in its name + ColumnDefinition columnDef = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test"), false); + string columnName = "DAB_ESCAPE$FirstName"; + SourceDefinition sourceDef = new() + { + IsInsertDMLTriggerEnabled = false, + IsUpdateDMLTriggerEnabled = false, + PrimaryKey = new List() { columnName }, + }; + sourceDef.Columns.Add(columnName, columnDef); + + DatabaseTable databaseTable = new DatabaseTable() + { + Name = "test_table", + SourceType = EntitySourceType.Table, + SchemaName = "dbo", + TableDefinition = sourceDef, + }; + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + } + }; + + // Serialize and verify double-escaping + string serialized = JsonSerializer.Serialize(databaseTable, _options); + Assert.IsTrue(serialized.Contains("DAB_ESCAPE$DAB_ESCAPE$FirstName"), + "Serialized JSON should contain double-escaped column name."); + + // Deserialize and verify the column name is correctly restored + DatabaseTable deserialized = JsonSerializer.Deserialize(serialized, _options)!; + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("DAB_ESCAPE$FirstName"), + "Deserialized table should have the original column name with escape sequence."); + Assert.AreEqual(1, deserialized.TableDefinition.Columns.Count, + "Should have exactly one column."); + } + + /// + /// Tests the edge case with multiple columns having different patterns: + /// - Regular column: 'RegularColumn' + /// - Dollar-prefixed column: '$DollarColumn' + /// - Escape-sequence-prefixed column: 'DAB_ESCAPE$EscapeColumn' + /// + [TestMethod] + public void TestDatabaseTableSerializationDeserialization_WithMixedColumnNames() + { + ColumnDefinition columnDef1 = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test1"), false); + ColumnDefinition columnDef2 = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test2"), false); + ColumnDefinition columnDef3 = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test3"), false); + + SourceDefinition sourceDef = new() + { + IsInsertDMLTriggerEnabled = false, + IsUpdateDMLTriggerEnabled = false, + PrimaryKey = new List() { "RegularColumn", "$DollarColumn", "DAB_ESCAPE$EscapeColumn" }, + }; + sourceDef.Columns.Add("RegularColumn", columnDef1); + sourceDef.Columns.Add("$DollarColumn", columnDef2); + sourceDef.Columns.Add("DAB_ESCAPE$EscapeColumn", columnDef3); + + DatabaseTable databaseTable = new DatabaseTable() + { + Name = "test_table", + SourceType = EntitySourceType.Table, + SchemaName = "dbo", + TableDefinition = sourceDef, + }; + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + } + }; + + // Serialize + string serialized = JsonSerializer.Serialize(databaseTable, _options); + + // Verify serialization patterns + Assert.IsTrue(serialized.Contains("RegularColumn"), + "Regular column should remain unchanged in serialization."); + Assert.IsTrue(serialized.Contains("DAB_ESCAPE$DollarColumn"), + "Dollar-prefixed column should be escaped."); + Assert.IsTrue(serialized.Contains("DAB_ESCAPE$DAB_ESCAPE$EscapeColumn"), + "Escape-sequence-prefixed column should be double-escaped."); + + // Deserialize and verify all columns are correctly restored + DatabaseTable deserialized = JsonSerializer.Deserialize(serialized, _options)!; + Assert.AreEqual(3, deserialized.TableDefinition.Columns.Count, "Should have three columns."); + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("RegularColumn"), + "Should have RegularColumn."); + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("$DollarColumn"), + "Should have $DollarColumn."); + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("DAB_ESCAPE$EscapeColumn"), + "Should have DAB_ESCAPE$EscapeColumn."); + } + + /// + /// Tests the edge case where a column name is exactly 'DAB_ESCAPE$'. + /// + [TestMethod] + public void TestDatabaseTableSerializationDeserialization_WithExactEscapeSequence() + { + ColumnDefinition columnDef = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test"), false); + string columnName = "DAB_ESCAPE$"; + SourceDefinition sourceDef = new() + { + IsInsertDMLTriggerEnabled = false, + IsUpdateDMLTriggerEnabled = false, + PrimaryKey = new List() { columnName }, + }; + sourceDef.Columns.Add(columnName, columnDef); + + DatabaseTable databaseTable = new DatabaseTable() + { + Name = "test_table", + SourceType = EntitySourceType.Table, + SchemaName = "dbo", + TableDefinition = sourceDef, + }; + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + } + }; + + // Serialize and deserialize + string serialized = JsonSerializer.Serialize(databaseTable, _options); + DatabaseTable deserialized = JsonSerializer.Deserialize(serialized, _options)!; + + // Verify the column name is correctly restored + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("DAB_ESCAPE$"), + "Deserialized table should have the exact escape sequence as column name."); + Assert.AreEqual(1, deserialized.TableDefinition.Columns.Count, + "Should have exactly one column."); + } } }