diff --git a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs index c6e694a394..b24bfc19a6 100644 --- a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs +++ b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs @@ -17,6 +17,11 @@ namespace Azure.DataApiBuilder.Core.Services.MetadataProviders.Converters public class DatabaseObjectConverter : JsonConverter { private const string TYPE_NAME = "TypeName"; + private const string DOLLAR_CHAR = "$"; + + // ``DAB_ESCAPE$`` is used to escape column names that start with `$` during serialization. + // It is chosen to be unique enough to avoid collisions with actual column names. + private const string ESCAPED_DOLLARCHAR = "DAB_ESCAPE$"; public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -29,6 +34,15 @@ public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConver DatabaseObject objA = (DatabaseObject)JsonSerializer.Deserialize(document, concreteType, options)!; + foreach (PropertyInfo prop in objA.GetType().GetProperties().Where(IsSourceDefinitionOrDerivedClassProperty)) + { + SourceDefinition? sourceDef = (SourceDefinition?)prop.GetValue(objA); + if (sourceDef is not null) + { + UnescapeDollaredColumns(sourceDef); + } + } + return objA; } } @@ -48,6 +62,10 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri // "TypeName": "Azure.DataApiBuilder.Config.DatabasePrimitives.DatabaseTable, Azure.DataApiBuilder.Config", writer.WriteString(TYPE_NAME, GetTypeNameFromType(value.GetType())); + // Track SourceDefinition objects we've already escaped to avoid double-escaping + // (SourceDefinition and TableDefinition/ViewDefinition/StoredProcedureDefinition can reference the same object) + HashSet escapedSourceDefs = new HashSet(); + // Add other properties of DatabaseObject foreach (PropertyInfo prop in value.GetType().GetProperties()) { @@ -58,12 +76,115 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri } writer.WritePropertyName(prop.Name); - JsonSerializer.Serialize(writer, prop.GetValue(value), options); + object? propVal = prop.GetValue(value); + + // Only escape columns for properties whose type(derived type) is SourceDefinition. + // Use HashSet to avoid double-escaping when multiple properties reference the same SourceDefinition object. + if (IsSourceDefinitionOrDerivedClassProperty(prop) && propVal is SourceDefinition sourceDef) + { + if (!escapedSourceDefs.Contains(sourceDef)) + { + EscapeDollaredColumns(sourceDef); + escapedSourceDefs.Add(sourceDef); + } + } + + JsonSerializer.Serialize(writer, propVal, options); } writer.WriteEndObject(); } + private static bool IsSourceDefinitionOrDerivedClassProperty(PropertyInfo prop) + { + // Return true for properties whose type is SourceDefinition or any class derived from SourceDefinition + return typeof(SourceDefinition).IsAssignableFrom(prop.PropertyType); + } + + /// + /// Escapes column keys that start with '$' or 'DAB_ESCAPE$' for serialization. + /// Uses a double-encoding approach to handle edge cases: + /// 1. First escapes columns starting with 'DAB_ESCAPE$' to 'DAB_ESCAPE$DAB_ESCAPE$...' + /// 2. Then escapes columns starting with '$' to 'DAB_ESCAPE$...' + /// This ensures that even if a column is named 'DAB_ESCAPE$xyz', it will be properly handled. + /// + private static void EscapeDollaredColumns(SourceDefinition sourceDef) + { + if (sourceDef.Columns is null || sourceDef.Columns.Count == 0) + { + return; + } + + // Step 1: Escape columns that start with the escape sequence itself + // This prevents collision when a column name already contains 'DAB_ESCAPE$' + List keysStartingWithEscapeSequence = sourceDef.Columns.Keys + .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysStartingWithEscapeSequence) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = ESCAPED_DOLLARCHAR + key; + sourceDef.Columns[newKey] = col; + } + + // Step 2: Escape columns that start with '$' + List keysToEscape = sourceDef.Columns.Keys + .Where(k => k.StartsWith(DOLLAR_CHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysToEscape) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = ESCAPED_DOLLARCHAR + key[1..]; + sourceDef.Columns[newKey] = col; + } + } + + /// + /// Unescapes column keys for deserialization using reverse double-encoding: + /// 1. First unescapes columns starting with 'DAB_ESCAPE$DAB_ESCAPE$' to 'DAB_ESCAPE$...' + /// 2. Then unescapes columns starting with 'DAB_ESCAPE$' to '$...' + /// This ensures proper reconstruction of original column names even in edge cases. + /// + private static void UnescapeDollaredColumns(SourceDefinition sourceDef) + { + if (sourceDef.Columns is null || sourceDef.Columns.Count == 0) + { + return; + } + + // Step 1: Unescape columns that were double-escaped (originally started with 'DAB_ESCAPE$') + string doubleEscapeSequence = ESCAPED_DOLLARCHAR + ESCAPED_DOLLARCHAR; + List doubleEscapedKeys = sourceDef.Columns.Keys + .Where(k => k.StartsWith(doubleEscapeSequence, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in doubleEscapedKeys) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + // Remove the first 'DAB_ESCAPE$' prefix + string newKey = key.Substring(ESCAPED_DOLLARCHAR.Length); + sourceDef.Columns[newKey] = col; + } + + // Step 2: Unescape columns that start with 'DAB_ESCAPE$' (originally started with '$') + List keysToUnescape = sourceDef.Columns.Keys + .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysToUnescape) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = DOLLAR_CHAR + key[ESCAPED_DOLLARCHAR.Length..]; + sourceDef.Columns[newKey] = col; + } + } + private static Type GetTypeFromName(string typeName) { Type? type = Type.GetType(typeName); diff --git a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs index 2b5e5bf3ba..8a38f84ebf 100644 --- a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs @@ -276,8 +276,114 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization() VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "FirstName"); } - private void InitializeObjects() + /// + /// Validates serialization and deserilization of Dictionary containing DatabaseTable + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDictionaryDatabaseObjectSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + }, + ReferenceHandler = ReferenceHandler.Preserve + }; + + Dictionary dict = new() { { "person", _databaseTable } }; + + string serializedDict = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDict.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDict, _options)!; + DatabaseTable deserializedDatabaseTable = (DatabaseTable)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseTable.SourceType, _databaseTable.SourceType); + Assert.AreEqual(deserializedDatabaseTable.FullName, _databaseTable.FullName); + deserializedDatabaseTable.Equals(_databaseTable); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.SourceDefinition, _databaseTable.SourceDefinition, "$FirstName"); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "$FirstName"); + } + + /// + /// Validates serialization and deserilization of Dictionary containing DatabaseView + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDatabaseViewSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + TestTypeNameChanges(_databaseView, "DatabaseView"); + + Dictionary dict = new(); + dict.Add("person", _databaseView); + + // Test to catch if there is change in number of properties/fields + // Note: On Addition of property make sure it is added in following object creation _databaseView and include in serialization + // and deserialization test. + int fields = typeof(DatabaseView).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; + Assert.AreEqual(fields, 6); + + string serializedDatabaseView = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDatabaseView.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDatabaseView, _options)!; + + DatabaseView deserializedDatabaseView = (DatabaseView)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseView.SourceType, _databaseView.SourceType); + deserializedDatabaseView.Equals(_databaseView); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.SourceDefinition, _databaseView.SourceDefinition, "$FirstName"); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.ViewDefinition, _databaseView.ViewDefinition, "$FirstName"); + } + + /// + /// Validates serialization and deserilization of Dictionary containing DatabaseStoredProcedure + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDatabaseStoredProcedureSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + TestTypeNameChanges(_databaseStoredProcedure, "DatabaseStoredProcedure"); + + Dictionary dict = new(); + dict.Add("person", _databaseStoredProcedure); + + // Test to catch if there is change in number of properties/fields + // Note: On Addition of property make sure it is added in following object creation _databaseStoredProcedure and include in serialization + // and deserialization test. + int fields = typeof(DatabaseStoredProcedure).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; + Assert.AreEqual(fields, 6); + + string serializedDatabaseSP = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDatabaseSP.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDatabaseSP, _options)!; + DatabaseStoredProcedure deserializedDatabaseSP = (DatabaseStoredProcedure)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseSP.SourceType, _databaseStoredProcedure.SourceType); + deserializedDatabaseSP.Equals(_databaseStoredProcedure); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.SourceDefinition, _databaseStoredProcedure.SourceDefinition, "$FirstName", true); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.StoredProcedureDefinition, _databaseStoredProcedure.StoredProcedureDefinition, "$FirstName", true); + } + + private void InitializeObjects(bool generateDollaredColumn = false) { + string columnName = generateDollaredColumn ? "$FirstName" : "FirstName"; _options = new() { // ObjectConverter behavior different in .NET8 most likely due to @@ -289,10 +395,11 @@ private void InitializeObjects() new DatabaseObjectConverter(), new TypeConverter() } + }; _columnDefinition = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("John"), false); - _sourceDefinition = GetSourceDefinition(false, false, new List() { "FirstName" }, _columnDefinition); + _sourceDefinition = GetSourceDefinition(false, false, new List() { columnName }, _columnDefinition); _databaseTable = new DatabaseTable() { @@ -311,10 +418,10 @@ private void InitializeObjects() { IsInsertDMLTriggerEnabled = false, IsUpdateDMLTriggerEnabled = false, - PrimaryKey = new List() { "FirstName" }, + PrimaryKey = new List() { columnName }, }, }; - _databaseView.ViewDefinition.Columns.Add("FirstName", _columnDefinition); + _databaseView.ViewDefinition.Columns.Add(columnName, _columnDefinition); _parameterDefinition = new() { @@ -331,10 +438,10 @@ private void InitializeObjects() SourceType = EntitySourceType.StoredProcedure, StoredProcedureDefinition = new() { - PrimaryKey = new List() { "FirstName" }, + PrimaryKey = new List() { columnName }, } }; - _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add("FirstName", _columnDefinition); + _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add(columnName, _columnDefinition); _databaseStoredProcedure.StoredProcedureDefinition.Parameters.Add("Id", _parameterDefinition); } @@ -489,5 +596,157 @@ private RelationShipPair GetRelationShipPair() }; return new(_databaseTable, table2); } + + /// + /// Tests the edge case where a column name starts with the escape sequence 'DAB_ESCAPE$'. + /// This validates the double-encoding mechanism: + /// - Original: 'DAB_ESCAPE$FirstName' -> Serialized: 'DAB_ESCAPE$DAB_ESCAPE$FirstName' -> Deserialized: 'DAB_ESCAPE$FirstName' + /// + [TestMethod] + public void TestDatabaseTableSerializationDeserialization_WithEscapeSequenceInColumnName() + { + // Create a column with the escape sequence in its name + ColumnDefinition columnDef = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test"), false); + string columnName = "DAB_ESCAPE$FirstName"; + SourceDefinition sourceDef = new() + { + IsInsertDMLTriggerEnabled = false, + IsUpdateDMLTriggerEnabled = false, + PrimaryKey = new List() { columnName }, + }; + sourceDef.Columns.Add(columnName, columnDef); + + DatabaseTable databaseTable = new DatabaseTable() + { + Name = "test_table", + SourceType = EntitySourceType.Table, + SchemaName = "dbo", + TableDefinition = sourceDef, + }; + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + } + }; + + // Serialize and verify double-escaping + string serialized = JsonSerializer.Serialize(databaseTable, _options); + Assert.IsTrue(serialized.Contains("DAB_ESCAPE$DAB_ESCAPE$FirstName"), + "Serialized JSON should contain double-escaped column name."); + + // Deserialize and verify the column name is correctly restored + DatabaseTable deserialized = JsonSerializer.Deserialize(serialized, _options)!; + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("DAB_ESCAPE$FirstName"), + "Deserialized table should have the original column name with escape sequence."); + Assert.AreEqual(1, deserialized.TableDefinition.Columns.Count, + "Should have exactly one column."); + } + + /// + /// Tests the edge case with multiple columns having different patterns: + /// - Regular column: 'RegularColumn' + /// - Dollar-prefixed column: '$DollarColumn' + /// - Escape-sequence-prefixed column: 'DAB_ESCAPE$EscapeColumn' + /// + [TestMethod] + public void TestDatabaseTableSerializationDeserialization_WithMixedColumnNames() + { + ColumnDefinition columnDef1 = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test1"), false); + ColumnDefinition columnDef2 = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test2"), false); + ColumnDefinition columnDef3 = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test3"), false); + + SourceDefinition sourceDef = new() + { + IsInsertDMLTriggerEnabled = false, + IsUpdateDMLTriggerEnabled = false, + PrimaryKey = new List() { "RegularColumn", "$DollarColumn", "DAB_ESCAPE$EscapeColumn" }, + }; + sourceDef.Columns.Add("RegularColumn", columnDef1); + sourceDef.Columns.Add("$DollarColumn", columnDef2); + sourceDef.Columns.Add("DAB_ESCAPE$EscapeColumn", columnDef3); + + DatabaseTable databaseTable = new DatabaseTable() + { + Name = "test_table", + SourceType = EntitySourceType.Table, + SchemaName = "dbo", + TableDefinition = sourceDef, + }; + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + } + }; + + // Serialize + string serialized = JsonSerializer.Serialize(databaseTable, _options); + + // Verify serialization patterns + Assert.IsTrue(serialized.Contains("RegularColumn"), + "Regular column should remain unchanged in serialization."); + Assert.IsTrue(serialized.Contains("DAB_ESCAPE$DollarColumn"), + "Dollar-prefixed column should be escaped."); + Assert.IsTrue(serialized.Contains("DAB_ESCAPE$DAB_ESCAPE$EscapeColumn"), + "Escape-sequence-prefixed column should be double-escaped."); + + // Deserialize and verify all columns are correctly restored + DatabaseTable deserialized = JsonSerializer.Deserialize(serialized, _options)!; + Assert.AreEqual(3, deserialized.TableDefinition.Columns.Count, "Should have three columns."); + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("RegularColumn"), + "Should have RegularColumn."); + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("$DollarColumn"), + "Should have $DollarColumn."); + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("DAB_ESCAPE$EscapeColumn"), + "Should have DAB_ESCAPE$EscapeColumn."); + } + + /// + /// Tests the edge case where a column name is exactly 'DAB_ESCAPE$'. + /// + [TestMethod] + public void TestDatabaseTableSerializationDeserialization_WithExactEscapeSequence() + { + ColumnDefinition columnDef = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("Test"), false); + string columnName = "DAB_ESCAPE$"; + SourceDefinition sourceDef = new() + { + IsInsertDMLTriggerEnabled = false, + IsUpdateDMLTriggerEnabled = false, + PrimaryKey = new List() { columnName }, + }; + sourceDef.Columns.Add(columnName, columnDef); + + DatabaseTable databaseTable = new DatabaseTable() + { + Name = "test_table", + SourceType = EntitySourceType.Table, + SchemaName = "dbo", + TableDefinition = sourceDef, + }; + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + } + }; + + // Serialize and deserialize + string serialized = JsonSerializer.Serialize(databaseTable, _options); + DatabaseTable deserialized = JsonSerializer.Deserialize(serialized, _options)!; + + // Verify the column name is correctly restored + Assert.IsTrue(deserialized.TableDefinition.Columns.ContainsKey("DAB_ESCAPE$"), + "Deserialized table should have the exact escape sequence as column name."); + Assert.AreEqual(1, deserialized.TableDefinition.Columns.Count, + "Should have exactly one column."); + } } }