From 8726a5924e159e2b9ab4fd5d3a118edb3013d05b Mon Sep 17 00:00:00 2001 From: Alekhya-Polavarapu Date: Thu, 4 Dec 2025 10:56:29 -0800 Subject: [PATCH] Fix the Serialization/Deserialization issue with $ prefix columns (#2944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serialization and deserialization of metadata currently fail when column names are prefixed with the $ symbol. This issue occurs because we’ve enabled the ReferenceHandler flag in our System.Text.Json serialization settings. When this flag is active, the serializer treats $ as a reserved character used for special metadata (e.g., $id, $ref). As a result, any property name starting with $ is interpreted as metadata and cannot be deserialized properly. This update introduces custom logic in the converter’s Write and Read methods to handle $-prefixed column names safely. - During serialization, columns beginning with $ are escaped as "_$". - During deserialization, this transformation is reversed to restore the original property names. - [x] Unit tests --------- Co-authored-by: Aniruddh Munde Fix serialization for StoredProcedureDefinition inheritance (#3045) - To apply correct serialization and deserialization logic for stored procedures. With the previous changes, serialization was not working correctly for the StoredProcedureDefinition type, which extends SourceDefinition. When the value type was passed explicitly for serialization, the parent type was used instead, causing some child-type properties to be omitted. Instead of manually specifying the value type during serialization, this change allows the library to infer the type automatically and perform the correct serialization. - [x] Unit Tests --------- Co-authored-by: Aniruddh Munde Update src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Fix documentation typos in serialization comments (#3057) Code review identified typos in XML documentation comments that inaccurately described the escaping mechanism and contained spelling errors. Fixed documentation comments to accurately reflect the implementation: - **DatabaseObjectConverter.cs**: Updated escape/unescape method summaries to correctly describe the `DAB_ESCAPE$` prefix transformation (previously incorrectly documented as `_$`) - **SerializationDeserializationTests.cs**: Corrected "deserilization" → "deserialization" in three test method comments - [x] Documentation-only change, no functional changes --- 💬 We'd love your input! Share your thoughts on Copilot coding agent in our [2 minute survey](https://gh.io/copilot-coding-agent-survey). --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Alekhya-Polavarapu <67075378+Alekhya-Polavarapu@users.noreply.github.com> --- .../Converters/DatabaseObjectConverter.cs | 76 ++++++++++- .../SerializationDeserializationTests.cs | 119 +++++++++++++++++- 2 files changed, 188 insertions(+), 7 deletions(-) diff --git a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs index c6e694a394..a6a4c46d48 100644 --- a/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs +++ b/src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs @@ -17,6 +17,11 @@ namespace Azure.DataApiBuilder.Core.Services.MetadataProviders.Converters public class DatabaseObjectConverter : JsonConverter { private const string TYPE_NAME = "TypeName"; + private const string DOLLAR_CHAR = "$"; + + // ``DAB_ESCAPE$`` is used to escape column names that start with `$` during serialization. + // It is chosen to be unique enough to avoid collisions with actual column names. + private const string ESCAPED_DOLLARCHAR = "DAB_ESCAPE$"; public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -29,6 +34,15 @@ public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConver DatabaseObject objA = (DatabaseObject)JsonSerializer.Deserialize(document, concreteType, options)!; + foreach (PropertyInfo prop in objA.GetType().GetProperties().Where(IsSourceDefinitionOrDerivedClassProperty)) + { + SourceDefinition? sourceDef = (SourceDefinition?)prop.GetValue(objA); + if (sourceDef is not null) + { + UnescapeDollaredColumns(sourceDef); + } + } + return objA; } } @@ -58,12 +72,72 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri } writer.WritePropertyName(prop.Name); - JsonSerializer.Serialize(writer, prop.GetValue(value), options); + object? propVal = prop.GetValue(value); + + // Only escape columns for properties whose type(derived type) is SourceDefinition. + if (IsSourceDefinitionOrDerivedClassProperty(prop) && propVal is SourceDefinition sourceDef) + { + EscapeDollaredColumns(sourceDef); + } + + JsonSerializer.Serialize(writer, propVal, options); } writer.WriteEndObject(); } + private static bool IsSourceDefinitionOrDerivedClassProperty(PropertyInfo prop) + { + // Return true for properties whose type is SourceDefinition or any class derived from SourceDefinition + return typeof(SourceDefinition).IsAssignableFrom(prop.PropertyType); + } + + /// + /// Escapes column keys that start with '$' by prefixing them with 'DAB_ESCAPE$' for serialization. + /// + private static void EscapeDollaredColumns(SourceDefinition sourceDef) + { + if (sourceDef.Columns is null || sourceDef.Columns.Count == 0) + { + return; + } + + List keysToEscape = sourceDef.Columns.Keys + .Where(k => k.StartsWith(DOLLAR_CHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysToEscape) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = ESCAPED_DOLLARCHAR + key[1..]; + sourceDef.Columns[newKey] = col; + } + } + + /// + /// Unescapes column keys that start with 'DAB_ESCAPE$' by removing the prefix and restoring the original '$' for deserialization. + /// + private static void UnescapeDollaredColumns(SourceDefinition sourceDef) + { + if (sourceDef.Columns is null || sourceDef.Columns.Count == 0) + { + return; + } + + List keysToUnescape = sourceDef.Columns.Keys + .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal)) + .ToList(); + + foreach (string key in keysToUnescape) + { + ColumnDefinition col = sourceDef.Columns[key]; + sourceDef.Columns.Remove(key); + string newKey = DOLLAR_CHAR + key[ESCAPED_DOLLARCHAR.Length..]; + sourceDef.Columns[newKey] = col; + } + } + private static Type GetTypeFromName(string typeName) { Type? type = Type.GetType(typeName); diff --git a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs index 2b5e5bf3ba..10f43f45eb 100644 --- a/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/UnitTests/SerializationDeserializationTests.cs @@ -276,8 +276,114 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization() VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "FirstName"); } - private void InitializeObjects() + /// + /// Validates serialization and deserialization of Dictionary containing DatabaseTable + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDictionaryDatabaseObjectSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + _options = new() + { + Converters = { + new DatabaseObjectConverter(), + new TypeConverter() + }, + ReferenceHandler = ReferenceHandler.Preserve + }; + + Dictionary dict = new() { { "person", _databaseTable } }; + + string serializedDict = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDict.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDict, _options)!; + DatabaseTable deserializedDatabaseTable = (DatabaseTable)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseTable.SourceType, _databaseTable.SourceType); + Assert.AreEqual(deserializedDatabaseTable.FullName, _databaseTable.FullName); + deserializedDatabaseTable.Equals(_databaseTable); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.SourceDefinition, _databaseTable.SourceDefinition, "$FirstName"); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "$FirstName"); + } + + /// + /// Validates serialization and deserialization of Dictionary containing DatabaseView + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDatabaseViewSerializationDeserialization_WithDollarColumn() + { + InitializeObjects(generateDollaredColumn: true); + + TestTypeNameChanges(_databaseView, "DatabaseView"); + + Dictionary dict = new(); + dict.Add("person", _databaseView); + + // Test to catch if there is change in number of properties/fields + // Note: On Addition of property make sure it is added in following object creation _databaseView and include in serialization + // and deserialization test. + int fields = typeof(DatabaseView).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; + Assert.AreEqual(fields, 6); + + string serializedDatabaseView = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDatabaseView.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDatabaseView, _options)!; + + DatabaseView deserializedDatabaseView = (DatabaseView)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseView.SourceType, _databaseView.SourceType); + deserializedDatabaseView.Equals(_databaseView); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.SourceDefinition, _databaseView.SourceDefinition, "$FirstName"); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.ViewDefinition, _databaseView.ViewDefinition, "$FirstName"); + } + + /// + /// Validates serialization and deserialization of Dictionary containing DatabaseStoredProcedure + /// The table will have dollar sign prefix ($) in the column name + /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict. + /// + [TestMethod] + public void TestDatabaseStoredProcedureSerializationDeserialization_WithDollarColumn() { + InitializeObjects(generateDollaredColumn: true); + + TestTypeNameChanges(_databaseStoredProcedure, "DatabaseStoredProcedure"); + + Dictionary dict = new(); + dict.Add("person", _databaseStoredProcedure); + + // Test to catch if there is change in number of properties/fields + // Note: On Addition of property make sure it is added in following object creation _databaseStoredProcedure and include in serialization + // and deserialization test. + int fields = typeof(DatabaseStoredProcedure).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length; + Assert.AreEqual(fields, 6); + + string serializedDatabaseSP = JsonSerializer.Serialize(dict, _options); + // Assert that the serialized JSON contains the escaped dollar sign in column name + Assert.IsTrue(serializedDatabaseSP.Contains("DAB_ESCAPE$FirstName"), + "Serialized JSON should contain the dollar-prefixed column name in SourceDefinition's Columns."); + Dictionary deserializedDict = JsonSerializer.Deserialize>(serializedDatabaseSP, _options)!; + DatabaseStoredProcedure deserializedDatabaseSP = (DatabaseStoredProcedure)deserializedDict["person"]; + + Assert.AreEqual(deserializedDatabaseSP.SourceType, _databaseStoredProcedure.SourceType); + deserializedDatabaseSP.Equals(_databaseStoredProcedure); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.SourceDefinition, _databaseStoredProcedure.SourceDefinition, "$FirstName", true); + VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.StoredProcedureDefinition, _databaseStoredProcedure.StoredProcedureDefinition, "$FirstName", true); + } + + private void InitializeObjects(bool generateDollaredColumn = false) + { + string columnName = generateDollaredColumn ? "$FirstName" : "FirstName"; _options = new() { // ObjectConverter behavior different in .NET8 most likely due to @@ -289,10 +395,11 @@ private void InitializeObjects() new DatabaseObjectConverter(), new TypeConverter() } + }; _columnDefinition = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("John"), false); - _sourceDefinition = GetSourceDefinition(false, false, new List() { "FirstName" }, _columnDefinition); + _sourceDefinition = GetSourceDefinition(false, false, new List() { columnName }, _columnDefinition); _databaseTable = new DatabaseTable() { @@ -311,10 +418,10 @@ private void InitializeObjects() { IsInsertDMLTriggerEnabled = false, IsUpdateDMLTriggerEnabled = false, - PrimaryKey = new List() { "FirstName" }, + PrimaryKey = new List() { columnName }, }, }; - _databaseView.ViewDefinition.Columns.Add("FirstName", _columnDefinition); + _databaseView.ViewDefinition.Columns.Add(columnName, _columnDefinition); _parameterDefinition = new() { @@ -331,10 +438,10 @@ private void InitializeObjects() SourceType = EntitySourceType.StoredProcedure, StoredProcedureDefinition = new() { - PrimaryKey = new List() { "FirstName" }, + PrimaryKey = new List() { columnName }, } }; - _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add("FirstName", _columnDefinition); + _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add(columnName, _columnDefinition); _databaseStoredProcedure.StoredProcedureDefinition.Parameters.Add("Id", _parameterDefinition); }