Sync Repos: Release 170.128.0

CONTRIBUTING.md

@@ -106,14 +106,36 @@ Example: To run all priority 0 tests
 dotnet test --filter Priority=0
 ```
 
+#### ⚠️ CRITICAL: Full Test Suite for Parser Changes
+
+**If you make ANY changes to grammar files (`.g` files) or AST definitions (`Ast.xml`), you MUST run the complete test suite** to ensure no regressions:
+
+```cmd
+dotnet test Test/SqlDom/UTSqlScriptDom.csproj -c Debug
+```
+
+**Why this is critical for parser changes:**
+- Grammar changes can have far-reaching effects on seemingly unrelated functionality
+- Shared grammar rules are used in multiple contexts throughout the parser
+- AST modifications can affect script generation and visitor patterns across the entire codebase
+- Token recognition changes can impact parsing of statements that don't even use the modified feature
+
+**Example of unexpected failures:**
+- Modifying a shared rule like `identifierColumnReferenceExpression` can cause other tests to fail because the rule now accepts syntax that should be rejected in different contexts
+- Changes to operator precedence can affect unrelated expressions
+- Adding new AST members without proper script generation support can break round-trip parsing
+
+Always verify that all ~557 tests pass before submitting your changes.
+
 ### Pull Request Process
 
 Before sending a Pull Request, please do the following:
 
-1. Ensure builds are still successful and tests, including any added or updated tests, pass prior to submitting the pull request.
-2. Update any documentation, user and contributor, that is impacted by your changes.
-3. Include your change description in `CHANGELOG.md` file as part of pull request.
-4. You may merge the pull request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you.
+1. **For parser changes (grammar/AST modifications): Run the complete test suite** (`dotnet test Test/SqlDom/UTSqlScriptDom.csproj -c Debug`) and ensure all ~557 tests pass. Grammar changes can have unexpected side effects.
+2. Ensure builds are still successful and tests, including any added or updated tests, pass prior to submitting the pull request.
+3. Update any documentation, user and contributor, that is impacted by your changes.
+4. Include your change description in `CHANGELOG.md` file as part of pull request.
+5. You may merge the pull request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you.
 
 ### Helpful notes for SQLDOM extensions
 

SqlScriptDom/Parser/TSql/Ast.xml

@@ -652,8 +652,9 @@
     <Member Name="IgnoreRespectNulls" Type="Identifier"  Collection="true" Summary="The ignore nulls is used to eliminate rows with null value in the output. Optional may be null."/>
     <Member Name="TrimOptions" Type="Identifier" Summary="TRIM intrinsic can take optional arguments like 'Leading', 'Trailing' or 'Both'."/>
     <Member Name="JsonParameters" Type="JsonKeyValue" Collection="true" Summary="The Json parameters to the function."/>
+    <Member Name="JsonOrderByClause" Type="OrderByClause" Summary="The order by clause used for JSON aggregation functions like JSON_ARRAYAGG. Optional may be null."/>
     <Member Name="AbsentOrNullOnNull" Type="Identifier"  Collection="true" Summary="The Absent or Null on Null will convert or remove sql null to json null"/>
-    <Member Name="ReturnType" Type="Identifier"  Collection="true" Summary="Return type of function. Used by json_arrayagg, json_objectagg, json_array, json_object and json_value"/>
+    <Member Name="ReturnType" Type="DataTypeReference"  Collection="true" Summary="Return type of function. Used by json_arrayagg, json_objectagg, json_array, json_object and json_value"/>
     <Member Name="WithArrayWrapper" Type="bool" Summary="Indicates whether WITH ARRAY WRAPPER clause is specified for JSON_QUERY function."/>
   </Class>
   <Class Name="CallTarget" Abstract="true" Summary="Represents a target of a function call.">

SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs

@@ -527,6 +527,7 @@ internal static class CodeGenerationSupporter
         internal const string JsonObjectAgg = "JSON_OBJECTAGG";
         internal const string JsonArrayAgg = "JSON_ARRAYAGG";
         internal const string JsonQuery = "JSON_QUERY";
+        internal const string JsonValue = "JSON_VALUE";
         internal const string Array = "ARRAY";
         internal const string Wrapper = "WRAPPER";
         internal const string Keep = "KEEP";

SqlScriptDom/Parser/TSql/TSql170.g

@@ -32631,22 +32631,86 @@ expressionList [TSqlFragment vParent, IList<ScalarExpression> expressions]
         )
     ;
 
+/* jsonReturningClause is used by json_object, json_objectagg, json_array, json_arrayagg where only 
+ RETURNING JSON  is supported. Any other type with JSON should return in error */
 jsonReturningClause [FunctionCall vParent]
 {
-    Identifier vJson;
+    DataTypeReference vDataType;
 }
 :
-    tReturning:Identifier tJson:Identifier
+    tReturning:Identifier vDataType=jsonDataType
     {
         Match(tReturning, CodeGenerationSupporter.Returning);
-        Match(tJson, CodeGenerationSupporter.Json);
-        UpdateTokenInfo(vParent,tJson);
-        vJson = FragmentFactory.CreateFragment<Identifier>();
-        AddAndUpdateTokenInfo(vParent, vParent.ReturnType, vJson);
-        vJson.SetUnquotedIdentifier(tJson.getText());
+        UpdateTokenInfo(vParent, tReturning);
+        vParent.ReturnType.Add(vDataType);
+    }
+;
+
+jsonDataType returns [SqlDataTypeReference vResult = null]
+{
+    SchemaObjectName vJsonTypeName;
+}
+:
+    vJsonTypeName=schemaObjectTwoPartName
+    {
+        // Only allow JSON as the data type
+        if (vJsonTypeName.BaseIdentifier.Value.ToUpper(CultureInfo.InvariantCulture) != CodeGenerationSupporter.Json)
+        {
+            ThrowParseErrorException("SQL46005", vJsonTypeName,
+                TSqlParserResource.SQL46005Message, CodeGenerationSupporter.Json, vJsonTypeName.BaseIdentifier.Value);
+        }
+
+        vResult = FragmentFactory.CreateFragment<SqlDataTypeReference>();
+        vResult.Name = vJsonTypeName;
+        vResult.SqlDataTypeOption = SqlDataTypeOption.Json;
+        vResult.UpdateTokenInfo(vJsonTypeName);
+    }
+;
+
+/* jsonValueReturningClause is used by json_value. Only json_value support RETURNING <SQL-TYPE> syntax*/
+jsonValueReturningClause [FunctionCall vParent]
+{
+    DataTypeReference vDataType;
+}
+:
+    tReturning:Identifier vDataType=scalarDataType
+    {
+        Match(tReturning, CodeGenerationSupporter.Returning);
+        UpdateTokenInfo(vParent, tReturning);
+
+        // JSON_VALUE only supports specific SQL data types in RETURNING clause
+        if (vDataType is SqlDataTypeReference sqlDataType)
+        {
+            bool isAllowedType = sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Int ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.TinyInt ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.SmallInt ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.BigInt ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Float ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Real ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Decimal ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Numeric ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Bit ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.VarChar ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.NVarChar ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Char ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.NChar ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Date ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.Time ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.DateTime2 ||
+                                sqlDataType.SqlDataTypeOption == SqlDataTypeOption.DateTimeOffset;
+
+            if (!isAllowedType)
+            {
+                ThrowParseErrorException("SQL46005", vDataType,
+                    TSqlParserResource.SQL46005Message, "supported data type", sqlDataType.SqlDataTypeOption.ToString());
+            }
+        }
+
+        vParent.ReturnType.Add(vDataType);
     }
 ;
 
+
 jsonKeyValueExpression returns [JsonKeyValue vResult = FragmentFactory.CreateFragment<JsonKeyValue>()]
 {
     ScalarExpression vKey;
@@ -32970,6 +33034,9 @@ builtInFunctionCall returns [FunctionCall vResult = FragmentFactory.CreateFragme
         |
         {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonQuery)}?
             jsonQueryBuiltInFunctionCall[vResult]
+        |
+        {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonValue)}?
+            jsonValueBuiltInFunctionCall[vResult]
         |
          {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.Trim) && 
           (NextTokenMatches(CodeGenerationSupporter.Leading) | NextTokenMatches(CodeGenerationSupporter.Trailing) | NextTokenMatches(CodeGenerationSupporter.Both))}?
@@ -33017,13 +33084,22 @@ jsonArrayBuiltInFunctionCall [FunctionCall vParent]
 jsonArrayAggBuiltInFunctionCall [FunctionCall vParent]
 {
     ScalarExpression vExpression;
+    OrderByClause vOrderByClause;
 }
     :   (
            vExpression=expression
            {
                AddAndUpdateTokenInfo(vParent, vParent.Parameters, vExpression);
            }
         )
+        (
+           vOrderByClause=orderByClause
+           {
+               vParent.JsonOrderByClause = vOrderByClause;
+           }
+        |
+            /* empty */
+        )
         (
            jsonNullClauseFunction[vParent]
         |
@@ -33085,10 +33161,6 @@ jsonQueryBuiltInFunctionCall [FunctionCall vParent]
                 AddAndUpdateTokenInfo(vParent, vParent.Parameters, vPath);
             }
         )?
-        tRParen:RightParenthesis
-        {
-            UpdateTokenInfo(vParent, tRParen);
-        }
         (
             With tArray:Identifier tWrapper:Identifier
             {
@@ -33103,6 +33175,34 @@ jsonQueryBuiltInFunctionCall [FunctionCall vParent]
                 vParent.WithArrayWrapper = true;
             }
         )?
+        tRParen:RightParenthesis
+        {
+            UpdateTokenInfo(vParent, tRParen);
+        }
+    ;
+
+jsonValueBuiltInFunctionCall [FunctionCall vParent]
+{
+    ScalarExpression vExpression;
+    ScalarExpression vPath;
+}
+    :   vExpression=expression
+        {
+            AddAndUpdateTokenInfo(vParent, vParent.Parameters, vExpression);
+        }
+        Comma vPath=expression
+        {
+            AddAndUpdateTokenInfo(vParent, vParent.Parameters, vPath);
+        }
+        (
+            jsonValueReturningClause[vParent]
+        |
+            /* empty */
+        )
+        tRParen:RightParenthesis
+        {
+			UpdateTokenInfo(vParent, tRParen);
+        }
     ;
 
 regularBuiltInFunctionCall [FunctionCall vParent]

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs

@@ -85,6 +85,8 @@ public override void ExplicitVisit(FunctionCall node)
 			else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonArrayAgg)
             {
                 GenerateCommaSeparatedList(node.Parameters);
+                // Generate ORDER BY clause if present
+                GenerateSpaceAndFragmentIfNotNull(node.JsonOrderByClause);
                 if (node.Parameters?.Count > 0 && node?.AbsentOrNullOnNull?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them
                     GenerateSpace();
                 GenerateNullOnNullOrAbsentOnNull(node?.AbsentOrNullOnNull);
@@ -96,9 +98,8 @@ public override void ExplicitVisit(FunctionCall node)
             else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonQuery)
             {
                 GenerateCommaSeparatedList(node.Parameters);
-                GenerateSymbol(TSqlTokenType.RightParenthesis);
 
-                // Handle WITH ARRAY WRAPPER clause
+                // Handle WITH ARRAY WRAPPER clause - inside parentheses
                 if (node.WithArrayWrapper)
                 {
                     GenerateSpace();
@@ -108,6 +109,18 @@ public override void ExplicitVisit(FunctionCall node)
                     GenerateSpace();
                     GenerateIdentifier(CodeGenerationSupporter.Wrapper);
                 }
+
+                GenerateSymbol(TSqlTokenType.RightParenthesis);
+            }
+            else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonValue)
+            {
+                GenerateCommaSeparatedList(node.Parameters);
+                if (node.ReturnType?.Count > 0) //If there are return types then generate space and return type clause
+                {
+                    GenerateSpace();
+                    GenerateReturnType(node?.ReturnType);
+                }
+                GenerateSymbol(TSqlTokenType.RightParenthesis);
             }
             else
             {
@@ -161,13 +174,47 @@ private void GenerateNullOnNullOrAbsentOnNull(IList<Identifier> list)
                 GenerateKeyword(TSqlTokenType.Null);
             }
         }
-        private void GenerateReturnType(IList<Identifier> list)
+
+        // Generate returning clause with SQLType.
+        private void GenerateReturnType(IList<DataTypeReference> list)
         {
-            if (list?.Count > 0 && list[0].Value?.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.Json)
+            if (list?.Count > 0)
             {
                 GenerateIdentifier("RETURNING");
                 GenerateSpace();
-                GenerateSpaceSeparatedList(list);
+
+                // Generate each data type correctly
+                for (int i = 0; i < list.Count; i++)
+                {
+                    if (i > 0)
+                        GenerateSpace();
+
+                    // Handle SqlDataTypeReference properly - need to generate the type name and parameters separately
+                    if (list[i] is SqlDataTypeReference sqlDataType)
+                    {
+                        // Generate the data type name (e.g., NVARCHAR)
+                        string dataTypeName = sqlDataType.SqlDataTypeOption.ToString().ToUpper(CultureInfo.InvariantCulture);
+                        GenerateIdentifier(dataTypeName);
+
+                        // Generate parameters if any (e.g., (50))
+                        if (sqlDataType.Parameters?.Count > 0)
+                        {
+                            GenerateSymbol(TSqlTokenType.LeftParenthesis);
+                            for (int j = 0; j < sqlDataType.Parameters.Count; j++)
+                            {
+                                if (j > 0)
+                                    GenerateSymbol(TSqlTokenType.Comma);
+                                GenerateFragmentIfNotNull(sqlDataType.Parameters[j]);
+                            }
+                            GenerateSymbol(TSqlTokenType.RightParenthesis);
+                        }
+                    }
+                    else
+                    {
+                        // For other data type references, use the default generation
+                        GenerateFragmentIfNotNull(list[i]);
+                    }
+                }
             }
         }
     }

Test/SqlDom/Baselines170/JsonArrayAggOrderBy170.sql

@@ -0,0 +1,19 @@
+SELECT JSON_ARRAYAGG(value ORDER BY value)
+FROM mytable;
+
+SELECT JSON_ARRAYAGG(name ORDER BY name ASC)
+FROM users;
+
+SELECT JSON_ARRAYAGG(score ORDER BY score DESC)
+FROM scores;
+
+SELECT JSON_ARRAYAGG(data ORDER BY priority DESC, created_at ASC)
+FROM records;
+
+SELECT JSON_ARRAYAGG(value ORDER BY value NULL ON NULL)
+FROM data;
+
+SELECT TOP (5) c.object_id,
+               JSON_ARRAYAGG(c.name ORDER BY c.column_id) AS column_list
+FROM sys.columns AS c
+GROUP BY c.object_id;

Test/SqlDom/Baselines170/JsonFunctionTests170.sql

@@ -140,7 +140,7 @@ SELECT JSON_QUERY('{ "a": 1 }');
 
 SELECT JSON_QUERY('{ "a": 1 }', '$.a');
 
-SELECT JSON_QUERY('{ "a": [1,2,3] }', '$.a') WITH ARRAY WRAPPER;
+SELECT JSON_QUERY('{ "a": [1,2,3] }', '$.a' WITH ARRAY WRAPPER);
 
 
 GO
@@ -154,4 +154,34 @@ GO
 SELECT TOP (5) c.object_id,
                JSON_OBJECTAGG(c.name:c.column_id) AS columns
 FROM sys.columns AS c
-GROUP BY c.object_id;
+GROUP BY c.object_id;
+
+SELECT JSON_VALUE('a', '$');
+SELECT JSON_VALUE('c', '$' RETURNING INT);
+SELECT JSON_VALUE('c', '$' RETURNING SMALLINT);
+SELECT JSON_VALUE('c', '$' RETURNING BIGINT);
+SELECT JSON_VALUE('c', '$' RETURNING TINYINT);
+SELECT JSON_VALUE('c', '$' RETURNING NUMERIC);
+SELECT JSON_VALUE('c', '$' RETURNING FLOAT);
+SELECT JSON_VALUE('c', '$' RETURNING REAL);
+SELECT JSON_VALUE('c', '$' RETURNING DECIMAL);
+SELECT JSON_VALUE('c', '$' RETURNING CHAR);
+SELECT JSON_VALUE('c', '$' RETURNING NVARCHAR(50));
+SELECT JSON_VALUE('c', '$' RETURNING NCHAR);
+SELECT JSON_VALUE('c', '$' RETURNING DATE);
+SELECT JSON_VALUE('c', '$' RETURNING DATETIME2);
+SELECT JSON_VALUE('c', '$' RETURNING TIME);
+SELECT JSON_VALUE('c', '$' RETURNING BIT);
+
+SELECT id,
+       json_col
+FROM tab1
+WHERE JSON_CONTAINS(json_col, 'abc', '$.a') = 1;
+
+SELECT id,
+       json_col
+FROM tab1
+WHERE JSON_CONTAINS(json_col, 'abc%', '$.a', 1) = 1;
+
+SELECT JSON_MODIFY(json_col, '$.a', 30)
+FROM tab1;