Skip to content

[AI Test Tool] Add Language Support and Capability #5623

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stkillen wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[AI Test Tool] Add Language Support and Capability #5623

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9a24cc5
Add language support
stkillen Nov 25, 2025
a960301
Code review comments
stkillen Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Tools/AI Test Toolkit/Permissions/AITestToolKitRead.PermissionSet.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ permissionset 149033 "AI Test Toolkit - Read"

Permissions = tabledata "AIT Run History" = R,
tabledata "AIT Test Suite" = R,
tabledata "AIT Test Suite Language" = R,
tabledata "AIT Test Method Line" = R,
tabledata "AIT Log Entry" = R;
}
1 change: 1 addition & 0 deletions src/Tools/AI Test Toolkit/Permissions/AITestToolKitView.PermissionSet.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ permissionset 149034 "AI Test Toolkit - View"

Permissions = tabledata "AIT Run History" = IMD,
tabledata "AIT Test Suite" = IMD,
tabledata "AIT Test Suite Language" = IMD,
tabledata "AIT Test Method Line" = IMD,
tabledata "AIT Log Entry" = IMD;
}
2 changes: 2 additions & 0 deletions src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ permissionset 149031 "AI Test Toolkit - Obj"
Permissions = table "AIT Run History" = X,
table "AIT Test Suite" = X,
table "AIT Test Method Line" = X,
table "AIT Test Suite Language" = X,
table "AIT Log Entry" = X,
codeunit "AIT AL Test Suite Mgt" = X,
codeunit "AIT Install" = X,
Expand All @@ -36,5 +37,6 @@ permissionset 149031 "AI Test Toolkit - Obj"
page "AIT Log Entry API" = X,
page "AIT Test Suite" = X,
page "AIT Test Suite List" = X,
page "AIT Test Suite Languages" = X,
page "AIT Run History" = X;
}
22 changes: 22 additions & 0 deletions src/Tools/AI Test Toolkit/src/AITCommandLineCard.Page.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,27 @@ page 149042 "AIT CommandLine Card"
UpdateAITestMethodLines();
end;
}
field("Language Tag"; LanguageTagFilter)
{
Caption = 'Language';
ToolTip = 'Specifies the language to run.';

trigger OnValidate()
var
AITTestSuite: Record "AIT Test Suite";
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
if not AITTestSuite.Get(AITCode) then
Error(CannotFindAITSuiteErr, AITCode);

Clear(LineNoFilter);

AITTestSuite.Validate("Language ID", AITTestSuiteLanguage.GetLanguageIDByTag(LanguageTagFilter));
AITTestSuite.Modify(true);

UpdateAITestMethodLines();
end;
}
field("Line No. Filter"; LineNoFilter)
{
Caption = 'Line No. Filter';
Expand Down Expand Up @@ -274,6 +295,7 @@ page 149042 "AIT CommandLine Card"
CannotFindAITSuiteErr: Label 'The specified Test Suite with code %1 cannot be found.', Comment = '%1 = Test Suite id.';
AITCode: Code[100];
LineNoFilter: Integer;
LanguageTagFilter: Text[80];
NoOfPendingTests: Integer;
InputDataset: Text;
SuiteDefinition: Text;
Expand Down
49 changes: 44 additions & 5 deletions src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,17 +232,56 @@ codeunit 149037 "AIT AL Test Suite Mgt"
/// <param name="DatasetFileName">The file name of the dataset file which will be used in the description of the dataset.</param>
/// <param name="DatasetInStream">The InStream of the dataset file.</param>
procedure ImportTestInputs(DatasetFileName: Text; var DatasetInStream: InStream)
var
CallerModuleInfo: ModuleInfo;
begin
NavApp.GetCallerModuleInfo(CallerModuleInfo);
ImportTestInputs(DatasetFileName, DatasetInStream, CallerModuleInfo, 0, DatasetFileName);
end;

/// <summary>
/// Import the Test Input Dataset from an InStream of a dataset in a supported format with a specific language.
/// Overwrite the dataset if the dataset with same filename is already imported by the same app
/// Error if the dataset with the same filename is created by a different app
/// </summary>
/// <param name="DatasetFileName">The file name of the dataset file which will be used in the description of the dataset.</param>
/// <param name="DatasetInStream">The InStream of the dataset file.</param>
/// <param name="LanguageID">The language ID to use for the dataset import.</param>
procedure ImportTestInputs(DatasetFileName: Text; var DatasetInStream: InStream; LanguageID: Integer)
var
CallerModuleInfo: ModuleInfo;
begin
NavApp.GetCallerModuleInfo(CallerModuleInfo);
ImportTestInputs(DatasetFileName, DatasetInStream, CallerModuleInfo, LanguageID, DatasetFileName);
end;

/// <summary>
/// Import the Test Input Dataset from an InStream of a dataset in a supported format with a specific language.
/// Overwrite the dataset if the dataset with same filename is already imported by the same app
/// Error if the dataset with the same filename is created by a different app
/// </summary>
/// <param name="DatasetFileName">The file name of the dataset file which will be used in the description of the dataset.</param>
/// <param name="DatasetInStream">The InStream of the dataset file.</param>
/// <param name="LanguageID">The language ID to use for the dataset import.</param>
/// <param name="Name">The name to use for the dataset import.</param>
procedure ImportTestInputs(DatasetFileName: Text; var DatasetInStream: InStream; LanguageID: Integer; Name: Text)
var
CallerModuleInfo: ModuleInfo;
begin
NavApp.GetCallerModuleInfo(CallerModuleInfo);
ImportTestInputs(DatasetFileName, DatasetInStream, CallerModuleInfo, LanguageID, Name);
end;

local procedure ImportTestInputs(DatasetFileName: Text; var DatasetInStream: InStream; CallerModuleInfo: ModuleInfo; LanguageID: Integer; Name: Text)
var
TestInputGroup: Record "Test Input Group";
TestInputsManagement: Codeunit "Test Inputs Management";
CallerModuleInfo: ModuleInfo;
EmptyGuid: Guid;
SameDatasetNameErr: Label 'The test input dataset %1 with the same file name already exists. The dataset was uploaded %2. Please rename the current dataset or delete the existing dataset.', Comment = '%1 = test input dataset Name, %2 = "from the UI" or "by the app id: {app_id}';
SourceOfTheDatasetIsUILbl: Label 'from the UI';
SourceOfTheDatasetIsAppIdLbl: Label 'by the app id: %1', Comment = '%1 = app id';
SourceOfTheDatasetIsAppIdLbl: Label 'previously by the app id: %1 and now with %2', Comment = '%1 = previous app id, %2 = current app id';
begin
// Check if the dataset with the same filename exists
NavApp.GetCallerModuleInfo(CallerModuleInfo);
TestInputGroup.SetLoadFields(Code, "Imported by AppId");

if TestInputGroup.Get(TestInputsManagement.GetTestInputGroupCodeFromFileName(DatasetFileName)) then
Expand All @@ -253,10 +292,10 @@ codeunit 149037 "AIT AL Test Suite Mgt"
EmptyGuid:
Error(SameDatasetNameErr, DatasetFileName, SourceOfTheDatasetIsUILbl)
else
Error(SameDatasetNameErr, DatasetFileName, StrSubstNo(SourceOfTheDatasetIsAppIdLbl, TestInputGroup."Imported by AppId"));
Error(SameDatasetNameErr, DatasetFileName, StrSubstNo(SourceOfTheDatasetIsAppIdLbl, TestInputGroup."Imported by AppId", CallerModuleInfo.Id));
end;

TestInputsManagement.UploadAndImportDataInputs(DatasetFileName, DatasetInStream, CallerModuleInfo.Id);
TestInputsManagement.UploadAndImportDataInputs(DatasetFileName, DatasetInStream, CallerModuleInfo.Id, LanguageID, Name);
end;

/// <summary>
Expand Down
13 changes: 13 additions & 0 deletions src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@ table 149032 "AIT Test Method Line"
DataClassification = CustomerContent;
TableRelation = "Test Input Group";
ToolTip = 'Specifies a dataset that overrides the default dataset for the suite.';

trigger OnValidate()
var
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
if "Input Dataset" = '' then
exit;

AITTestSuiteLanguage.AddLanguagesFromTestMethodLine(Rec);
end;
}
field(9; "Status"; Enum "AIT Line Status")
{
Expand Down Expand Up @@ -248,10 +258,13 @@ table 149032 "AIT Test Method Line"
trigger OnInsert()
var
AITTestSuite: Record "AIT Test Suite";
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
if Rec."Input Dataset" = '' then
if AITTestSuite.Get(Rec."Test Suite Code") then
Rec."Input Dataset" := AITTestSuite."Input Dataset";

AITTestSuiteLanguage.AddLanguagesFromTestMethodLine(Rec);
end;

trigger OnDelete()
Expand Down
36 changes: 36 additions & 0 deletions src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,25 @@ page 149031 "AIT Test Suite"
AITTestMethodLine.ModifyAll("Input Dataset", Rec."Input Dataset", true);
end;
}
field("Copilot Capability"; Rec."Copilot Capability")
{
ApplicationArea = All;
}
field("Language Tag"; Language)
{
ApplicationArea = All;
Caption = 'Language';
ToolTip = 'Specifies the language to use when running the test suite. Available languages are listed in the Languages factbox.';
Editable = false;

trigger OnAssistEdit()
var
AITTestSuiteLanguages: Codeunit "AIT Test Suite Language";
begin
AITTestSuiteLanguages.AssistEditTestSuiteLanguage(Rec);
CurrPage.Update(true);
end;
}
field("Test Runner Id"; TestRunnerDisplayName)
{
Caption = 'Test Runner';
Expand Down Expand Up @@ -200,6 +219,16 @@ page 149031 "AIT Test Suite"
}

}
area(FactBoxes)
{
part(Languages; "AIT Test Suite Languages Part")
{
ApplicationArea = All;
SubPageLink = "Test Suite Code" = field("Code");
Caption = 'Languages';
UpdatePropagation = Both;
}
}
}
actions
{
Expand Down Expand Up @@ -371,14 +400,19 @@ page 149031 "AIT Test Suite"
TotalDuration: Duration;
PageCaptionLbl: Label 'AI Test';
TestRunnerDisplayName: Text;
Language: Text;
InputDatasetChangedQst: Label 'You have modified the input dataset.\\Do you want to update the lines?';
EvaluationSetupTxt: Text;

trigger OnOpenPage()
var
FeatureTelemetry: Codeunit "Feature Telemetry";
AITTestSuiteLanguages: Codeunit "AIT Test Suite Language";
begin
FeatureTelemetry.LogUptake('0000NEV', AITTestSuiteMgt.GetFeatureName(), Enum::"Feature Uptake Status"::Discovered);

if AITTestSuiteLanguages.AddLanguagesFromTestSuite(Rec) then
CurrPage.Update(false);
end;

trigger OnNewRecord(BelowxRec: Boolean)
Expand All @@ -389,10 +423,12 @@ page 149031 "AIT Test Suite"
trigger OnAfterGetCurrRecord()
var
TestSuiteMgt: Codeunit "Test Suite Mgt.";
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
UpdateTotalDuration();
UpdateAverages();
TestRunnerDisplayName := TestSuiteMgt.GetTestRunnerDisplayName(Rec."Test Runner Id");
Language := AITTestSuiteLanguage.GetLanguageDisplayName(Rec."Language ID");
EvaluationSetupTxt := AITTestSuiteMgt.GetEvaluationSetupText(Rec.Code, 0);
end;

Expand Down
34 changes: 34 additions & 0 deletions src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
namespace System.TestTools.AITestToolkit;

using System.TestTools.TestRunner;
using System.AI;

#pragma warning disable AS0002
table 149030 "AIT Test Suite"
Expand Down Expand Up @@ -113,6 +114,11 @@ table 149030 "AIT Test Suite"
Editable = false;
ToolTip = 'Specifies the version of the current test run. It is used for comparing the results of the current test run with the results of the previous test run. The version will be stored in the Log entries.';
}
field(14; "Copilot Capability"; Enum "Copilot Capability")
Copy link
Contributor

@nikolakukrika nikolakukrika Nov 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this?
It could be solved in the different ways if we need to filter out the things differently. Do we need to store this information on the suite level?

Do we want to limit the test suites to test only single copilot capabilites?

Copy link
Contributor Author

@stkillen stkillen Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This adds better categorization and filtering options for test suites. Test suites clearly linked to capabilities will also make it easier to create dashboards, etc., to get count of tests per feature or results per feature.

I think it's a fair restriction to make that a test suite tests one capability. This is also true for our existing test suites.

Copy link
Contributor

@nikolakukrika nikolakukrika Dec 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@stkillen Are we sure that partners will use it in the same way? Are we sure that partners will use copilot capabilities.
For both us and partners: Is it a must to have a capability for unit testing, e.g. I just want to test a part of the prompt?
We could have it as a filter, and introduce handling of the zero/not specified.

Copy link
Contributor Author

@stkillen stkillen Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partners have to register their capability to use the AI SDK. Thus, there's no way to test e.g. prompt changes without testing a specific capability that you registered. The only way is if you build your own SDK and still want to use the test tool. If that's case, they can opt out of using the field as it's optional. But for all MS features and partner features built using the AI SDK, it makes sense.

I can introduce handling for 0 as "Not Specified" if there's any scenarios where it's not relevant.

{
Caption = 'Copilot Capability';
ToolTip = 'Specifies the Copilot Capability that the test suite tests.';
}
field(16; "Base Version"; Integer)
{
Caption = 'Base Version';
Expand Down Expand Up @@ -186,6 +192,34 @@ table 149030 "AIT Test Suite"
FieldClass = FlowField;
CalcFormula = count("AIT Column Mapping" where("Test Suite Code" = field("Code")));
}
field(40; "Language ID"; Integer)
{
Caption = 'Language ID';
TableRelation = "AIT Test Suite Language"."Language ID";
ValidateTableRelation = true;
ToolTip = 'Specifies the Windows Language ID for this test suite language.';
Copy link
Contributor

@nikolakukrika nikolakukrika Nov 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would not use Windows language ID, simply language that the test suite runs in/

Copy link
Contributor

@nikolakukrika nikolakukrika Nov 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to limit the test suite to a single language?
Should we support e.g. 1 codeunit in all languages as a test suite?


trigger OnValidate()
var
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
AITTestSuiteLanguage.UpdateLanguagesForTestSuite(Rec);
end;
}
field(41; "Language Tag"; Text[80])
{
Caption = 'Language Tag';
Editable = false;
FieldClass = FlowField;
CalcFormula = lookup("AIT Test Suite Language"."Language Tag" where("Test Suite Code" = field("Code"), "Language ID" = field("Language ID")));
}
field(42; "Language Name"; Text[80])
{
Caption = 'Language Name';
Editable = false;
FieldClass = FlowField;
CalcFormula = lookup("AIT Test Suite Language"."Language Name" where("Test Suite Code" = field("Code"), "Language ID" = field("Language ID")));
}
field(50; "Test Runner Id"; Integer)
{
Caption = 'Test Runner Id';
Expand Down
52 changes: 52 additions & 0 deletions src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,62 @@ xmlport 149031 "AIT Test Suite Import/Export"
{
Occurrence = Required;
}
fieldattribute(Capability; "AITSuite"."Copilot Capability")
{
Occurrence = Optional;
}
textattribute(DefaultLanguage)
{
Occurrence = Optional;

trigger OnBeforePassVariable()
begin
AITSuite.CalcFields("Language Tag");
DefaultLanguage := AITSuite."Language Tag";
end;

trigger OnAfterAssignVariable()
var
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
AITSuite."Language ID" := AITTestSuiteLanguage.GetLanguageIDByTag(Tag);
end;
}
fieldattribute(TestRunnerId; "AITSuite"."Test Runner Id")
{
Occurrence = Optional;
}
tableelement(AITLanguage; "AIT Test Suite Language")
{
LinkFields = "Test Suite Code" = field("Code");
LinkTable = "AITSuite";
MinOccurs = Zero;
XmlName = 'Language';

textattribute(Tag)
{
Occurrence = Required;

trigger OnBeforePassVariable()
begin
AITLanguage.CalcFields("Language Tag");
Tag := AITLanguage."Language Tag";
end;

trigger OnAfterAssignVariable()
var
AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
begin
AITLanguage."Language ID" := AITTestSuiteLanguage.GetLanguageIDByTag(Tag);
end;
}

trigger OnAfterInitRecord()
begin
if SkipTestSuites.Contains(AITSuite.Code) then
currXMLport.Skip();
end;
}
tableelement(AITEvaluator; "AIT Evaluator")
{
LinkFields = "Test Suite Code" = field("Code");
Expand Down
Loading
Loading