Skip to content

Commit cccf27d

Browse files
jriekenjrieken
authored
scaffold inline chat e2e test (#2432)
* debt - extract editing strategies * scaffold inline chat e2e test
1 parent c57b8ae commit cccf27d

File tree

2 files changed

+125
-35
lines changed

2 files changed

+125
-35
lines changed

‎src/extension/inlineChat/node/inlineChatIntent.ts‎

Lines changed: 59 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,21 @@ import { CopilotInteractiveEditorResponse, InteractionOutcome, InteractionOutcom
5757

5858
const INLINE_CHAT_EXIT_TOOL_NAME = 'inline_chat_exit';
5959

60-
interface Result {
60+
interface IInlineChatEditResult {
6161
telemetry: InlineChatTelemetry;
6262
lastResponse: ChatResponse;
63+
needsExitTool: boolean;
64+
}
65+
66+
interface IInlineChatEditStrategy {
67+
executeEdit(endpoint: IChatEndpoint, conversation: Conversation, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: CancellationToken, documentContext: IDocumentContext, chatTelemetry: ChatTelemetryBuilder): Promise<IInlineChatEditResult>;
6368
}
6469

6570
export class InlineChatIntent implements IIntent {
6671

6772
static readonly ID = Intent.InlineChat;
6873

69-
private static readonly _EDIT_TOOLS = new Set<string>([
74+
static readonly _EDIT_TOOLS = new Set<string>([
7075
ToolName.ApplyPatch,
7176
ToolName.EditFile,
7277
ToolName.ReplaceString,
@@ -208,11 +213,13 @@ export class InlineChatIntent implements IIntent {
208213
}
209214
}
210215

211-
let result: Result;
216+
let result: IInlineChatEditResult;
212217
try {
213-
result = useToolsForEdit
214-
? await this._handleRequestWithEditTools(endpoint, conversation, request, stream, token, documentContext, chatTelemetry)
215-
: await this._handleRequestWithEditHeuristic(endpoint, conversation, request, stream, token, documentContext, chatTelemetry);
218+
const strategy: IInlineChatEditStrategy = useToolsForEdit
219+
? this._instantiationService.createInstance(InlineChatEditToolsStrategy, this)
220+
: this._instantiationService.createInstance(InlineChatEditHeuristicStrategy, this);
221+
222+
result = await strategy.executeEdit(endpoint, conversation, request, stream, token, documentContext, chatTelemetry);
216223
} catch (err) {
217224
this._logService.error(err, 'InlineChatIntent: prompt rendering failed');
218225
return {
@@ -228,6 +235,11 @@ export class InlineChatIntent implements IIntent {
228235
return CanceledResult;
229236
}
230237

238+
if (result.needsExitTool) {
239+
// BAILOUT: when no edits were emitted, invoke the exit tool manually
240+
await this._toolsService.invokeTool(INLINE_CHAT_EXIT_TOOL_NAME, { toolInvocationToken: request.toolInvocationToken, input: undefined }, token);
241+
}
242+
231243
// store metadata for telemetry sending
232244
const turn = conversation.getLatestTurn();
233245
turn.setMetadata(new InteractionOutcome(didSeeAnyEdit ? 'inlineEdit' : 'none', []));
@@ -243,10 +255,6 @@ export class InlineChatIntent implements IIntent {
243255
buildPrompt: () => { throw new Error(); },
244256
}));
245257

246-
if (token.isCancellationRequested) {
247-
return CanceledResult;
248-
}
249-
250258
if (result.lastResponse.type !== ChatFetchResponseType.Success) {
251259
const details = getErrorDetailsFromChatFetchError(result.lastResponse, await this._endpointProvider.getChatEndpoint('copilot-base'), (await this._authenticationService.getCopilotToken()).copilotPlan);
252260
return {
@@ -260,9 +268,25 @@ export class InlineChatIntent implements IIntent {
260268
return {};
261269
}
262270

263-
// --- NEW world: edit tools
271+
invoke(): Promise<never> {
272+
throw new TypeError();
273+
}
274+
}
275+
276+
class InlineChatEditToolsStrategy implements IInlineChatEditStrategy {
264277

265-
private async _handleRequestWithEditTools(endpoint: IChatEndpoint, conversation: Conversation, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: CancellationToken, documentContext: IDocumentContext, chatTelemetry: ChatTelemetryBuilder): Promise<Result> {
278+
readonly id = InlineChatIntent.ID;
279+
readonly locations = [ChatLocation.Editor];
280+
readonly description = '';
281+
282+
constructor(
283+
private readonly _intent: InlineChatIntent,
284+
@IInstantiationService private readonly _instantiationService: IInstantiationService,
285+
@ILogService private readonly _logService: ILogService,
286+
@IToolsService private readonly _toolsService: IToolsService,
287+
) { }
288+
289+
async executeEdit(endpoint: IChatEndpoint, conversation: Conversation, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: CancellationToken, documentContext: IDocumentContext, chatTelemetry: ChatTelemetryBuilder): Promise<IInlineChatEditResult> {
266290
assertType(request.location2 instanceof ChatRequestEditorData);
267291
assertType(documentContext);
268292

@@ -286,7 +310,7 @@ export class InlineChatIntent implements IIntent {
286310

287311
const renderResult = await renderer.render(undefined, token, { trace: true });
288312

289-
telemetry = chatTelemetry.makeRequest(this, ChatLocation.Editor, conversation, renderResult.messages, renderResult.tokenCount, renderResult.references, endpoint, [], availableTools.length);
313+
telemetry = chatTelemetry.makeRequest(this._intent, ChatLocation.Editor, conversation, renderResult.messages, renderResult.tokenCount, renderResult.references, endpoint, [], availableTools.length);
290314

291315
stream = ChatResponseStreamImpl.spy(stream, part => {
292316
if (part instanceof ChatResponseTextEditPart) {
@@ -317,8 +341,7 @@ export class InlineChatIntent implements IIntent {
317341
}
318342

319343
if (result.toolCalls.length === 0) {
320-
// BAILOUT: when no tools have been used, invoke the exit tool manually
321-
await this._toolsService.invokeTool(INLINE_CHAT_EXIT_TOOL_NAME, { toolInvocationToken: request.toolInvocationToken, input: undefined }, token);
344+
// BAILOUT: when no tools have been used
322345
break;
323346
}
324347

@@ -329,15 +352,15 @@ export class InlineChatIntent implements IIntent {
329352

330353
if (editAttempts.push(...result.failedEdits) > 5) {
331354
// TOO MANY FAILED ATTEMPTS
332-
this._logService.warn(`Aborting inline chat edit: too many failed edit attempts`);
355+
this._logService.error(`Aborting inline chat edit: too many failed edit attempts`);
333356
break;
334357
}
335358
}
336359

337360
telemetry.sendToolCallingTelemetry(toolCallRounds, availableTools, token.isCancellationRequested ? 'cancelled' : lastResponse.type);
338361

339-
340-
return { lastResponse, telemetry };
362+
const needsExitTool = toolCallRounds.length === 0 || (toolCallRounds.length > 0 && toolCallRounds[toolCallRounds.length - 1].toolCalls.length === 0);
363+
return { lastResponse, telemetry, needsExitTool };
341364
}
342365

343366
private async _makeRequestAndRunTools(endpoint: IChatEndpoint, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, messages: Raw.ChatMessage[], inlineChatTools: vscode.LanguageModelToolInformation[], telemetry: InlineChatTelemetry, token: CancellationToken) {
@@ -372,9 +395,9 @@ export class InlineChatIntent implements IIntent {
372395
telemetryProperties: {
373396
messageId: telemetry.telemetryMessageId,
374397
conversationId: telemetry.sessionId,
375-
messageSource: this.id
398+
messageSource: this._intent.id
376399
},
377-
finishedCb: async (text, index, delta) => {
400+
finishedCb: async (_text, _index, delta) => {
378401

379402
telemetry.markReceivedToken();
380403

@@ -471,10 +494,20 @@ export class InlineChatIntent implements IIntent {
471494

472495
return [exitTool, ...editTools];
473496
}
497+
}
474498

475-
// ---- NEW world: edit prompt
499+
class InlineChatEditHeuristicStrategy implements IInlineChatEditStrategy {
476500

477-
private async _handleRequestWithEditHeuristic(endpoint: IChatEndpoint, conversation: Conversation, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: CancellationToken, documentContext: IDocumentContext, chatTelemetry: ChatTelemetryBuilder): Promise<Result> {
501+
readonly id = InlineChatIntent.ID;
502+
readonly locations = [ChatLocation.Editor];
503+
readonly description = '';
504+
505+
constructor(
506+
private readonly _intent: InlineChatIntent,
507+
@IInstantiationService private readonly _instantiationService: IInstantiationService,
508+
) { }
509+
510+
async executeEdit(endpoint: IChatEndpoint, conversation: Conversation, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: CancellationToken, documentContext: IDocumentContext, chatTelemetry: ChatTelemetryBuilder): Promise<IInlineChatEditResult> {
478511

479512
assertType(request.location2 instanceof ChatRequestEditorData);
480513

@@ -494,7 +527,7 @@ export class InlineChatIntent implements IIntent {
494527
const replyInterpreter = renderResult.metadata.get(ReplyInterpreterMetaData)?.replyInterpreter ?? new NoopReplyInterpreter();
495528
const telemetryData = renderResult.metadata.getAll(TelemetryData);
496529

497-
const telemetry = chatTelemetry.makeRequest(this, ChatLocation.Editor, conversation, renderResult.messages, renderResult.tokenCount, renderResult.references, endpoint, telemetryData, 0);
530+
const telemetry = chatTelemetry.makeRequest(this._intent, ChatLocation.Editor, conversation, renderResult.messages, renderResult.tokenCount, renderResult.references, endpoint, telemetryData, 0);
498531

499532
stream = ChatResponseStreamImpl.spy(stream, part => {
500533
if (part instanceof ChatResponseTextEditPart) {
@@ -523,7 +556,7 @@ export class InlineChatIntent implements IIntent {
523556
telemetryProperties: {
524557
messageId: telemetry.telemetryMessageId,
525558
conversationId: telemetry.sessionId,
526-
messageSource: this.id
559+
messageSource: this._intent.id
527560
},
528561
requestOptions: {
529562
stream: true,
@@ -543,19 +576,10 @@ export class InlineChatIntent implements IIntent {
543576
const responseText = fetchResult.type === ChatFetchResponseType.Success ? fetchResult.value : '';
544577
telemetry.sendTelemetry(
545578
fetchResult.requestId, fetchResult.type, responseText,
546-
new InteractionOutcome('inlineEdit', []),
579+
new InteractionOutcome(telemetry.editCount > 0 ? 'inlineEdit' : 'none', []),
547580
[]
548581
);
549582

550-
if (telemetry.editCount === 0) {
551-
// BAILOUT: when no edits were emitted, invoke the exit tool manually
552-
await this._toolsService.invokeTool(INLINE_CHAT_EXIT_TOOL_NAME, { toolInvocationToken: request.toolInvocationToken, input: undefined }, token);
553-
}
554-
555-
return { lastResponse: fetchResult, telemetry };
556-
}
557-
558-
invoke(): Promise<never> {
559-
throw new TypeError();
583+
return { lastResponse: fetchResult, telemetry, needsExitTool: telemetry.editCount === 0 };
560584
}
561585
}

‎src/extension/inlineChat/test/vscode-node/inlineChat.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import * as assert from 'assert';
7+
import * as vscode from 'vscode';
8+
import { DisposableStore } from '../../../../util/vs/base/common/lifecycle';
9+
10+
suite('Inline Chat', function () {
11+
// this.timeout(1000 * 60 * 1); // 1 minute
12+
13+
let store: DisposableStore;
14+
15+
teardown(function () {
16+
store.dispose();
17+
});
18+
19+
setup(function () {
20+
store = new DisposableStore();
21+
});
22+
23+
test.skip('E2E Inline Chat Test', async function () {
24+
store.add(vscode.lm.registerLanguageModelChatProvider('test', new class implements vscode.LanguageModelChatProvider {
25+
async provideLanguageModelChatInformation(options: { silent: boolean }, token: vscode.CancellationToken): Promise<vscode.LanguageModelChatInformation[]> {
26+
return [{
27+
id: 'test',
28+
name: 'test',
29+
family: 'test',
30+
version: '0.0.0',
31+
maxInputTokens: 1000,
32+
maxOutputTokens: 1000,
33+
requiresAuthorization: true,
34+
capabilities: {}
35+
}];
36+
}
37+
async provideLanguageModelChatResponse(model: vscode.LanguageModelChatInformation, messages: Array<vscode.LanguageModelChatMessage | vscode.LanguageModelChatMessage2>, options: vscode.ProvideLanguageModelChatResponseOptions, progress: vscode.Progress<vscode.LanguageModelResponsePart2>, token: vscode.CancellationToken): Promise<void> {
38+
throw new Error('Method not implemented.');
39+
}
40+
async provideTokenCount(model: vscode.LanguageModelChatInformation, text: string | vscode.LanguageModelChatMessage | vscode.LanguageModelChatMessage2, token: vscode.CancellationToken): Promise<number> {
41+
return 0;
42+
}
43+
}));
44+
45+
46+
47+
// Create and open a new file
48+
const document = await vscode.workspace.openTextDocument({ language: 'javascript' });
49+
await vscode.window.showTextDocument(document);
50+
51+
try {
52+
53+
await vscode.commands.executeCommand('vscode.editorChat.start', {
54+
blockOnResponse: true,
55+
autoSend: true,
56+
message: 'Write me a for loop in javascript',
57+
position: new vscode.Position(0, 0),
58+
initialSelection: new vscode.Selection(0, 0, 0, 0),
59+
modelSelector: { id: 'test' }
60+
});
61+
} catch (err) {
62+
assert.ok(false);
63+
}
64+
65+
});
66+
});

0 commit comments

Comments
 (0)