* drive inline chat feedback telemetry from CopilotInteractiveEditorResponse (#2435)

* `CopilotInteractiveEditorResponse.kind` is always 'ok' -> remove it

fixes microsoft/vscode#280959

Author: jrieken

src/extension/conversation/vscode-node/userActions.ts

@@ -6,7 +6,6 @@
 
 import * as vscode from 'vscode';
 import { editsAgentName, getChatParticipantIdFromName } from '../../../platform/chat/common/chatAgents';
-import { ChatLocation } from '../../../platform/chat/common/commonTypes';
 import { EditSurvivalResult } from '../../../platform/editSurvivalTracking/common/editSurvivalReporter';
 import { ILanguageDiagnosticsService } from '../../../platform/languages/common/languageDiagnosticsService';
 import { IMultiFileEditInternalTelemetryService } from '../../../platform/multiFileEdit/common/multiFileEditQualityTelemetry';
@@ -23,7 +22,6 @@ import { CopilotInteractiveEditorResponse, InteractionOutcome } from '../../inli
 import { participantIdToModeName } from '../../intents/common/intents';
 import { EditCodeStepTurnMetaData } from '../../intents/node/editCodeStep';
 import { Conversation, ICopilotChatResultIn } from '../../prompt/common/conversation';
-import { IntentInvocationMetadata } from '../../prompt/node/conversation';
 import { IFeedbackReporter } from '../../prompt/node/feedbackReporter';
 import { sendUserActionTelemetry } from '../../prompt/node/telemetry';
 
@@ -55,7 +53,7 @@ export class UserFeedbackService implements IUserFeedbackService {
 		const result = e.result as ICopilotChatResultIn;
 		const conversation = result.metadata?.responseId && this.conversationStore.getConversation(result.metadata.responseId);
 
-		if (typeof conversation === 'object' && conversation.getLatestTurn().getMetadata(IntentInvocationMetadata)?.value?.location === ChatLocation.Editor) {
+		if (typeof conversation === 'object' && conversation.getLatestTurn().getMetadata(CopilotInteractiveEditorResponse)) {
 			this._handleChatUserAction(result.metadata?.sessionId, agentId, conversation, e, undefined);
 			return;
 		}
@@ -380,16 +378,6 @@ export class UserFeedbackService implements IUserFeedbackService {
 			return;
 		}
 
-		const response = conversation.getLatestTurn().getMetadata(CopilotInteractiveEditorResponse);
-		if (!response) {
-			return;
-		}
-
-		const interactionOutcome = conversation.getLatestTurn().getMetadata(InteractionOutcome);
-		if (!interactionOutcome) {
-			return;
-		}
-
 		let kind: InteractiveEditorResponseFeedbackKind | undefined;
 		if (event?.action.kind === 'editor') {
 			kind = event.action.accepted ? InteractiveEditorResponseFeedbackKind.Accepted : InteractiveEditorResponseFeedbackKind.Undone;
@@ -405,6 +393,15 @@ export class UserFeedbackService implements IUserFeedbackService {
 			return;
 		}
 
+		const response = conversation.getLatestTurn().getMetadata(CopilotInteractiveEditorResponse);
+		if (!response) {
+			return;
+		}
+
+		let interactionOutcome = conversation.getLatestTurn().getMetadata(InteractionOutcome);
+		if (!interactionOutcome) {
+			interactionOutcome = new InteractionOutcome(!response.telemetry?.editCount ? 'none' : 'inlineEdit', []);
+		}
 
 		if (kind === InteractiveEditorResponseFeedbackKind.Bug && conversation) {
 			this.feedbackReporter.reportInline(conversation, response.promptQuery, interactionOutcome);
@@ -606,4 +603,4 @@ const outcomes = new Map([
 	[vscode.ChatEditingSessionActionOutcome.Accepted, 'accepted'],
 	[vscode.ChatEditingSessionActionOutcome.Rejected, 'rejected'],
 	[vscode.ChatEditingSessionActionOutcome.Saved, 'saved']
-]);
+]);

src/extension/inlineChat/node/inlineChatIntent.ts

@@ -40,7 +40,6 @@ import { Conversation, Turn } from '../../prompt/common/conversation';
 import { IToolCall } from '../../prompt/common/intents';
 import { ToolCallRound } from '../../prompt/common/toolCallRound';
 import { ChatTelemetryBuilder, InlineChatTelemetry } from '../../prompt/node/chatParticipantTelemetry';
-import { IntentInvocationMetadata } from '../../prompt/node/conversation';
 import { DefaultIntentRequestHandler } from '../../prompt/node/defaultIntentRequestHandler';
 import { IDocumentContext } from '../../prompt/node/documentContext';
 import { IIntent, NoopReplyInterpreter, ReplyInterpreterMetaData, TelemetryData } from '../../prompt/node/intents';
@@ -188,11 +187,9 @@ export class InlineChatIntent implements IIntent {
 		assertType(documentContext);
 
 		const editSurvivalTracker = this._editSurvivalTrackerService.initialize(request.location2.document);
-		let didSeeAnyEdit = false;
 
 		stream = ChatResponseStreamImpl.spy(stream, part => {
 			if (part instanceof ChatResponseTextEditPart) {
-				didSeeAnyEdit = true;
 				editSurvivalTracker.collectAIEdits(part.edits);
 			}
 		});
@@ -242,18 +239,11 @@ export class InlineChatIntent implements IIntent {
 
 		// store metadata for telemetry sending
 		const turn = conversation.getLatestTurn();
-		turn.setMetadata(new InteractionOutcome(didSeeAnyEdit ? 'inlineEdit' : 'none', []));
 		turn.setMetadata(new CopilotInteractiveEditorResponse(
-			'ok', undefined,
+			undefined,
 			{ ...documentContext, query: request.prompt, intent: this },
 			result.telemetry.telemetryMessageId, result.telemetry, editSurvivalTracker
 		));
-		turn.setMetadata(new IntentInvocationMetadata({ // UGLY fake intent invocation
-			location: ChatLocation.Editor,
-			intent: this,
-			endpoint,
-			buildPrompt: () => { throw new Error(); },
-		}));
 
 		if (result.lastResponse.type !== ChatFetchResponseType.Success) {
 			const details = getErrorDetailsFromChatFetchError(result.lastResponse, await this._endpointProvider.getChatEndpoint('copilot-base'), (await this._authenticationService.getCopilotToken()).copilotPlan);

src/extension/inlineChat/node/promptCraftingTypes.ts

@@ -138,13 +138,14 @@ export interface ICodeContextInfo {
 	above: CodeContextRegion;
 	range: CodeContextRegion;
 	below: CodeContextRegion;
-}export class CopilotInteractiveEditorResponse {
+}
+
+export class CopilotInteractiveEditorResponse {
 	constructor(
-		public readonly kind: 'ok',
-		public readonly store: ISessionTurnStorage | undefined,
-		public readonly promptQuery: PromptQuery,
-		public readonly messageId: string,
-		public readonly telemetry: ChatTelemetry | undefined,
-		public readonly editSurvivalTracker: IEditSurvivalTrackingSession
+		readonly store: ISessionTurnStorage | undefined,
+		readonly promptQuery: PromptQuery,
+		readonly messageId: string,
+		readonly telemetry: ChatTelemetry | undefined,
+		readonly editSurvivalTracker: IEditSurvivalTrackingSession
 	) { }
 }

src/extension/prompt/node/defaultIntentRequestHandler.ts

@@ -266,7 +266,6 @@ export class DefaultIntentRequestHandler {
 
 		if (this.documentContext) {
 			this.turn.setMetadata(new CopilotInteractiveEditorResponse(
-				'ok',
 				interactionOutcome.store,
 				{ ...this.documentContext, intent: this.intent, query: this.request.prompt },
 				this.chatTelemetryBuilder.telemetryMessageId,

test/simulation/inlineChatSimulator.ts

@@ -507,7 +507,7 @@ export async function simulateEditingScenario(
 			{
 				// TODO@Alex: extract to host object
 				const response = requestHandler.conversation.getLatestTurn()?.getMetadata(CopilotInteractiveEditorResponse);
-				intent = (response && response.kind === 'ok' ? response.promptQuery.intent : undefined);
+				intent = (response ? response.promptQuery.intent : undefined);
 			}
 			annotations = annotations.concat(requestHandler.conversation.getLatestTurn()?.getMetadata(InteractionOutcome)?.annotations ?? []);