When a task with problem matchers becomes idle, return those instead of using the LLM to evaluate the output (microsoft#261539)

meganrogge · web-flow · commit 18ca133342c8 · 2025-08-14T00:14:17.000Z
diff --git a/src/vs/workbench/contrib/tasks/common/taskService.ts b/src/vs/workbench/contrib/tasks/common/taskService.ts
@@ -8,7 +8,6 @@ import { Action } from '../../../../base/common/actions.js';
 import { Event } from '../../../../base/common/event.js';
 import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
 import { IDisposable } from '../../../../base/common/lifecycle.js';
-
 import { IWorkspaceFolder, IWorkspace } from '../../../../platform/workspace/common/workspace.js';
 import { Task, ContributedTask, CustomTask, ITaskSet, TaskSorter, ITaskEvent, ITaskIdentifier, ConfiguringTask, TaskRunSource } from './tasks.js';
 import { ITaskSummary, ITaskTerminateResponse, ITaskSystemInfo } from './taskSystem.js';
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/bufferOutputPolling.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/bufferOutputPolling.ts
@@ -14,6 +14,9 @@ import { IChatService } from '../../../chat/common/chatService.js';
 import { ChatMessageRole, ILanguageModelsService } from '../../../chat/common/languageModels.js';
 import { IToolInvocationContext } from '../../../chat/common/languageModelToolsService.js';
 import type { Terminal as RawXtermTerminal, IMarker as IXtermMarker } from '@xterm/xterm';
+import { Task } from '../../../tasks/common/taskService.js';
+import { IMarkerData, IMarkerService } from '../../../../../platform/markers/common/markers.js';
+import { ProblemMatcher, ProblemMatcherRegistry } from '../../../tasks/common/problemMatcher.js';
 
 export const enum PollingConsts {
 	MinNoDataEvents = 2, // Minimum number of no data checks before considering the terminal idle
@@ -34,7 +37,8 @@ export async function racePollingOrPrompt(
 	originalResult: { terminalExecutionIdleBeforeTimeout: boolean; output: string; pollDurationMs?: number; modelOutputEvalResponse?: string },
 	token: CancellationToken,
 	languageModelsService: ILanguageModelsService,
-	execution: { getOutput: () => string; isActive?: () => Promise<boolean> }
+	markerService: IMarkerService,
+	execution: { getOutput: () => string; isActive?: () => Promise<boolean>; task?: Task; beginsPattern?: string; endsPattern?: string; dependencyTasks?: Task[] }
 ): Promise<{ terminalExecutionIdleBeforeTimeout: boolean; output: string; pollDurationMs?: number; modelOutputEvalResponse?: string }> {
 	const pollPromise = pollFn();
 	const { promise: promptPromise, part } = promptFn();
@@ -63,7 +67,7 @@ export async function racePollingOrPrompt(
 		const promptResult = raceResult.result as boolean;
 		if (promptResult) {
 			// User accepted, poll again (extended)
-			return await pollForOutputAndIdle(execution, true, token, languageModelsService);
+			return await pollForOutputAndIdle(execution, true, token, languageModelsService, markerService);
 		} else {
 			return originalResult; // User rejected, return the original result
 		}
@@ -95,10 +99,12 @@ export function getOutput(terminal?: Pick<RawXtermTerminal, 'buffer'>, startMark
 }
 
 export async function pollForOutputAndIdle(
-	execution: { getOutput: () => string; isActive?: () => Promise<boolean> },
+	execution: { getOutput: () => string; isActive?: () => Promise<boolean>; task?: Pick<Task, 'configurationProperties'>; dependencyTasks?: Task[] },
 	extendedPolling: boolean,
 	token: CancellationToken,
-	languageModelsService: ILanguageModelsService,
+	languageModelsService: Pick<ILanguageModelsService, 'selectLanguageModels' | 'sendChatRequest'>,
+	markerService: Pick<IMarkerService, 'read'>,
+	knownMatchers?: ProblemMatcher[]
 ): Promise<{ terminalExecutionIdleBeforeTimeout: boolean; output: string; pollDurationMs?: number; modelOutputEvalResponse?: string }> {
 	const maxWaitMs = extendedPolling ? PollingConsts.ExtendedPollingMaxDuration : PollingConsts.FirstPollingMaxDuration;
 	const maxInterval = PollingConsts.MaxPollingIntervalDuration;
@@ -149,10 +155,32 @@ export async function pollForOutputAndIdle(
 				lastBufferLength = currentBufferLength;
 				continue;
 			}
-			terminalExecutionIdleBeforeTimeout = true;
-			const modelOutputEvalResponse = await assessOutputForErrors(buffer, token, languageModelsService);
-			return { modelOutputEvalResponse, terminalExecutionIdleBeforeTimeout, output: buffer, pollDurationMs: Date.now() - pollStartTime + (extendedPolling ? PollingConsts.FirstPollingMaxDuration : 0) };
 		}
+		terminalExecutionIdleBeforeTimeout = true;
+		if (execution.task) {
+			const problems = await getProblemsForTasks(execution.task, markerService, execution.dependencyTasks, knownMatchers);
+			if (problems) {
+				// Problem matchers exist for this task
+				const problemList: string[] = [];
+				for (const [, problemArray] of problems.entries()) {
+					if (problemArray.length) {
+						for (const p of problemArray) {
+							problemList.push(`${p.severity}: ${p.message}`);
+						}
+					}
+				}
+				if (problemList.length === 0) {
+					return { terminalExecutionIdleBeforeTimeout, output: 'The task succeeded with no problems.', pollDurationMs: Date.now() - pollStartTime + (extendedPolling ? PollingConsts.FirstPollingMaxDuration : 0) };
+				}
+				return {
+					terminalExecutionIdleBeforeTimeout,
+					output: problemList.join('\n'),
+					pollDurationMs: Date.now() - pollStartTime + (extendedPolling ? PollingConsts.FirstPollingMaxDuration : 0)
+				};
+			}
+		}
+		const modelOutputEvalResponse = await assessOutputForErrors(buffer, token, languageModelsService);
+		return { modelOutputEvalResponse, terminalExecutionIdleBeforeTimeout, output: buffer, pollDurationMs: Date.now() - pollStartTime + (extendedPolling ? PollingConsts.FirstPollingMaxDuration : 0) };
 	}
 	return { terminalExecutionIdleBeforeTimeout: false, output: buffer, pollDurationMs: Date.now() - pollStartTime + (extendedPolling ? PollingConsts.FirstPollingMaxDuration : 0) };
 }
@@ -168,7 +196,7 @@ export function promptForMorePolling(command: string, token: CancellationToken,
 			let part: ChatElicitationRequestPart | undefined = undefined;
 			const promise = new Promise<boolean>(resolve => {
 				const thePart = part = new ChatElicitationRequestPart(
-					new MarkdownString(localize('poll.terminal.waiting', "Continue waiting?")),
+					new MarkdownString(localize('poll.terminal.waiting', "Continue waiting for \`{0}\`?", command)),
 					new MarkdownString(localize('poll.terminal.polling', "This will continue to poll for output to determine when the terminal becomes idle for up to 2 minutes.")),
 					'',
 					localize('poll.terminal.accept', 'Yes'),
@@ -192,7 +220,7 @@ export function promptForMorePolling(command: string, token: CancellationToken,
 	return { promise: Promise.resolve(false) };
 }
 
-export async function assessOutputForErrors(buffer: string, token: CancellationToken, languageModelsService: ILanguageModelsService): Promise<string> {
+export async function assessOutputForErrors(buffer: string, token: CancellationToken, languageModelsService: Pick<ILanguageModelsService, 'selectLanguageModels' | 'sendChatRequest'>): Promise<string> {
 	const models = await languageModelsService.selectLanguageModels({ vendor: 'copilot', family: 'gpt-4o-mini' });
 	if (!models.length) {
 		return 'No models available';
@@ -223,3 +251,37 @@ export async function assessOutputForErrors(buffer: string, token: CancellationT
 		return 'Error occurred ' + err;
 	}
 }
+
+export function getProblemsForTasks(task: Pick<Task, 'configurationProperties'>, markerService: Pick<IMarkerService, 'read'>, dependencyTasks?: Task[], knownMatchers?: ProblemMatcher[]): Map<string, IMarkerData[]> | undefined {
+	const problemsMap = new Map<string, IMarkerData[]>();
+	let hadDefinedMatcher = false;
+
+	const collectProblems = (t: Pick<Task, 'configurationProperties'>) => {
+		const matchers = Array.isArray(t.configurationProperties.problemMatchers)
+			? t.configurationProperties.problemMatchers
+			: (t.configurationProperties.problemMatchers ? [t.configurationProperties.problemMatchers] : []);
+		for (const matcherRef of matchers) {
+			const matcher = typeof matcherRef === 'string'
+				? ProblemMatcherRegistry.get(matcherRef) ?? knownMatchers?.find(m => m.owner === matcherRef)
+				: matcherRef;
+			if (matcher?.owner) {
+				const markers = markerService.read({ owner: matcher.owner });
+				hadDefinedMatcher = true;
+				if (markers.length) {
+					problemsMap.set(matcher.owner, markers);
+				}
+			}
+		}
+	};
+
+	collectProblems(task);
+
+	if (problemsMap.size === 0 && dependencyTasks) {
+		for (const depTask of dependencyTasks) {
+			collectProblems(depTask);
+		}
+	}
+
+	return hadDefinedMatcher ? problemsMap : undefined;
+}
+
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/outputMonitor.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/outputMonitor.ts
@@ -9,6 +9,8 @@ import { Disposable } from '../../../../../base/common/lifecycle.js';
 import { ILanguageModelsService } from '../../../chat/common/languageModels.js';
 import { IChatService } from '../../../chat/common/chatService.js';
 import { racePollingOrPrompt, promptForMorePolling, pollForOutputAndIdle } from './bufferOutputPolling.js';
+import { IMarkerService } from '../../../../../platform/markers/common/markers.js';
+import { Task } from '../../../tasks/common/taskService.js';
 
 export interface IOutputMonitor extends Disposable {
 	readonly isIdle: boolean;
@@ -40,8 +42,9 @@ export class OutputMonitor extends Disposable implements IOutputMonitor {
 	}
 
 	constructor(
-		private readonly _execution: { getOutput: () => string; isActive?: () => Promise<boolean> },
-		@ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService
+		private readonly _execution: { getOutput: () => string; isActive?: () => Promise<boolean>; task?: Task; beginsPattern?: string; endsPattern?: string; dependencyTasks?: Task[] },
+		@ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService,
+		@IMarkerService private readonly _markerService: IMarkerService
 	) {
 		super();
 	}
@@ -52,15 +55,16 @@ export class OutputMonitor extends Disposable implements IOutputMonitor {
 		invocationContext: any,
 		token: CancellationToken
 	): Promise<{ terminalExecutionIdleBeforeTimeout: boolean; output: string; pollDurationMs?: number; modelOutputEvalResponse?: string }> {
-		let result = await pollForOutputAndIdle(this._execution, false, token, this._languageModelsService);
+		let result = await pollForOutputAndIdle(this._execution, false, token, this._languageModelsService, this._markerService);
 
 		if (!result.terminalExecutionIdleBeforeTimeout) {
 			result = await racePollingOrPrompt(
-				() => pollForOutputAndIdle(this._execution, true, token, this._languageModelsService),
+				() => pollForOutputAndIdle(this._execution, true, token, this._languageModelsService, this._markerService),
 				() => promptForMorePolling(command, token, invocationContext, chatService),
 				result,
 				token,
 				this._languageModelsService,
+				this._markerService,
 				this._execution
 			);
 		}
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/taskHelpers.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/taskHelpers.ts
@@ -8,6 +8,7 @@ import { IStringDictionary } from '../../../../../base/common/collections.js';
 import { MarkdownString } from '../../../../../base/common/htmlContent.js';
 import { URI } from '../../../../../base/common/uri.js';
 import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js';
+import { IMarkerService } from '../../../../../platform/markers/common/markers.js';
 import { IChatService } from '../../../chat/common/chatService.js';
 import { ILanguageModelsService } from '../../../chat/common/languageModels.js';
 import { ToolProgress } from '../../../chat/common/languageModelToolsService.js';
@@ -140,19 +141,20 @@ export async function resolveDependencyTasks(parentTask: Task, workspaceFolder:
  * Collects output, polling duration, and idle status for all terminals.
  */
 export async function collectTerminalResults(
-	terminals: ITerminalInstance[], task: Task, languageModelsService: ILanguageModelsService, chatService: IChatService, invocationContext: any, progress: ToolProgress, token: CancellationToken, isActive?: () => Promise<boolean>): Promise<Array<{ name: string; output: string; pollDurationMs: number; idle: boolean }>> {
+	terminals: ITerminalInstance[], task: Task, languageModelsService: ILanguageModelsService, markerService: IMarkerService, chatService: IChatService, invocationContext: any, progress: ToolProgress, token: CancellationToken, isActive?: () => Promise<boolean>, dependencyTasks?: Task[]): Promise<Array<{ name: string; output: string; pollDurationMs: number; idle: boolean }>> {
 	const results: Array<{ name: string; output: string; pollDurationMs: number; idle: boolean }> = [];
 	for (const terminal of terminals) {
 		progress.report({ message: new MarkdownString(`Checking output for \`${terminal.shellLaunchConfig.name ?? 'unknown'}\``) });
-		let outputAndIdle = await pollForOutputAndIdle({ getOutput: () => getOutput(terminal.xterm?.raw), isActive }, false, token, languageModelsService);
+		let outputAndIdle = await pollForOutputAndIdle({ getOutput: () => getOutput(terminal.xterm?.raw), isActive, task, dependencyTasks }, false, token, languageModelsService, markerService);
 		if (!outputAndIdle.terminalExecutionIdleBeforeTimeout) {
 			outputAndIdle = await racePollingOrPrompt(
-				() => pollForOutputAndIdle({ getOutput: () => getOutput(terminal.xterm?.raw), isActive }, true, token, languageModelsService),
+				() => pollForOutputAndIdle({ getOutput: () => getOutput(terminal.xterm?.raw), isActive, task, dependencyTasks }, true, token, languageModelsService, markerService),
 				() => promptForMorePolling(task._label, token, invocationContext, chatService),
 				outputAndIdle,
 				token,
 				languageModelsService,
-				{ getOutput: () => getOutput(terminal.xterm?.raw), isActive }
+				markerService,
+				{ getOutput: () => getOutput(terminal.xterm?.raw), isActive, dependencyTasks },
 			);
 		}
 		results.push({
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/tools/task/createAndRunTaskTool.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/tools/task/createAndRunTaskTool.ts
@@ -18,6 +18,7 @@ import { URI } from '../../../../../../../base/common/uri.js';
 import { IFileService } from '../../../../../../../platform/files/common/files.js';
 import { VSBuffer } from '../../../../../../../base/common/buffer.js';
 import { IConfigurationService } from '../../../../../../../platform/configuration/common/configuration.js';
+import { IMarkerService } from '../../../../../../../platform/markers/common/markers.js';
 
 type CreateAndRunTaskToolClassification = {
 	taskLabel: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The label of the task.' };
@@ -54,7 +55,8 @@ export class CreateAndRunTaskTool implements IToolImpl {
 		@ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService,
 		@IChatService private readonly _chatService: IChatService,
 		@IFileService private readonly _fileService: IFileService,
-		@IConfigurationService private readonly _configurationService: IConfigurationService
+		@IConfigurationService private readonly _configurationService: IConfigurationService,
+		@IMarkerService private readonly _markerService: IMarkerService
 	) { }
 
 	async invoke(invocation: IToolInvocation, _countTokens: CountTokensCallback, _progress: ToolProgress, token: CancellationToken): Promise<IToolResult> {
@@ -111,8 +113,8 @@ export class CreateAndRunTaskTool implements IToolImpl {
 		const raceResult = await Promise.race([this._tasksService.run(task), timeout(3000)]);
 		const result: ITaskSummary | undefined = raceResult && typeof raceResult === 'object' ? raceResult as ITaskSummary : undefined;
 
-		const resolvedDependencyTasks = await resolveDependencyTasks(task, args.workspaceFolder, this._configurationService, this._tasksService);
-		const resources = this._tasksService.getTerminalsForTasks(resolvedDependencyTasks ?? task);
+		const dependencyTasks = await resolveDependencyTasks(task, args.workspaceFolder, this._configurationService, this._tasksService);
+		const resources = this._tasksService.getTerminalsForTasks(dependencyTasks ?? task);
 		const terminals = resources?.map(resource => this._terminalService.instances.find(t => t.resource.path === resource?.path && t.resource.scheme === resource.scheme)).filter(Boolean) as ITerminalInstance[];
 		if (!terminals || terminals.length === 0) {
 			return { content: [{ kind: 'text', value: `Task started but no terminal was found for: ${args.task.label}` }], toolResultMessage: new MarkdownString(localize('copilotChat.noTerminal', 'Task started but no terminal was found for: `{0}`', args.task.label)) };
@@ -122,11 +124,13 @@ export class CreateAndRunTaskTool implements IToolImpl {
 			terminals,
 			task,
 			this._languageModelsService,
+			this._markerService,
 			this._chatService,
 			invocation.context!,
 			_progress,
 			token,
-			() => this._isTaskActive(task)
+			() => this._isTaskActive(task),
+			dependencyTasks
 		);
 		for (const r of terminalResults) {
 			this._telemetryService.publicLog2?.<CreateAndRunTaskToolEvent, CreateAndRunTaskToolClassification>('copilotChat.runTaskTool.createAndRunTask', {
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/tools/task/getTaskOutputTool.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/tools/task/getTaskOutputTool.ts
@@ -8,6 +8,7 @@ import { MarkdownString } from '../../../../../../../base/common/htmlContent.js'
 import { Disposable } from '../../../../../../../base/common/lifecycle.js';
 import { localize } from '../../../../../../../nls.js';
 import { IConfigurationService } from '../../../../../../../platform/configuration/common/configuration.js';
+import { IMarkerService } from '../../../../../../../platform/markers/common/markers.js';
 import { IChatService } from '../../../../../chat/common/chatService.js';
 import { ILanguageModelsService } from '../../../../../chat/common/languageModels.js';
 import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../../../chat/common/languageModelToolsService.js';
@@ -52,6 +53,7 @@ export class GetTaskOutputTool extends Disposable implements IToolImpl {
 		@IConfigurationService private readonly _configurationService: IConfigurationService,
 		@ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService,
 		@IChatService private readonly _chatService: IChatService,
+		@IMarkerService private readonly _markerService: IMarkerService
 	) {
 		super();
 	}
@@ -83,8 +85,8 @@ export class GetTaskOutputTool extends Disposable implements IToolImpl {
 			return { content: [{ kind: 'text', value: `Task not found: ${args.id}` }], toolResultMessage: new MarkdownString(localize('copilotChat.taskNotFound', 'Task not found: `{0}`', args.id)) };
 		}
 
-		const resolvedDependencyTasks = await resolveDependencyTasks(task, args.workspaceFolder, this._configurationService, this._tasksService);
-		const resources = this._tasksService.getTerminalsForTasks(resolvedDependencyTasks ?? task);
+		const dependencyTasks = await resolveDependencyTasks(task, args.workspaceFolder, this._configurationService, this._tasksService);
+		const resources = this._tasksService.getTerminalsForTasks(dependencyTasks ?? task);
 		const taskLabel = task._label;
 		const terminals = resources?.map(resource => this._terminalService.instances.find(t => t.resource.path === resource?.path && t.resource.scheme === resource.scheme)).filter(t => !!t);
 		if (!terminals || terminals.length === 0) {
@@ -94,10 +96,13 @@ export class GetTaskOutputTool extends Disposable implements IToolImpl {
 			terminals,
 			task,
 			this._languageModelsService,
+			this._markerService,
 			this._chatService,
 			invocation.context!,
 			_progress,
-			token
+			token,
+			undefined,
+			dependencyTasks
 		);
 		const details = terminalResults.map(r => `Terminal: ${r.name}\nOutput:\n${r.output}`).join('\n\n');
 		return {
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/tools/task/runTaskTool.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/tools/task/runTaskTool.ts
diff --git a/src/vs/workbench/contrib/terminalContrib/chatAgentTools/test/browser/bufferOutputPolling.test.ts b/src/vs/workbench/contrib/terminalContrib/chatAgentTools/test/browser/bufferOutputPolling.test.ts
