Skip to content

Commit 89d48c1

Browse files
committed
Improves e2e test reliability and speed
- Adds access to vscode api in e2e tests
1 parent a8d893c commit 89d48c1

File tree

8 files changed

+393
-30
lines changed

8 files changed

+393
-30
lines changed

package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26756,8 +26756,9 @@
2675626756
"rebuild": "pnpm run reset && pnpm run build",
2675726757
"reset": "pnpm run clean && pnpm install --force",
2675826758
"test": "vscode-test",
26759-
"test:e2e": "playwright test -c tests/e2e/playwright.config.ts",
26760-
"test:e2e:insiders": "set VSCODE_VERSION=insiders && playwright test -c tests/e2e/playwright.config.ts",
26759+
"test:e2e": "pnpm run build:e2e-runner && playwright test -c tests/e2e/playwright.config.ts",
26760+
"test:e2e:insiders": "pnpm run build:e2e-runner && set VSCODE_VERSION=insiders && playwright test -c tests/e2e/playwright.config.ts",
26761+
"build:e2e-runner": "esbuild tests/e2e/runner/src/index.ts --bundle --outfile=tests/e2e/runner/dist/index.js --platform=node --external:vscode --format=cjs",
2676126762
"watch": "webpack --watch --mode development",
2676226763
"watch:extension": "webpack --watch --mode development --config-name extension",
2676326764
"watch:quick": "webpack --watch --mode development --env quick",

tests/e2e/baseTest.ts

Lines changed: 65 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,41 @@ import type { ElectronApplication, Page } from '@playwright/test';
66
import { _electron, test as base } from '@playwright/test';
77
import { downloadAndUnzipVSCode } from '@vscode/test-electron/out/download';
88
import { GitFixture } from './fixtures/git';
9+
import { VSCodeEvaluator } from './fixtures/vscodeEvaluator';
910
import { GitLensPage } from './pageObjects/gitLensPage';
1011

1112
export { expect } from '@playwright/test';
1213
export { GitFixture } from './fixtures/git';
14+
export type { VSCode } from './fixtures/vscodeEvaluator';
1315

1416
export const MaxTimeout = 10000;
1517

18+
/** Default VS Code settings applied to all E2E tests */
19+
const defaultUserSettings: Record<string, unknown> = {
20+
// Disable telemetry
21+
'telemetry.telemetryLevel': 'off',
22+
// Disable distracting UI elements
23+
'workbench.tips.enabled': false,
24+
'workbench.startupEditor': 'none',
25+
'workbench.enableExperiments': false,
26+
'workbench.welcomePage.walkthroughs.openOnInstall': false,
27+
// Disable extension recommendations
28+
'extensions.ignoreRecommendations': true,
29+
'extensions.autoUpdate': false,
30+
// Disable update checks
31+
'update.mode': 'none',
32+
// Use custom dialogs for consistent behavior
33+
'files.simpleDialog.enable': true,
34+
'window.dialogStyle': 'custom',
35+
};
36+
1637
export interface LaunchOptions {
1738
vscodeVersion?: string;
39+
/**
40+
* User settings to apply to VS Code.
41+
* These are merged with (and override) the default test settings.
42+
*/
43+
userSettings?: Record<string, unknown>;
1844
/**
1945
* Optional async setup callback that runs before VS Code launches.
2046
* Use this to create a test repo, files, or any other setup needed.
@@ -28,6 +54,20 @@ export interface VSCodeInstance {
2854
page: Page;
2955
electronApp: ElectronApplication;
3056
gitlens: GitLensPage;
57+
/**
58+
* Evaluate a function in the VS Code Extension Host context.
59+
* The function receives the `vscode` module as its first argument.
60+
*
61+
* @example
62+
* ```ts
63+
* await vscode.evaluate(vscode => {
64+
* vscode.commands.executeCommand('gitlens.showCommitGraph');
65+
* });
66+
*
67+
* const version = await vscode.evaluate(vscode => vscode.version);
68+
* ```
69+
*/
70+
evaluate: VSCodeEvaluator['evaluate'];
3171
}
3272

3373
/** Base fixtures for all E2E tests */
@@ -57,6 +97,17 @@ export const test = base.extend<BaseFixtures, WorkerFixtures>({
5797
const tempDir = await createTmpDir();
5898
const vscodePath = await downloadAndUnzipVSCode(vscodeOptions.vscodeVersion ?? 'stable');
5999
const extensionPath = path.join(__dirname, '..', '..');
100+
const runnerPath = path.join(__dirname, 'runner', 'dist');
101+
const userDataDir = path.join(tempDir, 'user-data');
102+
103+
// Write user settings before launching VS Code
104+
const settingsDir = path.join(userDataDir, 'User');
105+
await fs.promises.mkdir(settingsDir, { recursive: true });
106+
const mergedSettings = { ...defaultUserSettings, ...vscodeOptions.userSettings };
107+
await fs.promises.writeFile(
108+
path.join(settingsDir, 'settings.json'),
109+
JSON.stringify(mergedSettings, null, '\t'),
110+
);
60111

61112
// Run setup callback if provided, otherwise open extension folder
62113
const workspacePath = vscodeOptions.setup ? await vscodeOptions.setup() : extensionPath;
@@ -71,21 +122,32 @@ export const test = base.extend<BaseFixtures, WorkerFixtures>({
71122
'--skip-release-notes',
72123
'--disable-workspace-trust',
73124
`--extensionDevelopmentPath=${extensionPath}`,
125+
`--extensionTestsPath=${runnerPath}`,
74126
`--extensions-dir=${path.join(tempDir, 'extensions')}`,
75-
`--user-data-dir=${path.join(tempDir, 'user-data')}`,
127+
`--user-data-dir=${userDataDir}`,
76128
workspacePath,
77129
],
78130
});
79131

132+
// Connect to the VS Code test server using Playwright's internal API
133+
const evaluator = await VSCodeEvaluator.connect(electronApp);
134+
const evaluate = evaluator.evaluate.bind(evaluator);
135+
80136
const page = await electronApp.firstWindow();
81-
const gitlens = new GitLensPage(page);
137+
const gitlens = new GitLensPage(page, evaluate);
82138

83139
// Wait for GitLens to activate before providing to tests
84140
await gitlens.waitForActivation();
85141

86-
await use({ page: page, electronApp: electronApp, gitlens: gitlens } satisfies VSCodeInstance);
142+
await use({
143+
page: page,
144+
electronApp: electronApp,
145+
gitlens: gitlens,
146+
evaluate: evaluate,
147+
} satisfies VSCodeInstance);
87148

88149
// Cleanup
150+
evaluator.close();
89151
await electronApp.close();
90152
await fs.promises.rm(tempDir, { recursive: true, force: true }).catch(() => {});
91153
},
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/**
2+
* VS Code Evaluator - HTTP Client
3+
*
4+
* Connects to the VS Code test runner HTTP server and allows
5+
* executing functions with access to the VS Code API.
6+
*
7+
* Uses Playwright's internal API to access the Electron process, similar
8+
* to how vscode-test-playwright does it.
9+
*/
10+
import type { ChildProcess } from 'child_process';
11+
import type { EventEmitter } from 'events';
12+
import readline from 'readline';
13+
import type { ElectronApplication } from '@playwright/test';
14+
import { _electron } from '@playwright/test';
15+
16+
// Re-export vscode types for use in evaluate callbacks
17+
export type VSCode = typeof import('vscode');
18+
19+
interface InvokeRequest {
20+
fn: string;
21+
params?: unknown[];
22+
}
23+
24+
interface InvokeResponse {
25+
result?: unknown;
26+
error?: { message: string; stack?: string };
27+
}
28+
29+
// Internal Playwright API types
30+
interface ElectronAppImpl {
31+
_process: ChildProcess;
32+
_nodeConnection?: {
33+
_browserLogsCollector?: {
34+
recentLogs(): string[];
35+
};
36+
};
37+
}
38+
39+
export class VSCodeEvaluator {
40+
private serverUrl: string;
41+
42+
private constructor(serverUrl: string) {
43+
this.serverUrl = serverUrl;
44+
}
45+
46+
/**
47+
* Connect to the VS Code test server using Playwright's internal API.
48+
* Uses the same approach as vscode-test-playwright to access the process.
49+
*
50+
* @param electronApp - The ElectronApplication from Playwright
51+
* @param timeout - Connection timeout in ms
52+
*/
53+
static async connect(electronApp: ElectronApplication, timeout = 30000): Promise<VSCodeEvaluator> {
54+
// Access Playwright's internal implementation to get the process
55+
// The _electron._connection.toImpl() method converts public API objects to internal implementations
56+
57+
const connection = (_electron as any)._connection;
58+
const electronAppImpl = connection.toImpl(electronApp) as ElectronAppImpl;
59+
const process = electronAppImpl._process;
60+
61+
// Check recent logs first (in case server already started)
62+
const vscodeTestServerRegExp = /VSCodeTestServer listening on (http:\/\/[^\s]+)/;
63+
const recentLogs = electronAppImpl._nodeConnection?._browserLogsCollector?.recentLogs() ?? [];
64+
let match = recentLogs.map((s: string) => s.match(vscodeTestServerRegExp)).find(Boolean) as
65+
| RegExpMatchArray
66+
| undefined;
67+
68+
// If not found in recent logs, wait for it
69+
if (!match) {
70+
match = await this.waitForLine(process, vscodeTestServerRegExp, timeout);
71+
}
72+
73+
const serverUrl = match[1];
74+
return new VSCodeEvaluator(serverUrl);
75+
}
76+
77+
/**
78+
* Wait for a line matching the regex in the process stderr.
79+
* Adapted from Playwright's electron.ts
80+
*/
81+
private static waitForLine(process: ChildProcess, regex: RegExp, timeout: number): Promise<RegExpMatchArray> {
82+
type Listener = { emitter: EventEmitter; eventName: string | symbol; handler: (...args: any[]) => void };
83+
84+
function addEventListener(
85+
emitter: EventEmitter,
86+
eventName: string | symbol,
87+
handler: (...args: any[]) => void,
88+
): Listener {
89+
emitter.on(eventName, handler);
90+
return { emitter: emitter, eventName: eventName, handler: handler };
91+
}
92+
93+
function removeEventListeners(listeners: Listener[]) {
94+
for (const listener of listeners) {
95+
listener.emitter.removeListener(listener.eventName, listener.handler);
96+
}
97+
listeners.splice(0, listeners.length);
98+
}
99+
100+
return new Promise((resolve, reject) => {
101+
const rl = readline.createInterface({ input: process.stderr! });
102+
const failError = new Error('Process failed to launch!');
103+
const timeoutError = new Error(`Timeout waiting for VSCodeTestServer (${timeout}ms)`);
104+
105+
const listeners = [
106+
addEventListener(rl, 'line', onLine),
107+
addEventListener(rl, 'close', () => reject(failError)),
108+
addEventListener(process, 'exit', () => reject(failError)),
109+
addEventListener(process, 'error', () => reject(failError)),
110+
];
111+
112+
const timer = setTimeout(() => {
113+
cleanup();
114+
reject(timeoutError);
115+
}, timeout);
116+
117+
function onLine(line: string) {
118+
const match = line.match(regex);
119+
if (!match) return;
120+
cleanup();
121+
resolve(match);
122+
}
123+
124+
function cleanup() {
125+
clearTimeout(timer);
126+
removeEventListeners(listeners);
127+
}
128+
});
129+
}
130+
131+
/**
132+
* Evaluate a function in the VS Code Extension Host context.
133+
* The function receives the `vscode` module as its first argument.
134+
*
135+
* @example
136+
* ```ts
137+
* await evaluator.evaluate(vscode => {
138+
* vscode.commands.executeCommand('gitlens.showCommitGraph');
139+
* });
140+
*
141+
* const version = await evaluator.evaluate(vscode => vscode.version);
142+
* ```
143+
*/
144+
evaluate<R>(fn: (vscode: VSCode) => R | Promise<R>): Promise<R>;
145+
evaluate<R, A>(fn: (vscode: VSCode, arg: A) => R | Promise<R>, arg: A): Promise<R>;
146+
async evaluate<R, A>(fn: (vscode: VSCode, arg?: A) => R | Promise<R>, arg?: A): Promise<R> {
147+
const params = arg !== undefined ? [arg] : [];
148+
149+
const request: InvokeRequest = {
150+
fn: fn.toString(),
151+
params: params,
152+
};
153+
154+
const res = await fetch(`${this.serverUrl}/invoke`, {
155+
method: 'POST',
156+
headers: { 'Content-Type': 'application/json' },
157+
body: JSON.stringify(request),
158+
});
159+
160+
const response = (await res.json()) as InvokeResponse;
161+
162+
if (response.error) {
163+
const err = new Error(response.error.message);
164+
err.stack = response.error.stack;
165+
throw err;
166+
}
167+
168+
return response.result as R;
169+
}
170+
171+
/**
172+
* Close the connection (no-op for HTTP, kept for API compatibility).
173+
*/
174+
close(): void {
175+
// No-op for HTTP - each request is independent
176+
}
177+
}

0 commit comments

Comments
 (0)