Implement local ssh daemon (#36)

Author: mustard-mh

package.json

@@ -4,6 +4,7 @@
 	"description": "Gitpod Support",
 	"publisher": "gitpod",
 	"version": "0.0.98",
+	"daemonVersion": "0.0.1",
 	"license": "MIT",
 	"icon": "resources/gitpod.png",
 	"repository": {
@@ -41,7 +42,8 @@
 		"onCommand:gitpod.installLocalExtensions",
 		"onAuthenticationRequest:gitpod",
 		"onUri",
-		"onStartupFinished"
+		"onStartupFinished",
+		"onResolveRemoteAuthority:ssh-remote"
 	],
 	"contributes": {
 		"authentication": [
@@ -71,6 +73,16 @@
 						"description": "Use the local companion app to connect to a remote workspace.\nWarning: Connecting to a remote workspace using local companion app will be removed in the near future.",
 						"default": false,
 						"scope": "application"
+					},
+					"gitpod.lsshPort": {
+						"type": "number",
+						"description": "The port to use for the local SSH server.",
+						"scope": "application"
+					},
+					"gitpod.lsshIpcPort": {
+						"type": "number",
+						"description": "The port to use for the local SSH ipc server.",
+						"scope": "application"
 					}
 				}
 			},
@@ -137,9 +149,10 @@
 		"webpack": "webpack --mode development",
 		"compile": "tsc -b",
 		"watch": "tsc -b -w",
-		"package": "npx vsce package --yarn",
+		"package": "vsce package --yarn",
 		"lint": "eslint . --ext .ts",
-		"test": "mocha -u tdd"
+		"test": "mocha -u tdd",
+		"proto-gen": "bash ./src/proto/generate.sh"
 	},
 	"devDependencies": {
 		"@types/google-protobuf": "^3.7.4",
@@ -148,20 +161,24 @@
 		"@types/node": "16.x",
 		"@types/semver": "^7.3.10",
 		"@types/ssh2": "^0.5.52",
+		"@types/sshpk": "^1.17.1",
 		"@types/tmp": "^0.2.1",
 		"@types/uuid": "8.0.0",
 		"@types/vscode": "1.74.0",
 		"@types/webpack": "^5.28.0",
-		"@types/ws": "^7.4.7",
+		"@types/ws": "^8.5.4",
 		"@types/yazl": "^2.4.2",
 		"@typescript-eslint/eslint-plugin": "^5.19.0",
 		"@typescript-eslint/parser": "^5.19.0",
+		"@vscode/vsce": "^2.18.0",
 		"eslint": "^8.13.0",
 		"eslint-plugin-header": "3.1.1",
 		"eslint-plugin-jsdoc": "^19.1.0",
+		"grpc-tools": "^1.12.4",
 		"minimist": "^1.2.6",
 		"mocha": "^10.0.0",
 		"ts-loader": "^9.2.7",
+		"ts-proto": "^1.140.0",
 		"typescript": "^4.6.3",
 		"webpack": "^5.42.0",
 		"webpack-cli": "^4.7.2"
@@ -171,21 +188,31 @@
 		"@gitpod/gitpod-protocol": "main",
 		"@gitpod/local-app-api-grpcweb": "main",
 		"@gitpod/public-api": "^0.1.5-main.6530",
+		"@gitpod/supervisor-api-grpc": "0.1.5-main.6711",
+		"@grpc/grpc-js": "^1.8.8",
 		"@improbable-eng/grpc-web-node-http-transport": "^0.14.0",
+		"@microsoft/dev-tunnels-ssh": "^3.11.2",
+		"@microsoft/dev-tunnels-ssh-keys": "^3.11.2",
+		"@microsoft/dev-tunnels-ssh-tcp": "^3.11.2",
 		"@segment/analytics-node": "^1.0.0-beta.24",
 		"configcat-node": "^8.0.0",
 		"js-yaml": "^4.1.0",
+		"long": "^5.2.1",
+		"nice-grpc": "^2.1.3",
+		"nice-grpc-common": "^2.0.1",
 		"node-fetch-commonjs": "^3.2.4",
+		"node-rsa": "^1.1.1",
 		"pkce-challenge": "^3.0.0",
+		"prom-client": "^14.1.1",
+		"protobufjs": "^7.2.2",
 		"semver": "^7.3.7",
 		"ssh-config": "^4.1.6",
 		"ssh2": "^1.10.0",
+		"sshpk": "^1.17.0",
 		"tmp": "^0.2.1",
 		"uuid": "8.1.0",
-		"yazl": "^2.5.1",
-		"@grpc/grpc-js": "^1.8.8",
-		"long":"^5.2.1",
-		"protobufjs": "^7.2.2",
-		"prom-client": "^14.1.1"
+		"winston": "^3.8.2",
+		"ws": "^8.13.0",
+		"yazl": "^2.5.1"
 	}
 }

src/commands/logs.ts

@@ -12,6 +12,7 @@ import { Command } from '../commandManager';
 import { ILogService } from '../services/logService';
 import { INotificationService } from '../services/notificationService';
 import { ITelemetryService, UserFlowTelemetry } from '../services/telemetryService';
+import { Configuration } from '../configuration';
 
 interface IFile {
 	path: string;
@@ -82,7 +83,18 @@ export class ExportLogsCommand implements Command {
 				return;
 			}
 
-			return zip(saveUri.fsPath, remoteLogFiles.concat(localLogFiles));
+			const daemonLogs: IFile[] = [];
+
+			const daemonPath = path.posix.join('./lssh', 'daemon.log');
+			const daemonLogContent = await vscode.workspace.fs.readFile(vscode.Uri.file(Configuration.getDaemonLogPath()));
+			if (daemonLogContent.byteLength > 0) {
+				daemonLogs.push({
+					path: daemonPath,
+					contents: Buffer.from(daemonLogContent)
+				});
+			}
+
+			return zip(saveUri.fsPath, remoteLogFiles.concat(localLogFiles).concat(daemonLogs));
 		});
 	}
 }

src/common/async.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 export function timeout(millis: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, millis));
+	return new Promise((resolve) => setTimeout(resolve, millis));
 }
 
 export interface ITask<T> {
@@ -26,3 +26,23 @@ export async function retry<T>(task: ITask<Promise<T>>, delay: number, retries:
 
 	throw lastError;
 }
+
+export async function retryWithStop<T>(task: (stop: () => void) => Promise<T>, delay: number, retries: number): Promise<T> {
+	let lastError: Error | undefined;
+	let stopped = false;
+	const stop = () => {
+		stopped = true;
+	};
+	for (let i = 0; i < retries; i++) {
+		try {
+			return await task(stop);
+		} catch (error) {
+			lastError = error;
+			if (stopped) {
+				break;
+			}
+			await timeout(delay);
+		}
+	}
+	throw lastError;
+}

src/common/telemetry.ts

@@ -263,7 +263,7 @@ export class BaseTelemetryReporter extends Disposable {
 			cleanupPatterns.push(new RegExp(this.extension.extensionPath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi'));
 		}
 
-		let updatedStack = stack;
+		let updatedStack = stack?.toString() ?? '';
 
 		if (anonymizeFilePaths) {
 			const cleanUpIndexes: [number, number][] = [];

src/common/utils.ts

@@ -120,3 +120,11 @@ export function arrayEquals<T>(one: ReadonlyArray<T> | undefined, other: Readonl
 
 	return true;
 }
+
+export function getServiceURL(gitpodHost: string): string {
+	return new URL(gitpodHost).toString().replace(/\/$/, '');
+}
+
+export function getLocalSSHUrl(gitpodHost: string): string {
+	return 'lssh.' + (new URL(gitpodHost)).hostname;
+}

src/configuration.ts

@@ -3,6 +3,8 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { tmpdir } from 'os';
+import { join } from 'path';
 import * as vscode from 'vscode';
 
 // Use these functions instead of `vscode.workspace.getConfiguration` API
@@ -17,12 +19,43 @@ function getShowReleaseNotes() {
     return vscode.workspace.getConfiguration('gitpod').get<boolean>('showReleaseNotes', true);
 }
 
-function getUseLocalApp() {
+function getUseLocalApp(useLocalSSHServer?: boolean) {
+    if (useLocalSSHServer) {
+        return false;
+    }
     return vscode.workspace.getConfiguration('gitpod').get<boolean>('remote.useLocalApp', false);
 }
 
+function getLocalSSHServerPort() {
+    // TODO(local-ssh): VSCodium?
+    // use `sudo lsof -i:<port>` to check if the port is already in use
+    let defaultPort = 42025;
+    if (vscode.env.appName.includes('Insiders')) {
+        defaultPort = 42026;
+    }
+    return vscode.workspace.getConfiguration('gitpod').get<number>('lsshPort', defaultPort) || defaultPort;
+}
+
+function getLocalSshIpcPort() {
+    let defaultPort = 41025;
+    if (vscode.env.appName.includes('Insiders')) {
+        defaultPort = 41026;
+    }
+    return vscode.workspace.getConfiguration('gitpod').get<number>('lsshIpcPort', defaultPort) || defaultPort;
+}
+
+function getDaemonLogPath(): string {
+    if (vscode.env.appName.includes('Insiders')) {
+        return join(tmpdir(), 'gitpod-vscode-daemon-insiders.log');
+    }
+    return join(tmpdir(), 'gitpod-vscode-daemon.log');
+}
+
 export const Configuration = {
     getGitpodHost,
     getShowReleaseNotes,
-    getUseLocalApp
+    getUseLocalApp,
+    getLocalSSHServerPort,
+    getLocalSshIpcPort,
+    getDaemonLogPath,
 };

src/daemonStarter.ts

@@ -0,0 +1,64 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Gitpod. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { join } from 'path';
+import { spawn } from 'child_process';
+import { DaemonOptions, ExitCode } from './local-ssh/common';
+import { ILogService } from './services/logService';
+import { timeout } from './common/async';
+import { Configuration } from './configuration';
+import { ITelemetryService } from './services/telemetryService';
+
+export async function ensureDaemonStarted(logService: ILogService, telemetryService: ITelemetryService, retry = 10) {
+    if (retry < 0) {
+        return;
+    }
+    const process = await tryStartDaemon(logService);
+    const ok = await new Promise<boolean>(resolve => {
+        process.once('exit', async code => {
+            const humanReadableCode = code !== null ? ExitCode[code] : 'UNSPECIFIED';
+            logService.error(`lssh exit with code ${humanReadableCode} ${code}`);
+            switch (code) {
+                case ExitCode.OK:
+                case ExitCode.ListenPortFailed:
+                    resolve(true);
+                    return;
+            }
+            logService.error('lssh unexpectedly exit with code: ' + code + ' attempt retry: ' + retry);
+            resolve(false);
+            telemetryService.sendTelemetryException(new Error(`unexpectedly exit with code ${humanReadableCode} ${code} attempt retry: ${retry}`), { humanReadableCode, code: code?.toString() ?? 'null' });
+        });
+    });
+    if (!ok) {
+        await timeout(1000);
+        await ensureDaemonStarted(logService, telemetryService, retry - 1);
+    }
+}
+
+const DefaultDaemonOptions: DaemonOptions = {
+    logLevel: 'info',
+    // use `sudo lsof -i:<port>` to check if the port is already in use
+    ipcPort: Configuration.getLocalSshIpcPort(),
+    serverPort: Configuration.getLocalSSHServerPort(),
+
+    logFilePath: Configuration.getDaemonLogPath(),
+};
+
+export function parseArgv(options: DaemonOptions): string[] {
+    return [options.logFilePath, options.logLevel, options.serverPort.toString(), options.ipcPort.toString()];
+}
+
+export async function tryStartDaemon(logService: ILogService, options?: Partial<DaemonOptions>) {
+    const opts: DaemonOptions = { ...DefaultDaemonOptions, ...options };
+    const args: string[] = [join(__dirname, 'local-ssh/daemon.js'), ...parseArgv(opts)];
+    logService.debug('going to start local-ssh daemon', opts, args);
+    const daemon = spawn(process.execPath, args, {
+        detached: true,
+        stdio: 'ignore',
+        env: process.env
+    });
+    daemon.unref();
+    return daemon;
+}

src/experiments.ts

@@ -11,7 +11,8 @@ import { ISessionService } from './services/sessionService';
 import { ILogService } from './services/logService';
 
 const EXPERTIMENTAL_SETTINGS = [
-    'gitpod.remote.useLocalApp'
+    'gitpod.remote.useLocalApp',
+    // 'gitpod.remote.useLocalSSHServer',
 ];
 
 export class ExperimentalSettings {
@@ -114,6 +115,17 @@ export class ExperimentalSettings {
     dispose(): void {
         this.configcatClient.dispose();
     }
+
+    /**
+     * @see https://app.configcat.com/08da1258-64fb-4a8e-8a1e-51de773884f6/08da1258-6541-4fc7-8b61-c8b47f82f3a0/08da1258-6512-4ec0-80a3-3f6aa301f853?settingId=75503
+     */
+    async getUseLocalSSHServer(gitpodHost: string): Promise<boolean> {
+        return (await this.getRaw<boolean>('gitpod_desktop_use_local_ssh_server', { gitpodHost })) ?? false;
+    }
+
+    async getUsePublicAPI(gitpodHost: string): Promise<boolean> {
+        return (await this.getRaw<boolean>('gitpod_experimental_publicApi', { gitpodHost })) ?? false;
+    }
 }
 
 export function isUserOverrideSetting(key: string): boolean {

src/extension.ts

@@ -21,6 +21,8 @@ import { SessionService } from './services/sessionService';
 import { CommandManager } from './commandManager';
 import { SignInCommand } from './commands/account';
 import { ExportLogsCommand } from './commands/logs';
+import { ensureDaemonStarted } from './daemonStarter';
+import { ExtensionServiceServer } from './local-ssh/ipc/extension';
 
 // connect-web uses fetch api, so we need to polyfill it
 if (!global.fetch) {
@@ -59,7 +61,7 @@ export async function activate(context: vscode.ExtensionContext) {
 
 		logger.info(`${extensionId}/${packageJSON.version} (${os.release()} ${os.platform()} ${os.arch()}) vscode/${vscode.version} (${vscode.env.appName})`);
 
-		telemetryService = new TelemetryService(extensionId, packageJSON.version, packageJSON.segmentKey, logger!);
+		telemetryService = new TelemetryService(extensionId, packageJSON.version, packageJSON.segmentKey, logger!, context.extensionMode === vscode.ExtensionMode.Production);
 
 		const notificationService = new NotificationService(telemetryService);
 
@@ -85,6 +87,9 @@ export async function activate(context: vscode.ExtensionContext) {
 		const remoteConnector = new RemoteConnector(context, sessionService, hostService, experiments, logger, telemetryService, notificationService);
 		context.subscriptions.push(remoteConnector);
 
+		const extensionIPCService = new ExtensionServiceServer(logger, sessionService, hostService, notificationService, telemetryService, experiments);
+		context.subscriptions.push(extensionIPCService);
+
 		context.subscriptions.push(vscode.window.registerUriHandler({
 			handleUri(uri: vscode.Uri) {
 				// logger.trace('Handling Uri...', uri.toString());
@@ -102,6 +107,8 @@ export async function activate(context: vscode.ExtensionContext) {
 		commandManager.register(new SignInCommand(sessionService));
 		commandManager.register(new ExportLogsCommand(context.logUri, notificationService, telemetryService, logger));
 
+		ensureDaemonStarted(logger, telemetryService, 3).then(() => { }).catch(e => { logger?.error(e) });
+
 		if (!context.globalState.get<boolean>(FIRST_INSTALL_KEY, false)) {
 			context.globalState.update(FIRST_INSTALL_KEY, true);
 			telemetryService.sendTelemetryEvent('gitpod_desktop_installation', { kind: 'install' });
@@ -138,7 +145,7 @@ export async function activate(context: vscode.ExtensionContext) {
 		logger?.info('Activation properties:', JSON.stringify(rawActivateProperties, undefined, 2));
 		telemetryService?.sendTelemetryEvent('vscode_desktop_activate', {
 			...rawActivateProperties,
-			remoteUri:	String(!!rawActivateProperties.remoteUri)
+			remoteUri: String(!!rawActivateProperties.remoteUri)
 		});
 	}
 }