From 9a58fdef7bc174a78454d2aa651082fedd9ff473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Thu, 30 Oct 2025 11:01:08 -0700 Subject: [PATCH] surface the database type in the DatabaseClient options; when we create an instance without options, return the type derived from databases.json (or the name) --- src/databases/index.ts | 25 +++ .../__snapshots__/transpile.test.ts.snap | 211 ++++++++++++++++++ src/javascript/databaseClient.ts | 37 +++ src/javascript/transpile.test.ts | 43 ++++ src/javascript/transpile.ts | 5 + src/runtime/stdlib/databaseClient.ts | 5 +- src/vite/observable.ts | 5 +- 7 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 src/javascript/databaseClient.ts diff --git a/src/databases/index.ts b/src/databases/index.ts index 312b2f6..6041100 100644 --- a/src/databases/index.ts +++ b/src/databases/index.ts @@ -96,3 +96,28 @@ export async function getQueryCachePath( const cacheName = `${await nameHash(databaseName)}-${await hash(strings, ...params)}.json`; return join(sourceDir, ".observable", "cache", cacheName); } + +/** + * Reads partial database configurations from the .observable/databases.json file. + * For security reasons, no identifiers or passwords are returned. + */ +export async function getDatabaseConfigs( + sourcePath: string +): Promise>> { + const sourceDir = dirname(sourcePath); + const configPath = join(sourceDir, ".observable", "databases.json"); + const databases = new Map>(); + + try { + const configs = (await json(createReadStream(configPath, "utf-8"))) as Record< + string, + DatabaseConfig + >; + for (const [name, config] of Object.entries(configs)) { + databases.set(name, {type: config.type}); + } + } catch (error) { + if (!isEnoent(error)) throw error; + } + return databases; +} diff --git a/src/javascript/__snapshots__/transpile.test.ts.snap b/src/javascript/__snapshots__/transpile.test.ts.snap index 317c830..e90381c 100644 --- a/src/javascript/__snapshots__/transpile.test.ts.snap +++ b/src/javascript/__snapshots__/transpile.test.ts.snap @@ -1,5 +1,216 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`does not rewrite DatabaseClient with existing options 1`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("duckdb-base", {id: 1}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`does not rewrite DatabaseClient with existing options 2`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("duckdb-base", {type: "postgres"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient calls with type option 1`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("duckdb-base", {type: "duckdb"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient calls with type option 2`] = ` +{ + "autodisplay": false, + "body": "(DatabaseClient) => { +const client = DatabaseClient("postgres-base", {type: "postgres"}); +return {client}; +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [ + "client", + ], +} +`; + +exports[`rewrites DatabaseClient calls with type option 3`] = ` +{ + "autodisplay": false, + "body": "(DatabaseClient) => { +const x = DatabaseClient("mydb-base", {type: "sqlite"}); +return {x}; +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [ + "x", + ], +} +`; + +exports[`rewrites DatabaseClient with default database names 1`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("postgres", {type: "postgres"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient with default database names 2`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("duckdb", {type: "duckdb"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient with default database names 3`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("sqlite", {type: "sqlite"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient with file extension detection 1`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("data.duckdb", {type: "duckdb"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient with file extension detection 2`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("mydata.db", {type: "sqlite"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient with file extension detection 3`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("path/to/file.duckdb", {type: "duckdb"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`rewrites DatabaseClient with file extension detection 4`] = ` +{ + "autodisplay": true, + "body": "(DatabaseClient) => { +return ( +DatabaseClient("path/to/file.db", {type: "sqlite"}) +) +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [], +} +`; + +exports[`transpiles DatabaseClient 1`] = ` +{ + "autodisplay": false, + "body": "(DatabaseClient) => { +const client = new DatabaseClient("name", {type: "duckdb"}); +return {client}; +}", + "inputs": [ + "DatabaseClient", + ], + "output": undefined, + "outputs": [ + "client", + ], +} +`; + exports[`transpiles JavaScript expressions 1`] = ` { "autodisplay": true, diff --git a/src/javascript/databaseClient.ts b/src/javascript/databaseClient.ts new file mode 100644 index 0000000..2bba71f --- /dev/null +++ b/src/javascript/databaseClient.ts @@ -0,0 +1,37 @@ +import type {Node} from "acorn"; +import type {Sourcemap} from "./sourcemap.js"; +import {getStringLiteralValue, isStringLiteral} from "./strings.js"; +import {simple} from "./walk.js"; +import {DatabaseConfig} from "../databases/index.js"; + +export function rewriteDatabaseClient( + output: Sourcemap, + body: Node, + databases: Map> = new Map() +): void { + simple(body, { + CallExpression(node) { + if (node.callee.type !== "Identifier" || node.callee.name !== "DatabaseClient") return; + if (!isStringLiteral(node.arguments[0])) + throw new Error("DatabaseClient name must be a string literal"); + + // if options are passed, don't change them + if (node.arguments.length !== 1) return; + + const name = getStringLiteralValue(node.arguments[0]); + let type: string | undefined; + if (databases.has(name)) { + ({type} = databases.get(name)!); + } else { + // see defaults in databases/index.ts ; @todo: unify? + if (name === "postgres") type = "postgres"; + else if (name === "duckdb") type = "duckdb"; + else if (name === "sqlite") type = "sqlite"; + else if (/\.duckdb$/i.test(name)) type = "duckdb"; + else if (/\.db$/i.test(name)) type = "sqlite"; + else throw new Error(`database not found: ${name}`); + } + output.insertRight(node.arguments[0].end, `, {type: ${JSON.stringify(type)}}`); + } + }); +} diff --git a/src/javascript/transpile.test.ts b/src/javascript/transpile.test.ts index c6f730a..4a544c1 100644 --- a/src/javascript/transpile.test.ts +++ b/src/javascript/transpile.test.ts @@ -60,3 +60,46 @@ it("transpiles node cells", () => { expect(transpile("process.stdout.write(`Node $\\{process.version}`);", "node")).toMatchSnapshot(); expect(transpile("process.stdout.write(`Node \\$\\{process.version}`);", "node")).toMatchSnapshot(); }); + +it("transpiles DatabaseClient", () => { + expect(transpile('const client = new DatabaseClient("name", {type: "duckdb"});', "js")).toMatchSnapshot(); +}); + +it("rewrites DatabaseClient calls with type option", () => { + const databases = new Map([["duckdb-base", {type: "duckdb"}], ["postgres-base", {type: "postgres"}], ["mydb-base", {type: "sqlite"}]]); + expect(transpile('DatabaseClient("duckdb-base")', "js", {databases})).toMatchSnapshot(); + expect(transpile('const client = DatabaseClient("postgres-base");', "js", {databases})).toMatchSnapshot(); + expect(transpile('const x = DatabaseClient("mydb-base");', "js", {databases})).toMatchSnapshot(); +}); + +it("does not rewrite DatabaseClient with existing options", () => { + const databases = new Map([["duckdb-base", {type: "duckdb"}]]); + expect(transpile('DatabaseClient("duckdb-base", {id: 1})', "js", {databases})).toMatchSnapshot(); + expect(transpile('DatabaseClient("duckdb-base", {type: "postgres"})', "js", {databases})).toMatchSnapshot(); +}); + +it("throws error for unknown database name", () => { + const databases = new Map([["duckdb-base", {type: "duckdb"}]]); + expect(() => transpile('DatabaseClient("unknown")', "js", {databases})).toThrow("database not found: unknown"); +}); + +it("throws error for non-string-literal database name", () => { + const databases = new Map([["duckdb-base", {type: "duckdb"}]]); + expect(() => transpile("DatabaseClient(dbName)", "js", {databases})).toThrow("DatabaseClient name must be a string literal"); + expect(() => transpile("DatabaseClient(`template-${x}`)", "js", {databases})).toThrow("DatabaseClient name must be a string literal"); +}); + +it("rewrites DatabaseClient with default database names", () => { + const databases = new Map(); + expect(transpile('DatabaseClient("postgres")', "js", {databases})).toMatchSnapshot(); + expect(transpile('DatabaseClient("duckdb")', "js", {databases})).toMatchSnapshot(); + expect(transpile('DatabaseClient("sqlite")', "js", {databases})).toMatchSnapshot(); +}); + +it("rewrites DatabaseClient with file extension detection", () => { + const databases = new Map(); + expect(transpile('DatabaseClient("data.duckdb")', "js", {databases})).toMatchSnapshot(); + expect(transpile('DatabaseClient("mydata.db")', "js", {databases})).toMatchSnapshot(); + expect(transpile('DatabaseClient("path/to/file.duckdb")', "js", {databases})).toMatchSnapshot(); + expect(transpile('DatabaseClient("path/to/file.db")', "js", {databases})).toMatchSnapshot(); +}); diff --git a/src/javascript/transpile.ts b/src/javascript/transpile.ts index 7c3bd72..55ded80 100644 --- a/src/javascript/transpile.ts +++ b/src/javascript/transpile.ts @@ -1,5 +1,7 @@ +import {DatabaseConfig} from "../databases/index.js"; import type {Cell} from "../lib/notebook.js"; import {toCell} from "../lib/notebook.js"; +import {rewriteDatabaseClient} from "./databaseClient.js"; import {rewriteFileExpressions} from "./files.js"; import {hasImportDeclaration} from "./imports.js"; import {rewriteImportDeclarations, rewriteImportExpressions} from "./imports.js"; @@ -31,6 +33,8 @@ export type TranspileOptions = { resolveLocalImports?: boolean; /** If true, resolve file using import.meta.url (so Vite treats it as an asset). */ resolveFiles?: boolean; + /** If present, a map of database names to their types for DatabaseClient rewriting. */ + databases?: Map>; }; /** @deprecated */ @@ -86,6 +90,7 @@ export function transpileJavaScript( rewriteImportDeclarations(output, cell.body, inputs, options); rewriteImportExpressions(output, cell.body, options); if (options?.resolveFiles) rewriteFileExpressions(output, cell.body); + if (options?.databases) rewriteDatabaseClient(output, cell.body, options.databases); if (cell.expression) output.insertLeft(0, `return (\n`); output.insertLeft(0, `${async ? "async " : ""}(${inputs}) => {\n`); if (outputs.length > 0) output.insertRight(input.length, `\nreturn {${outputs}};`); diff --git a/src/runtime/stdlib/databaseClient.ts b/src/runtime/stdlib/databaseClient.ts index 14da584..b1071f4 100644 --- a/src/runtime/stdlib/databaseClient.ts +++ b/src/runtime/stdlib/databaseClient.ts @@ -33,6 +33,8 @@ export interface QueryOptionsSpec { id?: number; /** if present, results are at least as fresh as the specified date */ since?: Date | string | number; + /** if present, the type of the database */ + type?: string; } export interface QueryOptions extends QueryOptionsSpec { @@ -49,10 +51,11 @@ export const DatabaseClient = (name: string, options?: QueryOptionsSpec): Databa return new DatabaseClientImpl(name, normalizeOptions(options)); }; -function normalizeOptions({id, since}: QueryOptionsSpec = {}): QueryOptions { +function normalizeOptions({id, since, type}: QueryOptionsSpec = {}): QueryOptions { const options: QueryOptions = {}; if (id !== undefined) options.id = id; if (since !== undefined) options.since = new Date(since); + if (type !== undefined) options.type = type; return options; } diff --git a/src/vite/observable.ts b/src/vite/observable.ts index 6da24b5..27acaab 100644 --- a/src/vite/observable.ts +++ b/src/vite/observable.ts @@ -7,7 +7,7 @@ import {fileURLToPath} from "node:url"; import type {TemplateLiteral} from "acorn"; import {JSDOM} from "jsdom"; import type {PluginOption, IndexHtmlTransformContext} from "vite"; -import {getQueryCachePath} from "../databases/index.js"; +import {getDatabaseConfigs, getQueryCachePath} from "../databases/index.js"; import {getInterpreterCachePath, getInterpreterCommand} from "../interpreters/index.js"; import {getInterpreterMethod, isInterpreter} from "../lib/interpreters.js"; import type {Cell, Notebook} from "../lib/notebook.js"; @@ -84,6 +84,7 @@ export function observable({ const statics = new Set(); const assets = new Set(); const md = MarkdownRenderer({document}); + const databases = await getDatabaseConfigs(context.filename); const {version} = (await import("../../package.json", {with: {type: "json"}})).default; let generator = document.querySelector("meta[name=generator]"); @@ -195,7 +196,7 @@ ${Array.from(assets) ${notebook.cells .filter((cell) => !statics.has(cell)) .map((cell) => { - const transpiled = transpile(cell, {resolveFiles: true}); + const transpiled = transpile(cell, {resolveFiles: true, databases}); return ` define( {