pass the address in the effects; implement a temporary server for building

Author: Fil

src/build.ts

@@ -13,6 +13,7 @@ import {parseMarkdown} from "./markdown.js";
 import {extractNodeSpecifier} from "./node.js";
 import {extractNpmSpecifier, populateNpmCache, resolveNpmImport} from "./npm.js";
 import {isAssetPath, isPathImport, relativePath, resolvePath} from "./path.js";
+import {preview} from "./preview.js";
 import {renderPage} from "./render.js";
 import type {Resolvers} from "./resolvers.js";
 import {getModuleResolver, getResolvers} from "./resolvers.js";
@@ -150,6 +151,12 @@ export async function build(
     }
   }
 
+  // Launch a server for chained data loaders. TODO configure host & port?
+  const {server} = await preview({root, verbose: false, hostname: "127.0.0.1"});
+  const a = server.address();
+  if (!a || typeof a !== "object") throw new Error("Couldn't launch server for chained data loaders!");
+  const address = `http://${a.address}:${a.port}/`;
+
   // Copy over the referenced files, accumulating hashed aliases.
   for (const file of files) {
     let sourcePath = join(root, file);
@@ -161,7 +168,7 @@ export async function build(
         continue;
       }
       try {
-        sourcePath = join(root, await loader.load(effects));
+        sourcePath = join(root, await loader.load({...effects, address}));
       } catch (error) {
         if (!isEnoent(error)) throw error;
         effects.logger.error(red("error: missing referenced file"));
@@ -176,6 +183,9 @@ export async function build(
     await effects.writeFile(alias, contents);
   }
 
+  // TODO: server.close() might be enough?
+  await new Promise((closed) => server.close(closed));
+
   // Download npm imports. TODO It might be nice to use content hashes for
   // these, too, but it would involve rewriting the files since populateNpmCache
   // doesn’t let you pass in a resolver.

src/dataloader.ts

@@ -34,13 +34,9 @@ export const defaultInterpreters: Record<string, string[]> = {
 export interface LoadEffects {
   logger: Logger;
   output: Writer;
+  address?: string;
 }
 
-const defaultEffects: LoadEffects = {
-  logger: console,
-  output: process.stdout
-};
-
 export interface LoaderOptions {
   root: string;
   path: string;
@@ -230,7 +226,7 @@ export abstract class Loader {
    * to the source root; this is within the .observablehq/cache folder within
    * the source root.
    */
-  async load(effects = defaultEffects): Promise<string> {
+  async load(effects): Promise<string> {
     const key = join(this.root, this.targetPath);
     let command = runningCommands.get(key);
     if (!command) {
@@ -287,7 +283,7 @@ export abstract class Loader {
     return command;
   }
 
-  abstract exec(output: WriteStream, effects?: LoadEffects): Promise<void>;
+  abstract exec(output: WriteStream, effects: LoadEffects): Promise<void>;
 }
 
 interface CommandLoaderOptions extends LoaderOptions {
@@ -316,10 +312,9 @@ class CommandLoader extends Loader {
     this.args = args;
   }
 
-  async exec(output: WriteStream): Promise<void> {
-    const address = process.env.OBSERVABLEHQ_ADDRESS;
-    if (address == null) throw new Error("chained data loaders are not implemented in this context");
-    const env = {...process.env, SERVER: `${address}_chain${this.targetPath}::`};
+  async exec(output: WriteStream, effects: LoadEffects): Promise<void> {
+    if (effects.address == null) throw new Error("chained data loaders are not implemented in this context");
+    const env = {...process.env, SERVER: `${effects.address}_chain${this.targetPath}::`};
     const subprocess = spawn(this.command, this.args, {windowsHide: true, stdio: ["ignore", output, "inherit"], env});
     const code = await new Promise((resolve, reject) => {
       subprocess.on("error", reject);
@@ -346,7 +341,7 @@ class ZipExtractor extends Loader {
     this.inflatePath = inflatePath;
   }
 
-  async exec(output: WriteStream, effects?: LoadEffects): Promise<void> {
+  async exec(output: WriteStream, effects: LoadEffects): Promise<void> {
     const archivePath = join(this.root, await this.preload(effects));
     const file = (await JSZip.loadAsync(await readFile(archivePath))).file(this.inflatePath);
     if (!file) throw Object.assign(new Error("file not found"), {code: "ENOENT"});
@@ -373,7 +368,7 @@ class TarExtractor extends Loader {
     this.gunzip = gunzip;
   }
 
-  async exec(output: WriteStream, effects?: LoadEffects): Promise<void> {
+  async exec(output: WriteStream, effects: LoadEffects): Promise<void> {
     const archivePath = join(this.root, await this.preload(effects));
     const tar = extract();
     const input = createReadStream(archivePath);

src/preview.ts

@@ -17,6 +17,7 @@ import {WebSocketServer} from "ws";
 import type {Config} from "./config.js";
 import {readConfig} from "./config.js";
 import type {LoaderResolver} from "./dataloader.js";
+import type {LoadEffects} from "./dataloader.js";
 import {HttpError, isEnoent, isHttpError, isSystemError} from "./error.js";
 import {getClientPath} from "./files.js";
 import type {FileWatchers} from "./fileWatchers.js";
@@ -54,17 +55,20 @@ export class PreviewServer {
   private readonly _server: ReturnType<typeof createServer>;
   private readonly _socketServer: WebSocketServer;
   private readonly _verbose: boolean;
+  private readonly _effects: LoadEffects;
 
   private constructor({
     config,
     root,
     server,
-    verbose
+    verbose,
+    effects
   }: {
     config?: string;
     root?: string;
     server: Server;
     verbose: boolean;
+    effects: LoadEffects;
   }) {
     this._config = config;
     this._root = root;
@@ -73,6 +77,7 @@ export class PreviewServer {
     this._server.on("request", this._handleRequest);
     this._socketServer = new WebSocketServer({server: this._server});
     this._socketServer.on("connection", this._handleConnection);
+    this._effects = effects;
   }
 
   static async start({verbose = true, hostname, port, open, ...options}: PreviewOptions) {
@@ -102,8 +107,12 @@ export class PreviewServer {
       console.log("");
     }
     if (open) openBrowser(address);
-    process.env.OBSERVABLEHQ_ADDRESS = address; // global!
-    return new PreviewServer({server, verbose, ...options});
+    const effects: LoadEffects = {
+      logger: console,
+      output: process.stdout,
+      address
+    };
+    return new PreviewServer({server, verbose, effects, ...options});
   }
 
   async _readConfig() {
@@ -159,12 +168,12 @@ export class PreviewServer {
         const [caller, path] = pathname.slice("/_chain".length).split("::");
         // now any file that watches caller should also watch path
         console.warn("chained data loader", {caller, path, loaders});
-        const file = await getFile(path, root, loaders);
+        const file = await this.getFile(path, root, loaders);
         if (file !== undefined) return void send(req, file, {root}).pipe(res);
         throw new HttpError(`Not found: ${pathname}`, 404);
       } else if (pathname.startsWith("/_file/")) {
         const path = pathname.slice("/_file".length);
-        const file = await getFile(path, root, loaders);
+        const file = await this.getFile(path, root, loaders);
         if (file !== undefined) return void send(req, file, {root}).pipe(res);
         throw new HttpError(`Not found: ${pathname}`, 404);
       } else {
@@ -235,25 +244,25 @@ export class PreviewServer {
   get server(): PreviewServer["_server"] {
     return this._server;
   }
-}
-
-async function getFile(path: string, root: string, loaders: LoaderResolver): Promise<string | undefined> {
-  const filepath = join(root, path);
-  try {
-    await access(filepath, constants.R_OK);
-    return path;
-  } catch (error) {
-    if (!isEnoent(error)) throw error;
-  }
 
-  // Look for a data loader for this file.
-  const loader = loaders.find(path);
-  if (loader) {
+  async getFile(path: string, root: string, loaders: LoaderResolver): Promise<string | undefined> {
+    const filepath = join(root, path);
     try {
-      return await loader.load();
+      await access(filepath, constants.R_OK);
+      return path;
     } catch (error) {
       if (!isEnoent(error)) throw error;
     }
+
+    // Look for a data loader for this file.
+    const loader = loaders.find(path);
+    if (loader) {
+      try {
+        return await loader.load(this._effects);
+      } catch (error) {
+        if (!isEnoent(error)) throw error;
+      }
+    }
   }
 }
 

test/dataloaders-test.ts

@@ -6,7 +6,8 @@ import {LoaderResolver} from "../src/dataloader.js";
 
 const noopEffects: LoadEffects = {
   logger: {log() {}, warn() {}, error() {}},
-  output: {write() {}}
+  output: {write() {}},
+  address: "not implemented"
 };
 
 describe("LoaderResolver.find(path)", () => {
@@ -55,7 +56,8 @@ describe("LoaderResolver.find(path, {useStale: true})", () => {
         write(a) {
           out.push(a);
         }
-      }
+      },
+      address: "not implemented"
     };
     const loader = loaders.find("dataloaders/data1.txt")!;
     // save the loader times.