From f405cd27add05ee97aa6bc025ce874f35b5a4c3d Mon Sep 17 00:00:00 2001
From: Mike Bostock <mbostock@gmail.com>
Date: Sat, 9 Nov 2024 08:59:00 -0800
Subject: [PATCH 1/5] async parseMarkdown

---
 src/loader.ts          |  2 +-
 src/markdown.ts        |  2 +-
 test/markdown-test.ts  |  2 +-
 test/resolvers-test.ts | 30 +++++++++++++++---------------
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/loader.ts b/src/loader.ts
index 324f1a8e1..2a9c1cef0 100644
--- a/src/loader.ts
+++ b/src/loader.ts
@@ -81,7 +81,7 @@ export class LoaderResolver {
     const loader = this.findPage(path);
     if (!loader) throw enoent(path);
     const input = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
-    return parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
+    return await parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
   }
 
   /**
diff --git a/src/markdown.ts b/src/markdown.ts
index cb43b5726..85c25b1e7 100644
--- a/src/markdown.ts
+++ b/src/markdown.ts
@@ -243,7 +243,7 @@ export function createMarkdownIt({
   return markdownIt === undefined ? md : markdownIt(md);
 }
 
-export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
+export async function parseMarkdown(input: string, options: ParseOptions): Promise<MarkdownPage> {
   const {md, path, source = path, params} = options;
   const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
diff --git a/test/markdown-test.ts b/test/markdown-test.ts
index eeecb222a..5e37b9222 100644
--- a/test/markdown-test.ts
+++ b/test/markdown-test.ts
@@ -23,7 +23,7 @@ describe("parseMarkdown(input)", () => {
 
     (only ? it.only : skip ? it.skip : it)(`test/input/${name}`, async () => {
       const source = await readFile(path, "utf8");
-      const snapshot = parseMarkdown(source, {path: name, md});
+      const snapshot = await parseMarkdown(source, {path: name, md});
       let allequal = true;
       for (const ext of ["html", "json"]) {
         const actual = ext === "json" ? jsonMeta(snapshot) : snapshot.body;
diff --git a/test/resolvers-test.ts b/test/resolvers-test.ts
index 090d34d21..a60c807d7 100644
--- a/test/resolvers-test.ts
+++ b/test/resolvers-test.ts
@@ -14,74 +14,74 @@ describe("getResolvers(page, {root, path})", () => {
   const builtins = ["observablehq:runtime", "observablehq:stdlib", "observablehq:client"];
   it("resolves directly-attached files", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${FileAttachment('foo.csv')}", options);
+    const page = await parseMarkdown("${FileAttachment('foo.csv')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.files, new Set(["./foo.csv"]));
   });
   it("ignores files that are outside of the source root", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${FileAttachment('../foo.csv')}", options);
+    const page = await parseMarkdown("${FileAttachment('../foo.csv')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.files, new Set([]));
   });
   it("detects file methods", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${FileAttachment('foo.csv').csv}", options);
+    const page = await parseMarkdown("${FileAttachment('foo.csv').csv}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["npm:d3-dsv", ...builtins]));
   });
   it("detects local static imports", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './bar.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './bar.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./bar.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./bar.js"]));
   });
   it("detects local transitive static imports", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './other/foo.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './other/foo.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./other/foo.js", "./bar.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./other/foo.js", "./bar.js"]));
   });
   it("detects local transitive static imports (2)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js", ...builtins])); // prettier-ignore
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local transitive dynamic imports", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./dynamic-import.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
   });
   it("detects local transitive dynamic imports (2)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
+    const page = await parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
   });
   it("detects local transitive dynamic imports (3)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
+    const page = await parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-dynamic-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local transitive dynamic imports (4)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
+    const page = await parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local dynamic imports", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${import('./foo.js')}", options);
+    const page = await parseMarkdown("${import('./foo.js')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./foo.js"]));
@@ -89,9 +89,9 @@ describe("getResolvers(page, {root, path})", () => {
 });
 
 describe("resolveLink(href) with {preserveExtension: true}", () => {
-  const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: true});
-  const page = parseMarkdown("", options);
   async function getResolveLink() {
+    const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: true});
+    const page = await parseMarkdown("", options);
     const resolvers = await getResolvers(page, options);
     return resolvers.resolveLink;
   }
@@ -164,9 +164,9 @@ describe("resolveLink(href) with {preserveExtension: true}", () => {
 });
 
 describe("resolveLink(href) with {preserveExtension: false}", () => {
-  const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: false});
-  const page = parseMarkdown("", options);
   async function getResolveLink() {
+    const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: false});
+    const page = await parseMarkdown("", options);
     const resolvers = await getResolvers(page, options);
     return resolvers.resolveLink;
   }

From 5fcb96d7bf5e8f165bbdc7585bb2afdad1456224 Mon Sep 17 00:00:00 2001
From: Mike Bostock <mbostock@gmail.com>
Date: Sat, 9 Nov 2024 09:35:24 -0800
Subject: [PATCH 2/5] page fragment loaders

---
 docs/index.md   |  8 +++++
 src/loader.ts   |  2 +-
 src/markdown.ts | 84 ++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 86 insertions(+), 8 deletions(-)

diff --git a/docs/index.md b/docs/index.md
index 83cc6a13f..a59dff688 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -2,6 +2,14 @@
 index: false
 ---
 
+```js server
+console.log(`The current version of Node is: <b>${process.env.npm_package_version}.`);
+```
+
+```sh server
+uname -a
+```
+
 <style>
 
 .hero {
diff --git a/src/loader.ts b/src/loader.ts
index 2a9c1cef0..e351e2220 100644
--- a/src/loader.ts
+++ b/src/loader.ts
@@ -62,7 +62,7 @@ export interface LoaderOptions {
 
 export class LoaderResolver {
   private readonly root: string;
-  private readonly interpreters: Map<string, string[]>;
+  public readonly interpreters: Map<string, string[]>; // TODO cleaner
 
   constructor({root, interpreters}: {root: string; interpreters?: Record<string, string[] | null>}) {
     this.root = root;
diff --git a/src/markdown.ts b/src/markdown.ts
index 85c25b1e7..a9d2e6d6e 100644
--- a/src/markdown.ts
+++ b/src/markdown.ts
@@ -1,6 +1,7 @@
 /* eslint-disable import/no-named-as-default-member */
 import {createHash} from "node:crypto";
 import slugify from "@sindresorhus/slugify";
+import {spawn} from "cross-spawn";
 import he from "he";
 import MarkdownIt from "markdown-it";
 import type {RuleCore} from "markdown-it/lib/parser_core.mjs";
@@ -12,7 +13,7 @@ import type {Config} from "./config.js";
 import {mergeStyle} from "./config.js";
 import type {FrontMatter} from "./frontMatter.js";
 import {readFrontMatter} from "./frontMatter.js";
-import {html, rewriteHtmlPaths} from "./html.js";
+import {html, parseHtml, rewriteHtmlPaths} from "./html.js";
 import {parseInfo} from "./info.js";
 import {transformJavaScriptSync} from "./javascript/module.js";
 import type {JavaScriptNode} from "./javascript/parse.js";
@@ -31,6 +32,12 @@ export interface MarkdownCode {
   mode: "inline" | "block" | "jsx";
 }
 
+export interface FragmentLoader {
+  id: string;
+  tag: string;
+  source: string;
+}
+
 export interface MarkdownPage {
   title: string | null;
   head: string | null;
@@ -46,6 +53,7 @@ export interface MarkdownPage {
 
 interface ParseContext {
   code: MarkdownCode[];
+  fragments: FragmentLoader[];
   startLine: number;
   currentLine: number;
   path: string;
@@ -119,14 +127,25 @@ function makeFenceRenderer(baseRenderer: RenderRule): RenderRule {
     let html = "";
     let source: string | undefined;
     try {
-      source = isFalse(attributes.run) ? undefined : getLiveSource(token.content, tag, attributes);
+      source =
+        attributes.server != null
+          ? token.content
+          : isFalse(attributes.run)
+          ? undefined
+          : getLiveSource(token.content, tag, attributes);
       if (source != null) {
+        let loading = false;
         const id = uniqueCodeId(context, source);
         // TODO const sourceLine = context.startLine + context.currentLine;
-        const node = parseJavaScript(source, {path, params});
-        context.code.push({id, node, mode: tag === "jsx" || tag === "tsx" ? "jsx" : "block"});
+        if (attributes.server != null) {
+          context.fragments.push({id, tag, source});
+        } else {
+          const node = parseJavaScript(source, {path, params});
+          context.code.push({id, node, mode: tag === "jsx" || tag === "tsx" ? "jsx" : "block"});
+          loading = node.expression;
+        }
         html += `<div class="observablehq observablehq--block">${
-          node.expression ? "<observablehq-loading></observablehq-loading>" : ""
+          loading ? "<observablehq-loading></observablehq-loading>" : ""
         }<!--:${id}:--></div>\n`;
       }
     } catch (error) {
@@ -214,6 +233,7 @@ export interface ParseOptions {
   path: string;
   style?: Config["style"];
   scripts?: Config["scripts"];
+  loaders?: Config["loaders"];
   head?: Config["head"];
   header?: Config["header"];
   footer?: Config["footer"];
@@ -247,10 +267,44 @@ export async function parseMarkdown(input: string, options: ParseOptions): Promi
   const {md, path, source = path, params} = options;
   const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
-  const context: ParseContext = {code, startLine: 0, currentLine: 0, path, params};
+  const fragments: FragmentLoader[] = [];
+  const context: ParseContext = {code, fragments, startLine: 0, currentLine: 0, path, params};
   const tokens = md.parse(content, context);
-  const body = md.renderer.render(tokens, md.options, context); // Note: mutates code!
   const title = data.title !== undefined ? data.title : findTitle(tokens);
+  let body = md.renderer.render(tokens, md.options, context); // Note: mutates code!
+
+  // Rewrite body to render fragments.
+  if (fragments.length) {
+    const {document} = parseHtml(body);
+    const roots = findRoots(document, document.body);
+    const interpreters = options.loaders!.interpreters;
+    for (const fragment of fragments) {
+      const root: Comment = roots.get(fragment.id);
+      const [command, ...args] = interpreters.get(`.${fragment.tag}`)!;
+      let target = "";
+      const subprocess = spawn(command, args, {
+        windowsHide: true,
+        stdio: ["pipe", "pipe", "inherit"]
+      });
+      subprocess.stdin.write(fragment.source);
+      subprocess.stdin.end();
+      subprocess.stdout.on("data", (data) => {
+        target += data.toString();
+      });
+      const code = await new Promise((resolve, reject) => {
+        subprocess.on("error", reject);
+        subprocess.on("close", resolve);
+      });
+      if (code !== 0) {
+        throw new Error(`loader exited with code ${code}`);
+      }
+      const template = document.createElement("template");
+      template.innerHTML = target;
+      root.replaceWith(template.content.cloneNode(true));
+    }
+    body = document.body.innerHTML;
+  }
+
   return {
     head: getHead(title, data, options),
     header: getHeader(title, data, options),
@@ -265,6 +319,22 @@ export async function parseMarkdown(input: string, options: ParseOptions): Promi
   };
 }
 
+function findRoots(document, root) {
+  const roots = new Map();
+  const iterator = document.createNodeIterator(root, 128, null);
+  let node;
+  while ((node = iterator.nextNode())) {
+    if (isRoot(node)) {
+      roots.set(node.data.slice(1, -1), node);
+    }
+  }
+  return roots;
+}
+
+function isRoot(node) {
+  return node.nodeType === 8 && /^:[0-9a-f]{8}(?:-\d+)?:$/.test(node.data);
+}
+
 /** Like parseMarkdown, but optimized to return only metadata. */
 export function parseMarkdownMetadata(input: string, options: ParseOptions): Pick<MarkdownPage, "data" | "title"> {
   const {md, path} = options;

From e7c7e3aab93cf4efd41a4e997c5317e732b871a2 Mon Sep 17 00:00:00 2001
From: Mike Bostock <mbostock@gmail.com>
Date: Sat, 9 Nov 2024 09:35:53 -0800
Subject: [PATCH 3/5] server echo

---
 docs/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/index.md b/docs/index.md
index a59dff688..702503c81 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -2,11 +2,11 @@
 index: false
 ---
 
-```js server
+```js server echo
 console.log(`The current version of Node is: <b>${process.env.npm_package_version}.`);
 ```
 
-```sh server
+```sh server echo
 uname -a
 ```
 

From b87e2457e79c4113813afda0144769acbfbc4e5d Mon Sep 17 00:00:00 2001
From: Mike Bostock <mbostock@gmail.com>
Date: Sat, 9 Nov 2024 09:36:27 -0800
Subject: [PATCH 4/5] date

---
 docs/index.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/index.md b/docs/index.md
index 702503c81..e83a1b75b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -10,6 +10,10 @@ console.log(`The current version of Node is: <b>${process.env.npm_package_versio
 uname -a
 ```
 
+```sh server echo
+date
+```
+
 <style>
 
 .hero {

From f5b5a1375acc17abaf29ac0985a184a72f0b3dd5 Mon Sep 17 00:00:00 2001
From: Mike Bostock <mbostock@gmail.com>
Date: Sun, 10 Nov 2024 09:07:37 -0800
Subject: [PATCH 5/5] params for page fragment loaders

---
 src/loader.ts   |  6 +++++-
 src/markdown.ts | 18 ++++++++++++++----
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/loader.ts b/src/loader.ts
index e351e2220..1d5740b7a 100644
--- a/src/loader.ts
+++ b/src/loader.ts
@@ -213,7 +213,7 @@ export class LoaderResolver {
       const eext = fext.slice(0, -iext.length); // .zip
       const loader = new CommandLoader({
         command: command ?? commandPath,
-        args: params ? args.concat(defineParams(params)) : args,
+        args: withParams(args, params),
         path,
         params,
         root: this.root,
@@ -332,6 +332,10 @@ export class LoaderResolver {
   }
 }
 
+export function withParams(args: string[], params?: Params): string[] {
+  return params ? args.concat(defineParams(params)) : args;
+}
+
 function defineParams(params: Params): string[] {
   return Object.entries(params)
     .filter(([name]) => /^[a-z0-9_]+$/i.test(name)) // ignore non-ASCII parameters
diff --git a/src/markdown.ts b/src/markdown.ts
index a9d2e6d6e..52834aa1a 100644
--- a/src/markdown.ts
+++ b/src/markdown.ts
@@ -18,6 +18,7 @@ import {parseInfo} from "./info.js";
 import {transformJavaScriptSync} from "./javascript/module.js";
 import type {JavaScriptNode} from "./javascript/parse.js";
 import {parseJavaScript} from "./javascript/parse.js";
+import {withParams} from "./loader.js";
 import {isAssetPath, relativePath} from "./path.js";
 import {parsePlaceholder} from "./placeholder.js";
 import type {Params} from "./route.js";
@@ -282,10 +283,19 @@ export async function parseMarkdown(input: string, options: ParseOptions): Promi
       const root: Comment = roots.get(fragment.id);
       const [command, ...args] = interpreters.get(`.${fragment.tag}`)!;
       let target = "";
-      const subprocess = spawn(command, args, {
-        windowsHide: true,
-        stdio: ["pipe", "pipe", "inherit"]
-      });
+      const subprocess = spawn(
+        command,
+        withParams(
+          command === "sh"
+            ? args.concat("-s", "--") // TODO make this configurable
+            : args.concat("-"), // TODO make this configurable
+          params
+        ),
+        {
+          windowsHide: true,
+          stdio: ["pipe", "pipe", "inherit"]
+        }
+      );
       subprocess.stdin.write(fragment.source);
       subprocess.stdin.end();
       subprocess.stdout.on("data", (data) => {