Exporters render through programs; Typst exports live

modules/exporters/latex/default.nix

@@ -24,13 +24,15 @@ let
 
   exportServer = callPackage ../lsp-export-server/lsp-export-server.nix {};
   exportServerVersion = import ../lsp-export-server/lsp-export-server-version.nix;
+  latexRender = callPackage ./latex-render.nix {};
 
-  # The export server with the TeX Live engines and codedown.sty in reach. `serve` is the language
-  # server behind live exports; `render` compiles a file once (the exporter test uses it).
+  # The export server with the TeX Live engines and codedown.sty in reach, rendering through
+  # latex-render. `serve` is the language server behind live exports; `render` compiles a file once
+  # (the exporter test uses it).
   wrapper = common.writeShellScriptBinWithAttrs {} "latex-export-server" ''
     export PATH="${texliveToUse}/bin''${PATH:+:''${PATH}}"
     export TEXINPUTS="${texInputs}''${TEXINPUTS:-}"
-    exec ${exportServer}/bin/lsp-export-server "$@"
+    exec ${exportServer}/bin/lsp-export-server "$@" --render ${latexRender}/bin/latex-render --settle ${latexRender}/bin/latex-render
   '';
 
   languageServers = [
@@ -59,7 +61,7 @@ let
     icon_monochrome = iconMonochrome;
     # Live exports run on the latex-export-server language server; these args are the one-shot
     # form of the same render, and carry the task's options (-o key=value)
-    args = ["${wrapper}/bin/latex-export-server" "render" "--backend" "latex" "-o" "engine=${settings.engine}"];
+    args = ["${wrapper}/bin/latex-export-server" "render" "-o" "engine=${settings.engine}"];
     input_extensions = ["tex"];
     kind = "render_server";
     render_server = "latex-export-server";
@@ -74,6 +76,7 @@ symlinkJoin {
   passthru = {
     meta = {
       name = "codedown-exporter-latex";
+      displayName = "LaTeX Exporter (TeX Live ${texliveYear})";
       description = "CodeDown exporter rendering LaTeX notebooks to PDF as you type.";
 
       # To separate these out in search results

modules/exporters/latex/language_server_export/default.nix

@@ -33,7 +33,6 @@ common.writeTextDirWithMetaAndPassthru { description = "Live LaTeX rendering for
   args = [
     "${wrapper}/bin/latex-export-server"
     "serve"
-    "--backend" "latex"
     "-o" "engine=${engine}"
   ];
 }])

modules/exporters/latex/latex-render.nix

@@ -0,0 +1,48 @@
+{ callPackage
+, gawk
+, gnugrep
+}:
+
+let
+  common = callPackage ../../kernels/common.nix {};
+
+in
+
+# The render program behind the latex-export-server language server (see the RENDER_* variables in
+# codedownio/lsp-export-server): one engine pass for the quick phase, latexmk for the settle phase.
+# The unsaved buffers come first on TEXINPUTS so they shadow their disk copies, the epoch pins the
+# PDF's timestamps, and the recorder's INPUT lines become the dependency list. LATEX_RENDER_INPUT
+# lets another program (pandoc-render) hand over a .tex it generated.
+common.writeShellScriptBinWithAttrs {} "latex-render" ''
+  set -euo pipefail
+
+  tex="''${LATEX_RENDER_INPUT:-$RENDER_ENTRY}"
+  engine="''${RENDER_OPT_engine:-pdflatex}"
+  export TEXINPUTS="$RENDER_OVERLAY//:''${TEXINPUTS:-}"
+  export LC_ALL=C SOURCE_DATE_EPOCH="$RENDER_EPOCH" FORCE_SOURCE_DATE=1
+
+  common=(-interaction=nonstopmode -halt-on-error -file-line-error -recorder "-jobname=$RENDER_JOB")
+  case "$RENDER_PHASE" in
+    settle)
+      case "$engine" in
+        xelatex) flag=-pdfxe ;;
+        lualatex) flag=-pdflua ;;
+        *) flag=-pdf ;;
+      esac
+      cmd=(latexmk "$flag" "-outdir=$RENDER_OUT" "''${common[@]}" "$tex")
+      ;;
+    *)
+      cmd=("$engine" "-output-directory=$RENDER_OUT" "''${common[@]}" "$tex")
+      ;;
+  esac
+
+  echo "''${cmd[*]}"
+  if ! "''${cmd[@]}"; then
+    # The engine's own account of what went wrong: the ! lines and the file:line: errors
+    log="$RENDER_OUT/$RENDER_JOB.log"
+    [ -f "$log" ] && ${gnugrep}/bin/grep -E -B1 '^!|:error:|\.tex:[0-9]+:' "$log" >&2 || true
+    exit 1
+  fi
+
+  ${gawk}/bin/awk '/^INPUT /{print $2}' "$RENDER_OUT/$RENDER_JOB.fls" > "$RENDER_DEPS"
+''

modules/exporters/lsp-export-server/lsp-export-server-version.nix

@@ -1 +1 @@
-"0.4.1.2"
+"0.1.0.0"

modules/exporters/lsp-export-server/lsp-export-server.nix

@@ -7,28 +7,28 @@
 {
   # HASHES_START
   "aarch64-linux" = fetchzip {
-    name = "lsp-export-server-0.4.1.2";
+    name = "lsp-export-server-0.1.0.0";
     stripRoot = false;
-    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.4.1.2/lsp-export-server-0.4.1.2-aarch64-linux.tar.gz";
-    hash = "sha256-Xl1gjqwwMmL66sKSuUFyoH8B3bTZgerdr7Dr7KKTNF4=";
+    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.1.0.0/lsp-export-server-0.1.0.0-aarch64-linux.tar.gz";
+    hash = "sha256-7LQTzqAYOHyr8IeohByxgV9BvhT8eNqO/OjpVlLCeQo=";
   };
   "x86_64-linux" = fetchzip {
-    name = "lsp-export-server-0.4.1.2";
+    name = "lsp-export-server-0.1.0.0";
     stripRoot = false;
-    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.4.1.2/lsp-export-server-0.4.1.2-x86_64-linux.tar.gz";
-    hash = "sha256-USAxpR5pSmrR3Kw7Huncv4nHFNTuVjzWxIr+Kfw/CTA=";
+    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.1.0.0/lsp-export-server-0.1.0.0-x86_64-linux.tar.gz";
+    hash = "sha256-Hh5sWvw3+UO6tbH5GGxhKAW+IGerRX12SBdrs30uSyY=";
   };
   "x86_64-darwin" = fetchzip {
-    name = "lsp-export-server-0.4.1.2";
+    name = "lsp-export-server-0.1.0.0";
     stripRoot = false;
-    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.4.1.2/lsp-export-server-0.4.1.2-x86_64-darwin.tar.gz";
-    hash = "sha256-h7eLA3ibe1LGQwV8FliiykkeulG2iZjCyoVjtHV+07U=";
+    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.1.0.0/lsp-export-server-0.1.0.0-x86_64-darwin.tar.gz";
+    hash = "sha256-1btJfUmtHIaqD4bd+NPo4LSZ//cYjtPhU2aQi3hrUuM=";
   };
   "aarch64-darwin" = fetchzip {
-    name = "lsp-export-server-0.4.1.2";
+    name = "lsp-export-server-0.1.0.0";
     stripRoot = false;
-    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.4.1.2/lsp-export-server-0.4.1.2-aarch64-darwin.tar.gz";
-    hash = "sha256-P160ZrABtp+DD4uR+GhFsKnhjZoRe2BZiXwTdQSqAxo=";
+    url = "https://github.com/codedownio/lsp-export-server/releases/download/v0.1.0.0/lsp-export-server-0.1.0.0-aarch64-darwin.tar.gz";
+    hash = "sha256-lBfC748fmgH5TfeIC1q/luRpgCGCAqd5UgR+2BM5m/A=";
   };
   # HASHES_END
 }.${stdenv.hostPlatform.system}

modules/exporters/nbconvert/default.nix

@@ -55,6 +55,7 @@ symlinkJoin {
   passthru = {
     meta = {
       name = "nbconvert-exporters";
+      displayName = "nbconvert Exporter ${nbconvert.version}";
       description = "CodeDown exporters for PDF, HTML, LaTeX, slides, etc.";
 
       # To separate these out in search results

modules/exporters/pandoc/default.nix

@@ -20,12 +20,15 @@ let
 
   exportServer = callPackage ../lsp-export-server/lsp-export-server.nix {};
   exportServerVersion = import ../lsp-export-server/lsp-export-server-version.nix;
+  latexRender = callPackage ../latex/latex-render.nix {};
+  pandocRender = callPackage ./pandoc-render.nix { inherit latexRender; };
 
-  # The export server with pandoc and TeX Live in reach. `serve` is the language server behind
-  # live exports; `render` converts a file once (the exporter test and notebook exports use it).
+  # The export server with pandoc and TeX Live in reach, rendering through pandoc-render. `serve`
+  # is the language server behind live exports; `render` converts a file once (the exporter test
+  # and notebook exports use it).
   wrapper = common.writeShellScriptBinWithAttrs {} "pandoc-export-server" ''
     export PATH="${pandoc}/bin:${texliveScheme}/bin''${PATH:+:''${PATH}}"
-    exec ${exportServer}/bin/lsp-export-server "$@"
+    exec ${exportServer}/bin/lsp-export-server "$@" --render ${pandocRender}/bin/pandoc-render --settle ${pandocRender}/bin/pandoc-render
   '';
 
   languageServers = [
@@ -60,7 +63,7 @@ let
     inherit icon;
     icon_monochrome = iconMonochrome;
     args = [
-      "${wrapper}/bin/pandoc-export-server" "render" "--backend" "pandoc"
+      "${wrapper}/bin/pandoc-export-server" "render"
       "-o" "to=${to}"
       "-o" "engine=pdflatex"
     ] ++ (if pandocArgs == [] then [] else ["-o" "pandoc_args=${builtins.concatStringsSep " " pandocArgs}"]);
@@ -112,6 +115,7 @@ symlinkJoin {
   passthru = {
     meta = {
       name = "codedown-exporter-pandoc";
+      displayName = "Pandoc Exporter ${pandoc.version}";
       description = "CodeDown exporters for PDF, HTML, LaTeX, slides, etc.";
 
       # To separate these out in search results

modules/exporters/pandoc/language_server_export/default.nix

@@ -31,7 +31,6 @@ common.writeTextDirWithMetaAndPassthru { description = "Live pandoc rendering fo
   args = [
     "${wrapper}/bin/pandoc-export-server"
     "serve"
-    "--backend" "pandoc"
     "-o" "engine=pdflatex"
   ];
 }])

modules/exporters/pandoc/pandoc-render.nix

@@ -0,0 +1,48 @@
+{ callPackage
+, pandoc
+
+, latexRender
+}:
+
+let
+  common = callPackage ../../kernels/common.nix {};
+
+in
+
+# The render program behind the pandoc-export-server language server (see the RENDER_* variables in
+# codedownio/lsp-export-server). Options: to (the writer; pdf means latex), from (the reader for
+# Markdown input), pandoc_args (extra arguments, space-separated), and engine for the LaTeX-based
+# writers, which hand the generated .tex to latex-render for both phases. The other writers have
+# nothing to settle.
+common.writeShellScriptBinWithAttrs {} "pandoc-render" ''
+  set -euo pipefail
+
+  to="''${RENDER_OPT_to:-html}"
+  writer="$to"
+  [ "$to" = pdf ] && writer=latex
+  case "$writer" in
+    latex|beamer) via_latex=1; out="$RENDER_SRC/$RENDER_JOB.tex" ;;
+    *) via_latex=0; out="$RENDER_OUTPUT" ;;
+  esac
+  if [ "$RENDER_PHASE" = settle ] && [ "$via_latex" = 0 ] && [ -f "$RENDER_OUTPUT" ]; then
+    exit 0
+  fi
+
+  reader=()
+  case "$RENDER_ENTRY" in
+    *.md) reader=(-f "''${RENDER_OPT_from:-markdown+tex_math_dollars+tex_math_single_backslash+raw_html+smart}") ;;
+  esac
+  extra=()
+  if [ -n "''${RENDER_OPT_pandoc_args:-}" ]; then
+    read -r -a extra <<< "$RENDER_OPT_pandoc_args"
+  fi
+
+  export LC_ALL=C
+  cmd=(${pandoc}/bin/pandoc "''${reader[@]}" -t "$writer" --standalone "''${extra[@]}" "$RENDER_ENTRY" -o "$out")
+  echo "''${cmd[*]}"
+  "''${cmd[@]}"
+
+  if [ "$via_latex" = 1 ]; then
+    LATEX_RENDER_INPUT="$out" exec ${latexRender}/bin/latex-render
+  fi
+''

modules/exporters/typst/default.nix

@@ -29,18 +29,24 @@ let
     cp ${./codedown-lib/lib.typ} "$dir/lib.typ"
   '';
 
-  # Also writes a dependency file at "<output>.deps" (typst's native JSON
-  # {"inputs": [...]}, paths relative to cwd) so the runner can watch imported
-  # files and re-render when any of them changes. See exporterInfoDeps below.
-  script = common.writeShellScriptBinWithAttrs {} "typst-export" ''
-    echo_and_run() { echo "$*" ; "$@" ; }
-    echo_and_run export PATH="''${PATH:+''${PATH}:}"
-    echo_and_run ${typst}/bin/typst compile --package-path ${codedownPackagePath} --deps "$2.deps" "$1" "$2"
-  '';
-
   typstToUse = typst.withPackages (ps: (map (x: ps.${x}) packages));
 
-  languageServers = lib.optionals settings.lsp.tinymist.enable
+  exportServer = callPackage ../lsp-export-server/lsp-export-server.nix {};
+  exportServerVersion = import ../lsp-export-server/lsp-export-server-version.nix;
+  typstRender = callPackage ./typst-render.nix { inherit typstToUse codedownPackagePath; };
+
+  # The export server rendering through typst-render. `serve` is the language server behind live
+  # exports; `render` compiles a file once (the exporter test uses it). Typst has no settle pass.
+  wrapper = common.writeShellScriptBinWithAttrs {} "typst-export-server" ''
+    exec ${exportServer}/bin/lsp-export-server "$@" --render ${typstRender}/bin/typst-render
+  '';
+
+  languageServers = [
+    (callPackage ./language_server_export {
+      inherit wrapper;
+      version = exportServerVersion;
+    })
+  ] ++ lib.optionals settings.lsp.tinymist.enable
     [(callPackage ./language_server_tinymist {
       inherit kernelName typstToUse codedownPackagePath;
       settings = settings.lsp.tinymist;
@@ -52,17 +58,20 @@ let
   icon = ./typst.png;
   iconMonochrome = ./typst.svg;
 
+  # Live exports run on the typst-export-server language server; the args are the one-shot form of
+  # the same render
   mkTypstExporter = display_name: extension: {
     name = "codedown-exporter-typst";
     inherit display_name;
     group = "Typst";
     inherit extension;
     inherit icon;
     icon_monochrome = iconMonochrome;
-    args = [(script + "/bin/typst-export")];
+    args = ["${wrapper}/bin/typst-export-server" "render"];
     input_extensions = ["typ"];
     pandoc = "${pandoc}/bin/pandoc";
-    deps = "typst_json";
+    kind = "render_server";
+    render_server = "typst-export-server";
   };
 
   # Live preview exporter. Instead of running a command, the runner attaches a tinymist
@@ -87,14 +96,15 @@ symlinkJoin {
   name = "codedown-exporter-typst";
   paths = [
     (callPackage ./kernel.nix { inherit attrs extensions typstToUse; })
-    script
+    wrapper
   ]
   ++ languageServers
   ;
 
   passthru = {
     meta = {
       name = "codedown-exporter-typst";
+      displayName = "Typst Exporter ${typst.version}";
       description = "CodeDown exporter using Typst.";
 
       # To separate these out in search results
@@ -118,6 +128,7 @@ symlinkJoin {
 
     versions = {
       typst = typst.version;
+      lsp-export-server = exportServerVersion;
     };
 
     inherit packageOptions packageSearch;

modules/exporters/typst/language_server_export/default.nix

@@ -0,0 +1,36 @@
+{ callPackage
+, lib
+
+, wrapper
+, version
+}:
+
+let
+  common = callPackage ../../../kernels/common.nix {};
+
+  languageServerName = "typst-export-server";
+
+  passthru = {
+    inherit languageServerName;
+  };
+
+in
+
+# The export server compiles the in-memory document for live exports (see the codedown.render
+# protocol in codedownio/lsp-export-server). It only handles its own commands, so tinymist keeps the
+# editing features.
+common.writeTextDirWithMetaAndPassthru { description = "Live Typst rendering for codedown exports"; } passthru "lib/codedown/language-servers/typst-export-server.yaml" (lib.generators.toYAML {} [{
+  name = languageServerName;
+  inherit version;
+  icon = ../typst.png;
+  icon_monochrome = ../typst.svg;
+  extensions = ["typ"];
+  notebook_suffix = ".typ";
+  attrs = ["typst"];
+  type = "stream";
+  primary = false;
+  args = [
+    "${wrapper}/bin/typst-export-server"
+    "serve"
+  ];
+}])

modules/exporters/typst/typst-render.nix

@@ -0,0 +1,35 @@
+{ callPackage
+, coreutils
+, gnused
+
+, typstToUse
+, codedownPackagePath
+}:
+
+let
+  common = callPackage ../../kernels/common.nix {};
+
+in
+
+# The render program behind the typst-export-server language server (see the RENDER_* variables in
+# codedownio/lsp-export-server). The buffer goes in on stdin, so imports resolve from the document's
+# directory (the working directory) rather than the work directory's copy; the epoch pins the PDF's
+# timestamps. Image formats render the first page only, since a single output file can't hold more.
+common.writeShellScriptBinWithAttrs {} "typst-render" ''
+  set -euo pipefail
+
+  export SOURCE_DATE_EPOCH="$RENDER_EPOCH"
+  extra=()
+  case "$RENDER_FORMAT" in
+    png|svg) extra=(--pages 1) ;;
+    html) extra=(--features html) ;;
+  esac
+  deps="$RENDER_WORK/deps.mk"
+
+  cmd=(${typstToUse}/bin/typst compile --package-path ${codedownPackagePath} --format "$RENDER_FORMAT" "''${extra[@]}" --deps "$deps" --deps-format make - "$RENDER_OUTPUT")
+  echo "''${cmd[*]} < $RENDER_ENTRY"
+  "''${cmd[@]}" < "$RENDER_ENTRY"
+
+  # The make-format line is "output: dep dep <stdin>"
+  ${gnused}/bin/sed -e 's/^[^:]*: //' -e 's/ *<stdin>//' "$deps" | ${coreutils}/bin/tr ' ' '\n' > "$RENDER_DEPS"
+''