astrolabe/src/core/lesson-parse.ts

/**
 * Parser for markdown lesson files (the deep-dive learning section). A lesson is
 * authored as a `.md` — prose is native markdown, specs are fenced JSON — and this
 * turns one into the ordered **blocks** the page renders in document order.
 *
 * A lesson is a free-form document, not a fixed template: prose, interactive
 * progressions, standalone charts, and callouts in any arrangement. The author
 * only ever writes markdown, fenced specs, and `:::directives` — never JSX.
 *
 * Pure and portable (string in → data out; no DOM, Vega, or React): the
 * structural parse and the spec `JSON.parse` live here and are tested hardest.
 * Rendering the prose markdown to HTML is the UI layer's job (`src/learn`), kept
 * out of core so core stays dependency-free.
 *
 * Format:
 *
 *     ---
 *     slug: binning
 *     title: From bars to a real histogram
 *     tagline: One word turns a noisy bar chart into a distribution.
 *     ---
 *
 *     Free intro prose (markdown)…
 *
 *     :::data
 *     { "ev": [ {…row…}, {…row…} ] }
 *     :::
 *
 *     :::progression
 *     ## raw counts
 *     Note prose for this stage (markdown)…
 *     ```vega-lite
 *     { "data": { "name": "ev" }, …spec… }
 *     ```
 *     ## + bin
 *     …
 *     :::
 *
 *     More prose, then a standalone chart:
 *
 *     ```vega-lite
 *     { …one-off chart… }
 *     ```
 *
 *     :::sharp-edge
 *     A callout (markdown)…
 *     :::
 *
 * A `:::data` block names datasets once for the whole lesson (body: a JSON object
 * of `name → rows`); specs reference them by `{ "name": … }` and the rows are
 * injected at render time, so they aren't repeated per stage. A `:::progression`
 * block holds `##`-delimited stages (heading → tab label, prose → note, fenced
 * `vega-lite` → spec). A bare fenced `vega-lite` block is a standalone chart. Any
 * other `:::name` block is a callout. Everything else is prose, emitted in the
 * order it appears.
 */

export interface ProgressionStage {
  /** Tab label — the `##` heading text. */
  label: string;
  /** Markdown prose shown under the chart for this stage. */
  note: string;
  /** The stage's Vega-Lite spec, parsed from its fenced block. */
  spec: Record<string, unknown>;
}

export interface ProseBlock {
  type: 'prose';
  markdown: string;
}
export interface ChartBlock {
  type: 'chart';
  spec: Record<string, unknown>;
}
export interface ProgressionBlock {
  type: 'progression';
  stages: ProgressionStage[];
}
export interface CalloutBlock {
  type: 'callout';
  /** The directive name, e.g. `sharp-edge` — the UI styles by variant. */
  variant: string;
  markdown: string;
}
export type LessonBlock = ProseBlock | ChartBlock | ProgressionBlock | CalloutBlock;

export interface ParsedLesson {
  slug: string;
  title: string;
  tagline: string;
  /**
   * Lesson-level named datasets, collected from `:::data` blocks. Specs reference
   * a dataset by `{ "data": { "name": … } }`; {@link injectDatasets} merges these
   * in at render time so the rows aren't repeated in every fenced spec.
   */
  datasets: Record<string, unknown[]>;
  /** The lesson body, in document order. */
  blocks: LessonBlock[];
}

const FRONTMATTER = /^---\n([\s\S]*?)\n---\n?/;
// A top-level region: a `:::name … :::` directive (which may itself contain
// vega-lite fences) OR a standalone vega-lite fence. The directive alternative is
// tried first and its lazy body runs to the next `\n:::`, so a progression's inner
// fences are consumed by its span rather than matched as standalone charts.
const REGION = /:::([a-z-]+)\n([\s\S]*?)\n:::|```vega-lite\n([\s\S]*?)\n```/g;
const STAGE_FENCE = /```vega-lite\n([\s\S]*?)\n```/;

export function parseLesson(source: string): ParsedLesson {
  const text = source.replace(/\r\n/g, '\n');

  const fm = FRONTMATTER.exec(text);
  if (!fm) throw new Error('lesson: missing `---` frontmatter block');
  const meta = parseFrontmatter(fm[1]);
  const slug = required(meta, 'slug');
  // The slug is the lesson's URL segment (`/learn/<slug>/`) and the route key; keep it
  // to the charset the router and the page-shell generator both assume, so a typo fails
  // loudly here rather than silently serving the wrong page.
  if (!/^[a-z0-9-]+$/.test(slug)) {
    throw new Error(`lesson "${slug}": slug must be lowercase letters, digits, and hyphens`);
  }
  const title = required(meta, 'title');
  const tagline = required(meta, 'tagline');

  const { blocks, datasets } = parseBlocks(text.slice(fm[0].length), slug);
  return { slug, title, tagline, datasets, blocks };
}

function parseBlocks(
  body: string,
  slug: string,
): { blocks: LessonBlock[]; datasets: Record<string, unknown[]> } {
  const blocks: LessonBlock[] = [];
  const datasets: Record<string, unknown[]> = {};
  const pushProse = (text: string): void => {
    const trimmed = text.trim();
    if (trimmed) blocks.push({ type: 'prose', markdown: trimmed });
  };

  REGION.lastIndex = 0;
  let cursor = 0;
  let m: RegExpExecArray | null;
  while ((m = REGION.exec(body)) !== null) {
    pushProse(body.slice(cursor, m.index));
    if (m[1] !== undefined) {
      const variant = m[1];
      const inner = m[2];
      if (variant === 'progression') {
        blocks.push({ type: 'progression', stages: parseStages(inner, slug) });
      } else if (variant === 'data') {
        // Lesson metadata, not a rendered block: fold the named rows into the
        // lesson's dataset map for render-time injection.
        Object.assign(datasets, parseDatasets(inner, slug));
      } else {
        blocks.push({ type: 'callout', variant, markdown: inner.trim() });
      }
    } else {
      blocks.push({ type: 'chart', spec: parseSpec(m[3], slug, 'chart') });
    }
    cursor = m.index + m[0].length;
  }
  pushProse(body.slice(cursor));

  if (blocks.length === 0) throw new Error(`lesson "${slug}": no content`);
  return { blocks, datasets };
}

/** Parse a `:::data` body — a JSON object of `name → rows[]` — into named datasets. */
function parseDatasets(inner: string, slug: string): Record<string, unknown[]> {
  let parsed: unknown;
  try {
    parsed = JSON.parse(inner.trim());
  } catch (e) {
    throw new Error(`lesson "${slug}", :::data: invalid JSON — ${(e as Error).message}`, {
      cause: e,
    });
  }
  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
    throw new Error(`lesson "${slug}", :::data: body must be a JSON object of { name: rows }`);
  }
  const out: Record<string, unknown[]> = {};
  for (const [name, rows] of Object.entries(parsed)) {
    if (!Array.isArray(rows)) {
      throw new Error(`lesson "${slug}", :::data "${name}": value must be an array of rows`);
    }
    out[name] = rows;
  }
  return out;
}

/**
 * Merge lesson-level named datasets into a spec for *rendering* — the authored
 * spec (shown in the source pane and diffed) keeps its `{ "data": { "name": … } }`
 * reference and never carries the rows. Returns a clone with the spec's own
 * `datasets` winning on a name clash; mirrors the clone-before-embed rule the app's
 * other render-time spec tweaks follow (never mutate the authored spec). A no-op
 * (same reference) when there are no datasets to inject.
 */
export function injectDatasets(
  spec: Record<string, unknown>,
  datasets: Record<string, unknown[]>,
): Record<string, unknown> {
  if (Object.keys(datasets).length === 0) return spec;
  const own = (spec.datasets as Record<string, unknown> | undefined) ?? {};
  return { ...spec, datasets: { ...datasets, ...own } };
}

function parseStages(inner: string, slug: string): ProgressionStage[] {
  const parts = inner.split(/^## +/m);
  parts.shift(); // text before the first `##` (blank by convention)
  if (parts.length === 0) {
    throw new Error(`lesson "${slug}": a :::progression has no \`##\` stages`);
  }
  return parts.map((part) => parseStage(part, slug));
}

function parseStage(part: string, slug: string): ProgressionStage {
  const nl = part.indexOf('\n');
  const label = (nl === -1 ? part : part.slice(0, nl)).trim();
  const rest = nl === -1 ? '' : part.slice(nl + 1);

  const block = STAGE_FENCE.exec(rest);
  if (!block) throw new Error(`lesson "${slug}", stage "${label}": no \`\`\`vega-lite block`);

  const spec = parseSpec(block[1], slug, `stage "${label}"`);
  const note = (rest.slice(0, block.index) + rest.slice(block.index + block[0].length)).trim();
  return { label, note, spec };
}

function parseSpec(json: string, slug: string, where: string): Record<string, unknown> {
  let spec: unknown;
  try {
    spec = JSON.parse(json);
  } catch (e) {
    throw new Error(`lesson "${slug}", ${where}: invalid JSON — ${(e as Error).message}`, {
      cause: e,
    });
  }
  if (typeof spec !== 'object' || spec === null || Array.isArray(spec)) {
    throw new Error(`lesson "${slug}", ${where}: spec must be a JSON object`);
  }
  return spec as Record<string, unknown>;
}

function parseFrontmatter(raw: string): Map<string, string> {
  const map = new Map<string, string>();
  for (const line of raw.split('\n')) {
    if (!line.trim()) continue;
    const i = line.indexOf(':');
    if (i === -1) throw new Error(`lesson frontmatter: expected \`key: value\`, got "${line}"`);
    map.set(line.slice(0, i).trim(), line.slice(i + 1).trim());
  }
  return map;
}

function required(meta: Map<string, string>, key: string): string {
  const v = meta.get(key);
  if (!v) throw new Error(`lesson frontmatter: missing "${key}"`);
  return v;
}