Snapshot URL datasets locally on add; preview tabular data as a table

src/core/dataset.ts

@@ -7,22 +7,29 @@
  * a simple delimited-text parser, the profiling orchestration, and a factory that
  * stamps timestamps/version and fills the derived summary fields.
  *
- * A dataset has one of two **sources** — `inline` (data stored in the record) or
- * `url` (only the link is stored, fetched on demand at render time) — and one of
- * four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
- * The `data` field's shape follows source/format: a URL string for `url`; raw
- * text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
+ * A dataset has one of two **sources** — `inline` (data pasted into the record) or
+ * `url` (fetched once from a remote address and **snapshotted** into the record) —
+ * and one of four **formats** (reused from format-detection: `json`/`csv`/`tsv`/
+ * `topojson`). Either way `data` holds the actual payload, shaped by format: raw
+ * text for CSV/TSV, a parsed value for JSON/TopoJSON. A `url` dataset additionally
+ * keeps its source `url` (so it can be re-fetched) and a `fetchedAt` timestamp;
+ * until its first successful fetch `data` is `null` (an unfetched reference).
  *
- * Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
- * non-tabular data get an N/A profile but are still sized (see profile.ts).
+ * Any tabular payload (JSON array-of-objects, CSV, TSV) is profiled — including a
+ * fetched URL snapshot; non-tabular or not-yet-fetched data gets an N/A profile but
+ * is still sized (see profile.ts).
  */
 
-import type { DataFormat } from './format-detection';
+import { detectFormat, detectFormatFromUrl, type DataFormat } from './format-detection';
 import { profileData, type ColumnStats, type DatasetProfile } from './profile';
 import type { ColumnType } from './type-inference';
 
-/** Current schema version for a Dataset record (read-time migration target). */
-export const CURRENT_DATASET_VERSION = 1;
+/**
+ * Current schema version for a Dataset record (read-time migration target).
+ * v2 moved a URL dataset's address out of `data` into its own `url` field and made
+ * `data` hold the fetched snapshot (see `migrateDataset`).
+ */
+export const CURRENT_DATASET_VERSION = 2;
 
 /** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
 export type DataSource = 'inline' | 'url';
@@ -35,14 +42,25 @@ export interface Dataset {
   /** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
   name: string;
   /**
-   * The payload. For `source = url`: the URL string. For `source = inline`: the
-   * raw CSV/TSV text, or the parsed JSON/TopoJSON value.
+   * The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON
+   * value. For `source = url` this is the fetched snapshot, or `null` before the
+   * first successful fetch.
    */
   data: unknown;
   /** One of `json`, `csv`, `tsv`, `topojson`. */
   format: DataFormat;
   /** One of `inline` or `url`. */
   source: DataSource;
+  /**
+   * For `source = url`: the remote address the snapshot was fetched from, retained
+   * so the dataset can be re-fetched ("Refresh"). Absent for inline datasets.
+   */
+  url?: string;
+  /**
+   * For `source = url`: ISO timestamp of the last successful fetch, or `null` when
+   * it has never been fetched. Absent for inline datasets.
+   */
+  fetchedAt?: string | null;
   /** Free-form user note about the dataset. */
   comment: string;
   /** Data rows, or `null` when N/A (URL / non-tabular). */
@@ -187,26 +205,52 @@ function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
 }
 
 /**
- * Orchestrate profiling for a dataset payload: compute `size` (always), decide
- * tabular vs N/A by source/format, and delegate to `profileData`.
- *
- *   - `url` (any format)        → N/A profile; size = byte length of the URL string.
- *   - inline `json`             → rows when a non-empty array of objects, else N/A.
- *   - inline `topojson`         → N/A (non-tabular).
- *   - inline `csv` / `tsv`      → `parseDelimited` rows.
- *
- * `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
- * `JSON.stringify(data)` for json/topojson.
+ * The tabular rows of a dataset payload for a table preview, or `null` when the
+ * payload isn't tabular (a single JSON object, TopoJSON, or an unfetched URL). Uses
+ * the **same** parsing as profiling — `parseDelimited` for CSV/TSV, `asObjectRows`
+ * for JSON — so the previewed rows agree exactly with the profiled `columns`. A
+ * positive `limit` returns only the head (a preview needs a sample, not the whole
+ * payload). Returns non-null on precisely the inputs `computeDatasetProfile` counts
+ * as rows, so a caller can gate "table vs. raw text" on this alone.
  */
-export function computeDatasetProfile(
+export function tabularRows(
   data: unknown,
   format: DataFormat,
-  source: DataSource,
-): DatasetProfile {
-  if (source === 'url') {
-    const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
-    return profileData(null, byteLength(url));
+  limit?: number,
+): Array<Record<string, unknown>> | null {
+  if (data == null) return null;
+  let rows: Array<Record<string, unknown>> | null;
+  switch (format) {
+    case 'csv':
+    case 'tsv':
+      rows = parseDelimited(typeof data === 'string' ? data : '', format);
+      break;
+    case 'json':
+      rows = asObjectRows(data);
+      break;
+    default:
+      rows = null;
   }
+  if (!rows || rows.length === 0) return null;
+  return limit != null && limit >= 0 && rows.length > limit ? rows.slice(0, limit) : rows;
+}
+
+/**
+ * Orchestrate profiling for a dataset payload: compute `size` (always), decide
+ * tabular vs N/A by format, and delegate to `profileData`. Identical for inline
+ * data and for a fetched URL snapshot — both carry the payload in `data`.
+ *
+ *   - `null` data (unfetched URL) → N/A profile, size 0.
+ *   - `json`                      → rows when a non-empty array of objects, else N/A.
+ *   - `topojson`                  → N/A (non-tabular).
+ *   - `csv` / `tsv`               → `parseDelimited` rows.
+ *
+ * `size` is the UTF-8 byte length of the raw string for csv/tsv, or of
+ * `JSON.stringify(data)` for json/topojson.
+ */
+export function computeDatasetProfile(data: unknown, format: DataFormat): DatasetProfile {
+  // An unfetched URL reference (or a genuinely absent payload): nothing to profile.
+  if (data == null) return profileData(null, 0);
 
   switch (format) {
     case 'csv':
@@ -226,6 +270,29 @@ export function computeDatasetProfile(
   }
 }
 
+/**
+ * Shape a freshly-fetched URL body into the `{ data, format }` a snapshot stores
+ * (spec §05 → URL datasets, snapshot model). Format is sniffed from the **content**
+ * first — authoritative, since `detectFormat` only reports `json` when the body
+ * actually parses — falling back to the URL's file extension, then JSON.
+ * JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text — the same
+ * per-format shaping inline data uses, so a fetched dataset profiles and renders
+ * identically to an inline one (see `computeDatasetProfile`, rendering.ts).
+ */
+export function snapshotFromText(text: string, url: string): { data: unknown; format: DataFormat } {
+  const format = detectFormat(text).format ?? detectFormatFromUrl(url) ?? 'json';
+  if (format === 'json' || format === 'topojson') {
+    try {
+      return { data: JSON.parse(text) as unknown, format };
+    } catch {
+      // The extension promised JSON but the body isn't — keep the raw text so the
+      // render surfaces a readable error instead of us throwing mid-commit.
+      return { data: text, format };
+    }
+  }
+  return { data: text, format };
+}
+
 export interface CreateDatasetOptions {
   /** The dataset name (uniqueness is enforced upstream — see naming.ts). */
   name: string;
@@ -235,6 +302,10 @@ export interface CreateDatasetOptions {
   format: DataFormat;
   /** One of `inline` or `url`. */
   source: DataSource;
+  /** For `source = url`: the remote address (retained for Refresh). */
+  url?: string;
+  /** For `source = url`: ISO timestamp of the fetch that produced `data`. */
+  fetchedAt?: string | null;
   /** Optional free-form note. */
   comment?: string;
   /** Clock injection for deterministic tests; defaults to the current time. */
@@ -253,7 +324,7 @@ export interface CreateDatasetOptions {
 export function createDataset(options: CreateDatasetOptions): Dataset {
   const now = options.now ?? new Date();
   const iso = now.toISOString();
-  const profile = computeDatasetProfile(options.data, options.format, options.source);
+  const profile = computeDatasetProfile(options.data, options.format);
 
   return {
     id: options.id ?? Date.now(),
@@ -262,6 +333,8 @@ export function createDataset(options: CreateDatasetOptions): Dataset {
     data: options.data,
     format: options.format,
     source: options.source,
+    // URL datasets carry their address + fetch time; inline records stay clean.
+    ...(options.source === 'url' ? { url: options.url, fetchedAt: options.fetchedAt ?? null } : {}),
     comment: options.comment ?? '',
     rowCount: profile.rowCount,
     columnCount: profile.columnCount,