astrolabe/src/core/dataset.ts

/**
 * Dataset — a named, reusable data source snippets reference by name
 * (spec §09B → Dataset; spec §05 → Datasets).
 *
 * Portable core: no browser APIs, no React. Defines the record shape, the current
 * record schema version, the by-name reference object (spec §05 → Copy Reference),
 * a simple delimited-text parser, the profiling orchestration, and a factory that
 * stamps timestamps/version and fills the derived summary fields.
 *
 * A dataset has one of two **sources** — `inline` (data pasted into the record) or
 * `url` (fetched once from a remote address and **snapshotted** into the record) —
 * and one of four **formats** (reused from format-detection: `json`/`csv`/`tsv`/
 * `topojson`). Either way `data` holds the actual payload, shaped by format: raw
 * text for CSV/TSV, a parsed value for JSON/TopoJSON. A `url` dataset additionally
 * keeps its source `url` (so it can be re-fetched) and a `fetchedAt` timestamp;
 * until its first successful fetch `data` is `null` (an unfetched reference).
 *
 * Any tabular payload (JSON array-of-objects, CSV, TSV) is profiled — including a
 * fetched URL snapshot; non-tabular or not-yet-fetched data gets an N/A profile but
 * is still sized (see profile.ts).
 */

import { detectFormat, detectFormatFromUrl, type DataFormat } from './format-detection';
import { profileData, type ColumnStats, type DatasetProfile } from './profile';
import type { ColumnType } from './type-inference';

/**
 * Current schema version for a Dataset record (read-time migration target).
 * v2 moved a URL dataset's address out of `data` into its own `url` field and made
 * `data` hold the fetched snapshot (see `migrateDataset`).
 */
export const CURRENT_DATASET_VERSION = 2;

/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
export type DataSource = 'inline' | 'url';

export interface Dataset {
  /** Unique numeric identifier. */
  id: number;
  /** Record schema version, for read-time migration. */
  version: number;
  /** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
  name: string;
  /**
   * The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON
   * value. For `source = url` this is the fetched snapshot, or `null` before the
   * first successful fetch.
   */
  data: unknown;
  /** One of `json`, `csv`, `tsv`, `topojson`. */
  format: DataFormat;
  /** One of `inline` or `url`. */
  source: DataSource;
  /**
   * For `source = url`: the remote address the snapshot was fetched from, retained
   * so the dataset can be re-fetched ("Refresh"). Absent for inline datasets.
   */
  url?: string;
  /**
   * For `source = url`: ISO timestamp of the last successful fetch, or `null` when
   * it has never been fetched. Absent for inline datasets.
   */
  fetchedAt?: string | null;
  /** Free-form user note about the dataset. */
  comment: string;
  /** Data rows, or `null` when N/A (URL / non-tabular). */
  rowCount: number | null;
  /** Columns, or `null` when N/A. */
  columnCount: number | null;
  /** Column names, in order. */
  columns: string[];
  /** Per-column inferred type. */
  columnTypes: Array<{ name: string; type: ColumnType }>;
  /** Per-column cardinality + numeric range (empty for URL / non-tabular). */
  columnStats: ColumnStats[];
  /** Approximate payload size in bytes. */
  size: number;
  /** ISO timestamp — when first added. */
  created: string;
  /** ISO timestamp — when last changed. */
  modified: string;
}

/**
 * The by-name reference object copied into a spec via "Copy Reference"
 * (spec §05 → Actions): `{ "data": { "name": "MyDataset" } }`.
 */
export function datasetReference(name: string): { data: { name: string } } {
  return { data: { name } };
}

/** UTF-8 byte length of a string (`TextEncoder` is a platform global, not DOM). */
function byteLength(str: string): number {
  return new TextEncoder().encode(str).length;
}

/**
 * Tokenize delimited text into rows of string cells, honoring RFC-4180 quoting:
 * a field wrapped in `"` may contain the delimiter, newlines, and escaped quotes
 * (`""` → `"`); a quote is only special at the start of a field. This mirrors how
 * d3-dsv (the parser Vega-Lite uses at render time) reads the same text, so the
 * profile Astrolabe shows agrees with the field names the chart actually sees.
 * Whitespace is NOT trimmed — like d3, the cell is taken verbatim (type inference
 * trims internally when classifying, so numeric columns still detect).
 */
function tokenizeDelimited(text: string, delimiter: string): string[][] {
  const rows: string[][] = [];
  let row: string[] = [];
  let field = '';
  let inQuotes = false;
  let started = false; // any character seen for the current record?
  let i = text.charCodeAt(0) === 0xfeff ? 1 : 0; // skip a leading BOM
  const n = text.length;

  const endField = () => {
    row.push(field);
    field = '';
  };
  const endRow = () => {
    endField();
    rows.push(row);
    row = [];
    started = false;
  };

  while (i < n) {
    const c = text[i];
    if (inQuotes) {
      if (c === '"') {
        if (text[i + 1] === '"') {
          field += '"';
          i += 2;
          continue;
        }
        inQuotes = false;
        i++;
        continue;
      }
      field += c;
      i++;
      continue;
    }
    if (c === '"' && field === '') {
      inQuotes = true;
      started = true;
      i++;
      continue;
    }
    if (c === delimiter) {
      endField();
      started = true;
      i++;
      continue;
    }
    if (c === '\n' || c === '\r') {
      if (c === '\r' && text[i + 1] === '\n') i++;
      endRow();
      i++;
      continue;
    }
    field += c;
    started = true;
    i++;
  }
  // Flush a final field/row only if the last record had content (no phantom row
  // from a trailing newline).
  if (started || field !== '' || row.length > 0) endRow();
  return rows;
}

/**
 * Parse delimited text into rows-of-objects (header row → keys), RFC-4180 quoting
 * aware (see `tokenizeDelimited`). The first record is the header; each later
 * record is zipped header→cell. A record with fewer cells than the header leaves
 * the missing columns `undefined`; extra cells beyond the header are ignored.
 * Fully-empty records (e.g. a blank line) are skipped.
 */
export function parseDelimited(
  text: string,
  format: 'csv' | 'tsv',
): Array<Record<string, unknown>> {
  const delimiter = format === 'tsv' ? '\t' : ',';
  const records = tokenizeDelimited(text, delimiter);
  if (records.length < 2) return [];

  const header = records[0];
  const rows: Array<Record<string, unknown>> = [];
  for (let i = 1; i < records.length; i++) {
    const cells = records[i];
    if (cells.length === 1 && cells[0] === '') continue; // blank record
    const row: Record<string, unknown> = {};
    for (let c = 0; c < header.length; c++) {
      row[header[c]] = c < cells.length ? cells[c] : undefined;
    }
    rows.push(row);
  }
  return rows;
}

/** A JSON value is tabular when it is a non-empty array of plain objects. */
function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
  if (!Array.isArray(value) || value.length === 0) return null;
  const allObjects = value.every((v) => v !== null && typeof v === 'object' && !Array.isArray(v));
  return allObjects ? (value as Array<Record<string, unknown>>) : null;
}

/**
 * The tabular rows of a dataset payload for a table preview, or `null` when the
 * payload isn't tabular (a single JSON object, TopoJSON, or an unfetched URL). Uses
 * the **same** parsing as profiling — `parseDelimited` for CSV/TSV, `asObjectRows`
 * for JSON — so the previewed rows agree exactly with the profiled `columns`. A
 * positive `limit` returns only the head (a preview needs a sample, not the whole
 * payload). Returns non-null on precisely the inputs `computeDatasetProfile` counts
 * as rows, so a caller can gate "table vs. raw text" on this alone.
 */
export function tabularRows(
  data: unknown,
  format: DataFormat,
  limit?: number,
): Array<Record<string, unknown>> | null {
  if (data == null) return null;
  let rows: Array<Record<string, unknown>> | null;
  switch (format) {
    case 'csv':
    case 'tsv':
      rows = parseDelimited(typeof data === 'string' ? data : '', format);
      break;
    case 'json':
      rows = asObjectRows(data);
      break;
    default:
      rows = null;
  }
  if (!rows || rows.length === 0) return null;
  return limit != null && limit >= 0 && rows.length > limit ? rows.slice(0, limit) : rows;
}

/**
 * Orchestrate profiling for a dataset payload: compute `size` (always), decide
 * tabular vs N/A by format, and delegate to `profileData`. Identical for inline
 * data and for a fetched URL snapshot — both carry the payload in `data`.
 *
 *   - `null` data (unfetched URL) → N/A profile, size 0.
 *   - `json`                      → rows when a non-empty array of objects, else N/A.
 *   - `topojson`                  → N/A (non-tabular).
 *   - `csv` / `tsv`               → `parseDelimited` rows.
 *
 * `size` is the UTF-8 byte length of the raw string for csv/tsv, or of
 * `JSON.stringify(data)` for json/topojson.
 */
export function computeDatasetProfile(data: unknown, format: DataFormat): DatasetProfile {
  // An unfetched URL reference (or a genuinely absent payload): nothing to profile.
  if (data == null) return profileData(null, 0);

  switch (format) {
    case 'csv':
    case 'tsv': {
      const text = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
      return profileData(parseDelimited(text, format), byteLength(text));
    }
    case 'json': {
      const size = byteLength(JSON.stringify(data) ?? '');
      return profileData(asObjectRows(data), size);
    }
    case 'topojson':
    default: {
      const size = byteLength(JSON.stringify(data) ?? '');
      return profileData(null, size);
    }
  }
}

/**
 * Shape a freshly-fetched URL body into the `{ data, format }` a snapshot stores
 * (spec §05 → URL datasets, snapshot model). Format is sniffed from the **content**
 * first — authoritative, since `detectFormat` only reports `json` when the body
 * actually parses — falling back to the URL's file extension, then JSON.
 * JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text — the same
 * per-format shaping inline data uses, so a fetched dataset profiles and renders
 * identically to an inline one (see `computeDatasetProfile`, rendering.ts).
 */
export function snapshotFromText(text: string, url: string): { data: unknown; format: DataFormat } {
  const format = detectFormat(text).format ?? detectFormatFromUrl(url) ?? 'json';
  if (format === 'json' || format === 'topojson') {
    try {
      return { data: JSON.parse(text) as unknown, format };
    } catch {
      // The extension promised JSON but the body isn't — keep the raw text so the
      // render surfaces a readable error instead of us throwing mid-commit.
      return { data: text, format };
    }
  }
  return { data: text, format };
}

export interface CreateDatasetOptions {
  /** The dataset name (uniqueness is enforced upstream — see naming.ts). */
  name: string;
  /** The payload, shaped per source/format (see `Dataset.data`). */
  data: unknown;
  /** One of `json`, `csv`, `tsv`, `topojson`. */
  format: DataFormat;
  /** One of `inline` or `url`. */
  source: DataSource;
  /** For `source = url`: the remote address (retained for Refresh). */
  url?: string;
  /** For `source = url`: ISO timestamp of the fetch that produced `data`. */
  fetchedAt?: string | null;
  /** Optional free-form note. */
  comment?: string;
  /** Clock injection for deterministic tests; defaults to the current time. */
  now?: Date;
  /** Id injection for deterministic tests; defaults to `Date.now()`. */
  id?: number;
}

/**
 * Create a Dataset: stamps version/timestamps and runs `computeDatasetProfile` to
 * fill the derived summary fields. `id` defaults to `Date.now()` (numeric) and is
 * injectable for tests; note that `DatasetStore` is the id authority and reassigns
 * a collision-free id on insertion (`nextDatasetId`), so this default never reaches
 * storage through the normal create paths.
 */
export function createDataset(options: CreateDatasetOptions): Dataset {
  const now = options.now ?? new Date();
  const iso = now.toISOString();
  const profile = computeDatasetProfile(options.data, options.format);

  return {
    id: options.id ?? Date.now(),
    version: CURRENT_DATASET_VERSION,
    name: options.name,
    data: options.data,
    format: options.format,
    source: options.source,
    // URL datasets carry their address + fetch time; inline records stay clean.
    ...(options.source === 'url' ? { url: options.url, fetchedAt: options.fetchedAt ?? null } : {}),
    comment: options.comment ?? '',
    rowCount: profile.rowCount,
    columnCount: profile.columnCount,
    columns: profile.columns,
    columnTypes: profile.columnTypes,
    columnStats: profile.columnStats,
    size: profile.size,
    created: iso,
    modified: iso,
  };
}