astrolabe/src/core/profile.ts

/**
 * Dataset profiling (spec §05 → Profiling; docs/architecture/06 §3–§4).
 *
 * Portable core: no browser APIs, no React. A **profile** is the set of derived
 * summary fields stored on a dataset record so the UI can describe it without
 * re-parsing the payload: row/column counts, column names (in order), a per-column
 * inferred type, and an approximate byte size.
 *
 * `profileData` takes already-parsed rows-of-objects (the tabular form of a
 * CSV/TSV/JSON-array payload) plus a precomputed `size`. Parsing the delimited
 * text and deciding the payload shape happen *upstream* (see dataset.ts) so this
 * function stays pure and trivially testable.
 *
 *   - `null` rows (URL data) or an empty array (non-tabular) → the **N/A profile**
 *     (`rowCount`/`columnCount` null, empty columns), but `size` is still carried.
 *   - Columns are the union of keys across all rows, in **first-seen order**.
 *   - `rowCount`/`columnCount`/`size` reflect the **whole** payload; only type
 *     inference samples — capped at the first `SAMPLE_SIZE` rows for speed and
 *     determinism (arch 06 §4).
 */

import { inferColumnType, isEmpty, isNumeric, type ColumnType } from './type-inference';

export interface ColumnTypeInfo {
  /** The column name. */
  name: string;
  /** The inferred display type for the column. */
  type: ColumnType;
}

/**
 * Cap on distinct values counted per column. Past this the exact count stops
 * mattering for the chart-guidance hints (a legend or category axis with 50
 * entries is already unreadable), so counting stops and `distinctCapped` flags
 * that the true cardinality is at least this. Sampled like type inference (head of
 * the data), so it is an estimate — fine for soft advisories, not a guarantee.
 */
export const DISTINCT_CAP = 50;

/**
 * Per-column cardinality + numeric range, derived in the same sample pass as type
 * inference (arch 06 §4) to power the Chart Builder's data-aware Tier-B hints:
 * a crowded colour legend / category axis (`distinct`), and the negative-value
 * Size guard (`numericExtent`). Distinct from `ColumnTypeInfo` because it answers
 * "what shape is this data" rather than "what type to encode it as".
 */
export interface ColumnStats {
  /** Column name (matches a `columns` entry). */
  name: string;
  /** Distinct non-empty values in the sample, capped at `DISTINCT_CAP`. */
  distinct: number;
  /** True when `distinct` hit the cap — the real cardinality is ≥ `DISTINCT_CAP`. */
  distinctCapped: boolean;
  /** Min/max over numeric values, for numeric columns only; `null` otherwise. */
  numericExtent: { min: number; max: number } | null;
}

export interface DatasetProfile {
  /** Data rows, or `null` when N/A (URL / non-tabular). */
  rowCount: number | null;
  /** Columns, or `null` when N/A. */
  columnCount: number | null;
  /** Column names, in first-seen order. */
  columns: string[];
  /** Per-column inferred type. */
  columnTypes: ColumnTypeInfo[];
  /** Per-column cardinality + numeric range (empty when N/A). */
  columnStats: ColumnStats[];
  /** Approximate payload size in bytes. */
  size: number;
}

/** The N/A profile for URL / non-tabular data — still carries the byte size. */
const naProfile = (size: number): DatasetProfile => ({
  rowCount: null,
  columnCount: null,
  columns: [],
  columnTypes: [],
  columnStats: [],
  size,
});

/**
 * Cap on rows fed to type inference. Counts and size scan the whole payload; only
 * the per-value type check is bounded, sampling the head for determinism (§4).
 */
const SAMPLE_SIZE = 200;

const sampleRows = <T>(rows: ReadonlyArray<T>): ReadonlyArray<T> =>
  rows.length <= SAMPLE_SIZE ? rows : rows.slice(0, SAMPLE_SIZE);

/** A canonical key for distinct-counting (trim strings; stringify the rest). */
const distinctKey = (v: unknown): string => (typeof v === 'string' ? v.trim() : String(v));

/**
 * Cardinality + numeric extent for one column's sampled values. Distinct counting
 * stops growing the set once it passes `DISTINCT_CAP` (the extra value reveals the
 * overflow), but the extent scan continues over every value. Extent is reported
 * only for `number` columns — the only ones the Size channel can encode — using
 * the same numeric test as type inference, so the two never disagree.
 */
function columnStatsFor(name: string, values: readonly unknown[], type: ColumnType): ColumnStats {
  const seen = new Set<string>();
  let capped = false;
  let min = Infinity;
  let max = -Infinity;
  let sawNumber = false;

  for (const v of values) {
    if (isEmpty(v)) continue;
    if (!capped) {
      seen.add(distinctKey(v));
      if (seen.size > DISTINCT_CAP) capped = true;
    }
    if (type === 'number' && isNumeric(v)) {
      sawNumber = true;
      const n = typeof v === 'number' ? v : Number(String(v).trim());
      if (n < min) min = n;
      if (n > max) max = n;
    }
  }

  return {
    name,
    distinct: capped ? DISTINCT_CAP : seen.size,
    distinctCapped: capped,
    numericExtent: sawNumber ? { min, max } : null,
  };
}

/**
 * Profile a dataset payload. `rows` is the tabular form (CSV/TSV/JSON-array)
 * already parsed to rows-of-objects, or `null` for URL / non-tabular data;
 * `size` is the precomputed byte length of the stored payload.
 */
export function profileData(
  rows: ReadonlyArray<Record<string, unknown>> | null,
  size: number,
): DatasetProfile {
  if (!rows || rows.length === 0) return naProfile(size);

  // Column order = first-seen order across all rows (handles ragged rows).
  const columns: string[] = [];
  const seen = new Set<string>();
  for (const row of rows) {
    for (const key of Object.keys(row)) {
      if (!seen.has(key)) {
        seen.add(key);
        columns.push(key);
      }
    }
  }
  if (columns.length === 0) return naProfile(size);

  const sample = sampleRows(rows);
  // One pass per column over the sample: the inferred display type, plus the
  // cardinality + numeric extent that power the Chart Builder's data-aware hints
  // (crowded legend / category axis, negative-value Size guard — see chart-builder
  // `builderWarnings`; docs/chart-builder-research.md §8).
  const columnTypes: ColumnTypeInfo[] = [];
  const columnStats: ColumnStats[] = [];
  for (const name of columns) {
    const values = sample.map((r) => r[name]);
    const type = inferColumnType(values);
    columnTypes.push({ name, type });
    columnStats.push(columnStatsFor(name, values, type));
  }

  return {
    rowCount: rows.length,
    columnCount: columns.length,
    columns,
    columnTypes,
    columnStats,
    size,
  };
}