astrolabe/src/core/chart-builder.ts

/**
 * Chart Builder — pure Vega-Lite spec assembler (spec §06).
 *
 * Portable core: no browser APIs, no React, no store access. Turns a no-JSON
 * builder configuration (a mark, four optional encoding channels mapped to dataset
 * columns, with per-channel transforms, plus chart-level sort/stack and optional
 * pixel dimensions) into a complete Vega-Lite spec that references the source
 * dataset by name. The UI layer owns the controls; this module owns the spec
 * grammar — what a configuration *means* as Vega-Lite — and the defaults the spec
 * prescribes (pre-population, field-type derivation, smart mark).
 *
 * Beyond the Tier-B floor, a channel may carry a **transform** — an `aggregate`
 * (count/sum/mean/…), a quantitative `bin`, or a temporal `timeUnit` granularity —
 * and the chart may carry a `sort` (rank a categorical axis by its measure) and a
 * `stack` (part-to-whole for bar/area + colour). See
 * docs/chart-builder-research.md §8 for why each exists.
 *
 * The produced spec mirrors what the rest of Astrolabe authors by hand: a `$schema`
 * stamp (shared with the sample template), a named-data reference the renderer
 * resolves at preview time (rendering.ts), a mark with tooltips enabled, the mapped
 * encodings, and any explicit width/height. It is the same string-spec shape the
 * editor and preview consume — `buildSnippetSpecText` serializes it.
 */

import type { ColumnStats } from './profile';
import { DISTINCT_CAP } from './profile';
import type { ColumnType } from './type-inference';
import { VEGA_LITE_SCHEMA_URL } from './snippet';
import { validateExpression } from './expr-validate';
import { escapeVegaField } from './rendering';

/** The five mark types the builder offers, in selector order (spec §06). */
export const MARK_TYPES = ['bar', 'line', 'point', 'area', 'circle'] as const;
export type MarkType = (typeof MARK_TYPES)[number];

/** The four Vega-Lite field types a channel may carry, in override-menu order. */
export const FIELD_TYPES = ['quantitative', 'nominal', 'ordinal', 'temporal'] as const;
export type FieldType = (typeof FIELD_TYPES)[number];

/** The four encoding channels the builder offers, in display order (spec §06). */
export const CHANNELS = ['x', 'y', 'color', 'size'] as const;
export type ChannelName = (typeof CHANNELS)[number];

/**
 * Aggregation operators a channel may apply (Vega-Lite `aggregate`). `count` is
 * special — it is **field-less** (counts records), so a `count` mapping carries no
 * `field`. The rest reduce a quantitative `field`.
 */
export const AGGREGATE_OPS = ['count', 'sum', 'mean', 'median', 'min', 'max'] as const;
export type AggregateOp = (typeof AGGREGATE_OPS)[number];

/**
 * Temporal granularities (Vega-Lite `timeUnit`), coarse → fine, with the combined
 * units that make a real time axis. Offered when a channel's type is Temporal.
 */
export const TIME_UNITS = [
  'year',
  'yearquarter',
  'yearmonth',
  'yearmonthdate',
  'quarter',
  'month',
  'week',
  'date',
  'day',
  'hours',
] as const;
export type TimeUnit = (typeof TIME_UNITS)[number];

/** Sort the categorical axis by its measure (spec §06 → Ranking). */
export const SORT_ORDERS = ['ascending', 'descending'] as const;
export type SortOrder = (typeof SORT_ORDERS)[number];

/** Part-to-whole stacking for bar/area + a colour series: absolute vs 100%. */
export const STACK_MODES = ['zero', 'normalize'] as const;
export type StackMode = (typeof STACK_MODES)[number];

/**
 * Comparison operators a guarded filter predicate offers (Vega-Lite field
 * predicates), in menu order. The set a given field admits is narrowed by
 * `validFilterOps` — only a measure/temporal field offers ordering (`lt`…`gt`) and
 * `range`; a category offers membership (`oneOf`). `notEqual` is expressed as a
 * `{ not: { …equal } }` logical wrapper (Vega-Lite has no bare `!=` predicate).
 */
export const FILTER_OPS = [
  'equal',
  'notEqual',
  'lt',
  'lte',
  'gt',
  'gte',
  'range',
  'oneOf',
] as const;
export type FilterOp = (typeof FILTER_OPS)[number];

/** How a filter expresses its predicate: a guarded shelf, or a raw expression. */
export type FilterMode = 'predicate' | 'expression';

/**
 * One top-level data filter (Vega-Lite `transform: [{ filter }]`), applied to the
 * raw rows **before** encoding aggregation — so "filter rows, then aggregate" is the
 * natural reading. Two modes share one shape (the UI toggles between them on a single
 * row):
 *
 *   - **predicate** — a guarded `field op value` triple (Voyager-style, no expression
 *     to write for the common case). `range` carries a second bound (`value2`);
 *     `oneOf` reads `value` as a comma-separated membership list. Values are coerced
 *     by `fieldType` (a quantitative field compares as a number).
 *   - **expression** — a raw Vega expression string (`datum.x > 0`), the power-form
 *     for what the guarded shelf can't say (validated in the UI via
 *     `expr-validate.ts`).
 *
 * `id` is a stable key for list editing only; it never reaches the produced spec.
 * A partially-filled filter (no value yet, empty expression) is simply skipped by
 * the assembler, so the live preview keeps rendering while the user types.
 */
export interface BuilderFilter {
  /** Stable list key (UI-assigned); not serialized into the spec. */
  id: string;
  /** Which form this filter takes. */
  mode: FilterMode;
  /** Predicate mode: the column being filtered (a dataset or calculated field). */
  field?: string;
  /** Predicate mode: the field's Vega-Lite type — drives the op menu + coercion. */
  fieldType?: FieldType;
  /** Predicate mode: the comparison operator. */
  op?: FilterOp;
  /** Predicate mode: the comparison value (`range` lower bound; `oneOf` CSV list). */
  value?: string;
  /** Predicate mode: the upper bound, for `range` only. */
  value2?: string;
  /** Expression mode: a raw Vega predicate expression. */
  expr?: string;
}

/**
 * One derived field (Vega-Lite `transform: [{ calculate, as }]`): a row-wise Vega
 * expression producing a new column the rest of the builder can encode like any
 * other. Calculates are emitted **before** filters (a row-wise calculate is
 * order-independent — it computes the same per-row value regardless of which rows
 * survive — so running it first is equivalent and lets a filter reference the
 * derived field). `id` is a UI list key only.
 */
export interface BuilderCalculate {
  /** Stable list key (UI-assigned); not serialized into the spec. */
  id: string;
  /** The Vega expression, e.g. `datum.price * datum.quantity`. */
  expr: string;
  /** The new field's name (appears in the channel dropdowns once non-empty). */
  as: string;
}

/**
 * One channel's mapping: a dataset column `field` plus its `type`, with optional
 * transforms. `field` is omitted only for a `count` aggregate (which counts records
 * rather than reducing a column). A channel left on "None" is `null` in the config
 * (omitted from the spec).
 */
export interface ChannelMapping {
  /** The dataset column. Omitted only when `aggregate === 'count'`. */
  field?: string;
  /** The Vega-Lite field type (see `validFieldTypes`). */
  type: FieldType;
  /** Aggregation op; `count` is field-less, the rest reduce a quantitative field. */
  aggregate?: AggregateOp;
  /** Bin a quantitative field into ranges (e.g. for a histogram). */
  bin?: boolean;
  /** Temporal granularity for a Temporal field. */
  timeUnit?: TimeUnit;
}

/** The full builder configuration the assembler consumes. */
export interface BuilderConfig {
  /** The dataset the spec references by name (`{ data: { name } }`). */
  datasetName: string;
  /** The active mark type. */
  mark: MarkType;
  /** Per-channel mapping; `null` (or absent) means the channel is unmapped. */
  encodings: Partial<Record<ChannelName, ChannelMapping | null>>;
  /** Optional explicit chart width in pixels. */
  width?: number;
  /** Optional explicit chart height in pixels. */
  height?: number;
  /** Sort the categorical positional axis by the measure axis (spec §06 → Ranking). */
  sort?: SortOrder;
  /** Stacking for bar/area + a colour series (part-to-whole). */
  stack?: StackMode;
  /** Derived fields (`transform: [{ calculate, as }]`), emitted before `filters`. */
  calculates?: BuilderCalculate[];
  /** Row filters (`transform: [{ filter }]`), applied before encoding aggregation. */
  filters?: BuilderFilter[];
}

/**
 * The field types a column may legitimately carry, given its inferred type — the
 * options the channel's type control offers (spec §06 → Tier B, valid-type
 * locking). A string/boolean is never Quantitative and a non-date is never
 * Temporal (Vega-Lite would mis-encode or error); a number defaults to
 * Quantitative but may be treated as a category. Ordinal is offered wherever the
 * user might reasonably assert an order (numbers, text), a deliberate superset of
 * Voyager's stricter menu. The list head is the default (see `defaultFieldType`).
 *
 * Convergent across the research: Voyager `getValidTypes`
 * (data-pane/field-list.tsx) + Draco `hard.lp` enc_type_valid (a string/boolean
 * can't be quantitative; temporal requires datetime). See
 * docs/chart-builder-research.md §4.
 */
export function validFieldTypes(columnType: ColumnType): FieldType[] {
  switch (columnType) {
    case 'number':
      return ['quantitative', 'ordinal', 'nominal'];
    case 'date':
      return ['temporal'];
    case 'boolean':
      return ['nominal'];
    default: // 'string'
      return ['nominal', 'ordinal'];
  }
}

/**
 * Default Vega-Lite field type for a column from its inferred type (spec §06):
 * numeric → Quantitative, date → Temporal, everything else (text, boolean) →
 * Nominal. The default is the head of `validFieldTypes`, so the two never drift.
 * The user may override afterward, within `validFieldTypes`.
 */
export function defaultFieldType(columnType: ColumnType): FieldType {
  return validFieldTypes(columnType)[0];
}

/** Continuous (measure-like) field types — quantitative and temporal. */
function isContinuous(type: FieldType): boolean {
  return type === 'quantitative' || type === 'temporal';
}

/**
 * Whether a field type may be placed on a channel at all (spec §06 → Tier B, Size
 * discipline). X/Y/Color accept any type; **Size accepts only Quantitative or
 * Ordinal** — encoding a category or a date by symbol size is misleading (size
 * implies ordered magnitude). Draco makes this a hard constraint
 * (`hard.lp:53` size_nominal); we surface it as a UI gate that disables Size for
 * unsuitable columns rather than letting the user produce the bad encoding.
 * (Negative-value exclusion, `hard.lp:56`, needs row data — `builderWarnings`
 * surfaces it as a soft hint from the column's profiled numeric extent; this
 * type-level gate is the part the builder can enforce structurally.)
 */
export function isChannelTypeAllowed(channel: ChannelName, type: FieldType): boolean {
  if (channel === 'size') return type === 'quantitative' || type === 'ordinal';
  return true;
}

/** Whether a non-count aggregate (sum/mean/…) can apply to this field type. */
export function supportsAggregate(type: FieldType): boolean {
  return type === 'quantitative';
}

/** Whether binning into ranges can apply to this field type. */
export function supportsBin(type: FieldType): boolean {
  return type === 'quantitative';
}

/** Whether a temporal granularity (timeUnit) can apply to this field type. */
export function supportsTimeUnit(type: FieldType): boolean {
  return type === 'temporal';
}

/**
 * The mark that best fits the X/Y field-type shape (spec §06 → Tier B, smart
 * default mark) — the research's strongest convergence (Draco mark-by-shape soft
 * constraints, Voyager effectiveness, FT, Datawrapper all agree;
 * docs/chart-builder-research.md §4):
 *
 *   - temporal × quantitative → **Line** (a time series)
 *   - quantitative × quantitative → **Point** (a scatter)
 *   - one continuous + one discrete axis → **Bar** (category vs measure)
 *   - both axes discrete → **Point** (Bar/Line/Area need a continuous axis)
 *   - a single mapped axis, or nothing yet → **Bar** (the safe default)
 *
 * `null` means the channel is unmapped. This is the *default*; the user can switch
 * to any of the five marks afterward.
 */
export function defaultMark(xType: FieldType | null, yType: FieldType | null): MarkType {
  if (xType === null || yType === null) return 'bar';
  const timeVsMeasure =
    (xType === 'temporal' && yType === 'quantitative') ||
    (xType === 'quantitative' && yType === 'temporal');
  if (timeVsMeasure) return 'line';
  const xc = isContinuous(xType);
  const yc = isContinuous(yType);
  if (xc && yc) return 'point'; // two measures → scatter
  if (xc === yc) return 'point'; // both discrete → bar/line/area are invalid here
  return 'bar'; // one continuous axis, one categorical → category-vs-measure bar
}

/** A dataset's columns paired with their inferred types — the builder's input. */
export interface BuilderColumns {
  columns: readonly string[];
  columnTypes: ReadonlyArray<{ name: string; type: ColumnType }>;
  /**
   * Per-column cardinality + numeric range, when the dataset was profiled with it
   * (datasets created before this profiling, and URL/non-tabular data, omit it).
   * The data-aware `builderWarnings` use it where present and silently skip those
   * hints where absent.
   */
  columnStats?: ReadonlyArray<ColumnStats>;
}

/** The derived field type for a named column, defaulting to Nominal if unknown. */
function fieldTypeForColumn(name: string, columns: BuilderColumns): FieldType {
  const match = columns.columnTypes.find((c) => c.name === name);
  return defaultFieldType(match?.type ?? 'string');
}

/**
 * Above this many distinct values, a column is too high-cardinality to be a good
 * default category axis: its labels overlap into an unreadable axis, and an
 * unaggregated chart that wide can exceed the canvas size limit. Matches the
 * crowded-axis warning threshold.
 */
const CATEGORY_DEFAULT_MAX_DISTINCT = 30;

/**
 * Pick a *sensible, renderable* default X/Y from the data shape, using profiled
 * cardinality (`columnStats`). This avoids opening the builder on a degenerate
 * one-mark-per-row chart — the shape a positional first-two-columns rule yields when
 * the leading columns are an id and a high-cardinality key. Returns `null` when there
 * are no stats to reason about (older / URL datasets), so the caller falls back to the
 * positional default.
 *
 * Preference order (each guaranteed to render and read cleanly):
 *   1. a low-cardinality **category** vs a **count of records** → a tidy bar;
 *   2. else a **temporal** axis vs count → a time series (continuous x, always fits);
 *   3. else two **measures** → a scatter (continuous axes, always fit).
 *
 * The category case pairs with the field-less **count**, not a raw measure: count is
 * always meaningful and avoids summing an id-like numeric (Row ID, Postal Code) into
 * nonsense. This is the builder's *opening* state only; a later intent-first entry
 * point can layer richer recommendations on top.
 */
function smartDefaultEncodings(
  columns: BuilderColumns,
): { x: ChannelMapping; y: ChannelMapping } | null {
  const stats = columns.columnStats;
  if (!stats || stats.length === 0) return null; // no profiling → positional fallback
  const typeOf = (name: string): ColumnType =>
    columns.columnTypes.find((c) => c.name === name)?.type ?? 'string';
  const knownDistinct = (name: string): number | undefined => {
    const s = stats.find((x) => x.name === name);
    return s && !s.distinctCapped ? s.distinct : undefined;
  };
  const count: ChannelMapping = { type: 'quantitative', aggregate: 'count' };
  const numbers = columns.columns.filter((name) => typeOf(name) === 'number');

  // 1. The lowest-cardinality readable category (string/boolean) → a tidy bar. Pair it
  //    with **count** (not a raw measure): a raw measure would draw one bar per row.
  const category = columns.columns
    .filter((name) => {
      const t = typeOf(name);
      if (t !== 'string' && t !== 'boolean') return false;
      const d = knownDistinct(name);
      return d !== undefined && d >= 2 && d <= CATEGORY_DEFAULT_MAX_DISTINCT;
    })
    .sort((a, b) => (knownDistinct(a) ?? 0) - (knownDistinct(b) ?? 0))[0];
  if (category) return { x: { field: category, type: 'nominal' }, y: count };

  // 2. A date → a time series of the first measure (a temporal axis is continuous, so a
  //    line of raw values always fits); fall back to count if there is no measure.
  const temporal = columns.columns.find((name) => typeOf(name) === 'date');
  if (temporal) {
    const y: ChannelMapping = numbers[0] ? { field: numbers[0], type: 'quantitative' } : count;
    return { x: { field: temporal, type: 'temporal' }, y };
  }

  // 3. Two measures → a scatter (continuous axes, always renderable).
  if (numbers.length >= 2) {
    return {
      x: { field: numbers[0], type: 'quantitative' },
      y: { field: numbers[1], type: 'quantitative' },
    };
  }
  return null;
}

/**
 * The builder's opening configuration for a dataset (spec §06 → Default
 * pre-population, Tier B). When the dataset is profiled, X/Y are chosen as a
 * **data-aware "safest bet"** (`smartDefaultEncodings`) — a low-cardinality category
 * vs a count of records (a tidy bar), else a time series, else a scatter — so the
 * builder never opens on a degenerate one-mark-per-row chart that can't render. When
 * there are no stats to reason about, it falls back to the positional rule: first
 * column on X, second (if any) on Y, each with its derived field type. Either way the
 * mark is the **smart default** for the resulting X/Y shape (`defaultMark`), Color
 * and Size start unmapped with no transforms, and a dataset with no detected columns
 * yields an all-unmapped config (the modal then prompts).
 */
export function defaultBuilderConfig(datasetName: string, columns: BuilderColumns): BuilderConfig {
  const encodings: Partial<Record<ChannelName, ChannelMapping | null>> = {
    x: null,
    y: null,
    color: null,
    size: null,
  };
  // Prefer a data-aware "safest bet" (a renderable, readable chart) when the dataset
  // is profiled; otherwise fall back to the positional first-on-X, second-on-Y rule.
  const smart = smartDefaultEncodings(columns);
  if (smart) {
    encodings.x = smart.x;
    encodings.y = smart.y;
  } else {
    const [first, second] = columns.columns;
    if (first !== undefined) {
      encodings.x = { field: first, type: fieldTypeForColumn(first, columns) };
    }
    if (second !== undefined) {
      encodings.y = { field: second, type: fieldTypeForColumn(second, columns) };
    }
  }
  const mark = defaultMark(encodings.x?.type ?? null, encodings.y?.type ?? null);
  return { datasetName, mark, encodings };
}

/** The channels actually mapped (a column field, or a field-less count), in order. */
function mappedChannels(config: BuilderConfig): Array<[ChannelName, ChannelMapping]> {
  return CHANNELS.flatMap((channel) => {
    const mapping = config.encodings[channel];
    return mapping ? [[channel, mapping] as [ChannelName, ChannelMapping]] : [];
  });
}

/** The effective field type a mapping encodes (a count is quantitative). */
function effectiveType(mapping: ChannelMapping): FieldType {
  return mapping.aggregate === 'count' ? 'quantitative' : mapping.type;
}

/** True when a mapping reads as a continuous measure (count/aggregate or continuous type). */
function isMeasureMapping(mapping: ChannelMapping): boolean {
  return isContinuous(effectiveType(mapping));
}

/**
 * Whether the configuration is renderable / saveable (spec §06 → Validation): at
 * least one channel must be mapped. The modal gates the Create action and the
 * preview prompt on this.
 */
export function isBuilderConfigValid(config: BuilderConfig): boolean {
  return mappedChannels(config).length > 0;
}

/**
 * The categorical positional channel to sort, when exactly one of X/Y is a discrete
 * category and the other is a measure (spec §06 → Ranking). Returns the channel to
 * carry `sort`, or undefined when sorting doesn't apply (no clear category axis).
 */
export function sortableCategoryChannel(config: BuilderConfig): 'x' | 'y' | undefined {
  const x = config.encodings.x ?? null;
  const y = config.encodings.y ?? null;
  if (!x || !y) return undefined;
  const xMeasure = isMeasureMapping(x);
  const yMeasure = isMeasureMapping(y);
  if (xMeasure && !yMeasure) return 'y';
  if (yMeasure && !xMeasure) return 'x';
  return undefined;
}

/** The quantitative positional channel (x or y) that stacking applies to, if any. */
function stackMeasureChannel(config: BuilderConfig): 'x' | 'y' | undefined {
  for (const channel of ['x', 'y'] as const) {
    const mapping = config.encodings[channel];
    if (mapping && effectiveType(mapping) === 'quantitative') return channel;
  }
  return undefined;
}

/**
 * Whether sorting can be offered for this config (a clear category-vs-measure axis
 * pair exists). The UI shows the Sort control only when true.
 */
export function supportsSort(config: BuilderConfig): boolean {
  return sortableCategoryChannel(config) !== undefined;
}

/**
 * Whether stacking can be offered: a bar/area mark with a colour series and a
 * quantitative positional axis to stack along (spec §06 → part-to-whole).
 */
export function supportsStack(config: BuilderConfig): boolean {
  return (
    (config.mark === 'bar' || config.mark === 'area') &&
    !!config.encodings.color &&
    stackMeasureChannel(config) !== undefined
  );
}

/**
 * A one-click remedy a warning can offer. `apply` is a pure config→config transform;
 * the modal renders `label` as a button that runs it. It is an *offer*, never a forced
 * change — once applied, the warning re-derives away. Lives in core so the remedies
 * unit-test alongside the warnings.
 */
export interface BuilderWarningFix {
  /** The button label naming the remedy, e.g. "Aggregate as Sum". */
  label: string;
  /** Produce the corrected configuration from the current one (pure). */
  apply: (config: BuilderConfig) => BuilderConfig;
}

/** A non-blocking advisory about a configuration (spec §06 → Tier B warnings). */
export interface BuilderWarning {
  /** The channel the hint is about, when it's channel-specific. */
  channel?: ChannelName;
  /** A short, plain-language hint the modal shows inline (not an error). */
  message: string;
  /** Optional one-click remedies the modal renders as buttons next to the hint. */
  fixes?: BuilderWarningFix[];
}

/**
 * Above this many rows, a category-vs-measure bar/line/area with a **raw**
 * (un-aggregated) measure draws so many marks — one per row — that the category
 * axis becomes an unreadable picket fence of labels. The threshold is a legibility
 * estimate, not a hard limit (the chart still renders); it's set where vertical bar
 * labels reliably start overlapping. See `builderWarnings`.
 */
const CROWDED_CATEGORY_ROWS = 30;

/**
 * Above this many *distinct* category values, even an aggregated category axis (one
 * mark per category, not per row) has too many labels to read. Matched to
 * `CROWDED_CATEGORY_ROWS`: both flag a category axis that overruns its labels.
 */
const CROWDED_CATEGORY_DISTINCT = 30;

/**
 * Above this many distinct colour values a discrete legend is too long to scan and
 * the palette starts recycling hues. Qualitative colour scales top out around 8–12
 * across the canon (Datawrapper, FT Visual Vocabulary); 12 is the generous end.
 */
const CROWDED_LEGEND_DISTINCT = 12;

/** Profiled stats for a mapped field, when the dataset carries them. */
function statsFor(field: string | undefined, columns?: BuilderColumns): ColumnStats | undefined {
  if (!field || !columns?.columnStats) return undefined;
  return columns.columnStats.find((s) => s.name === field);
}

/** "N" or "more than 50" — distinct count, honouring the profiler's cap. */
function cardinalityText(stats: ColumnStats): string {
  return stats.distinctCapped ? `more than ${DISTINCT_CAP}` : `${stats.distinct}`;
}

// --- Pure config transforms backing the actionable-hint fixes. They mirror the
// store's setChannelAggregate / swapXY / setStack / setChannelColumn(null) / setMark
// actions, so applying a fix and making the equivalent manual edit land on the same
// config. Kept here (not the store) so the remedies are pure and unit-testable. ---

/** Aggregate one channel's field (clearing any bin — the two are mutually exclusive). */
function withChannelAggregate(
  config: BuilderConfig,
  channel: ChannelName,
  aggregate: AggregateOp,
): BuilderConfig {
  const current = config.encodings[channel];
  if (!current) return config;
  const next: ChannelMapping = { ...current, aggregate };
  delete next.bin;
  return { ...config, encodings: { ...config.encodings, [channel]: next } };
}

/** Exchange the X and Y mappings (the manual Swap X/Y, as a pure transform). */
function withSwappedXY(config: BuilderConfig): BuilderConfig {
  return {
    ...config,
    encodings: {
      ...config.encodings,
      x: config.encodings.y ?? null,
      y: config.encodings.x ?? null,
    },
  };
}

/** Clear one channel back to "None" (drops its mapping from the spec). */
function withChannelCleared(config: BuilderConfig, channel: ChannelName): BuilderConfig {
  return { ...config, encodings: { ...config.encodings, [channel]: null } };
}

/**
 * Non-blocking advisories for the current configuration (spec §06 → Tier B): the
 * encodings that render but read poorly, drawn from the research's soft rules
 * (docs/chart-builder-research.md §4, §7). These never block Create — `isBuilder
 * ConfigValid` is the only gate — they just steer the user toward a better chart.
 * Returned in a stable order so the UI list doesn't jitter as config changes.
 *
 * `rowCount` (the dataset's row count, when known) powers the one-mark-per-row hint;
 * `columns` (with per-column `columnStats`) powers the data-aware hints — crowded
 * legend / category axis (cardinality) and the negative-value Size guard (numeric
 * extent). Either omitted (URL / non-tabular / pre-cardinality datasets) simply
 * skips the hints that need it.
 */
export function builderWarnings(
  config: BuilderConfig,
  rowCount?: number | null,
  columns?: BuilderColumns,
): BuilderWarning[] {
  const warnings: BuilderWarning[] = [];
  const x = config.encodings.x ?? null;
  const y = config.encodings.y ?? null;
  const { mark } = config;

  // Line/area are two-axis marks: a single mapped axis can't draw a meaningful line
  // or band (Draco hard.lp:91 line_area requires both x and y).
  if ((mark === 'line' || mark === 'area') && (x === null || y === null)) {
    warnings.push({
      message: `${mark === 'line' ? 'Line' : 'Area'} charts need both an X and a Y axis.`,
    });
  }

  // Bar/line/area need a measure on one axis; two categories give nothing to compare
  // (Draco soft.lp:47 only_discrete — the loudest nudge; hard.lp:97/:100 for bar).
  if (
    (mark === 'bar' || mark === 'line' || mark === 'area') &&
    x !== null &&
    y !== null &&
    !isMeasureMapping(x) &&
    !isMeasureMapping(y)
  ) {
    warnings.push({
      message: `${markLabel(mark)} charts need a measure (a value or count) on the X or Y axis.`,
    });
  }

  // Two measures on a non-scatter mark: a line/bar/area over two quantitative axes
  // misleads; a scatter is the conventional choice (Draco soft.lp c_c weights).
  if (
    x !== null &&
    y !== null &&
    effectiveType(x) === 'quantitative' &&
    effectiveType(y) === 'quantitative' &&
    mark !== 'point' &&
    mark !== 'circle'
  ) {
    warnings.push({
      message: 'Two measures usually read best as a scatter.',
      fixes: [{ label: 'Switch to Point', apply: (c) => ({ ...c, mark: 'point' }) }],
    });
  }

  // Area split into many series hides per-component change (FT Visual Vocabulary:
  // "seeing change in components can be very difficult").
  // (Stacking turns overlapping series into a cumulative part-to-whole, a valid read,
  // so a stacked area is not flagged — applying the [Stack] fix below clears this.)
  if (mark === 'area' && config.encodings.color && !config.stack) {
    const fixes: BuilderWarningFix[] = [];
    if (supportsStack(config)) {
      fixes.push({ label: 'Stack', apply: (c) => ({ ...c, stack: 'zero' }) });
    }
    fixes.push({ label: 'Remove colour', apply: (c) => withChannelCleared(c, 'color') });
    warnings.push({
      channel: 'color',
      message: 'Area charts make per-series change hard to read.',
      fixes,
    });
  }

  // Crowded category axis: a bar/line/area whose discrete category axis carries too
  // many entries to label legibly. Two ways it happens, mutually exclusive:
  //   1. A *raw* (un-aggregated, un-binned) measure draws one mark per row, so the
  //      label count == rowCount — flagged from rowCount alone (the builder's own
  //      default does this: first column on X, second on Y, no aggregate).
  //   2. Otherwise (aggregated/binned), one mark per *category* — crowded only when
  //      the category column itself has many distinct values, which needs the
  //      profiled cardinality (columnStats) and is skipped without it.
  // The remedy follows the canon: for case 1 aggregate to one mark per category; for
  // either, a bar can flip to horizontal where long labels stay readable (FT Visual
  // Vocabulary: bar is "good when … labels have long category names"; Datawrapper:
  // long category lists belong on a horizontal bar).
  if (mark === 'bar' || mark === 'line' || mark === 'area') {
    const category = sortableCategoryChannel(config); // discrete axis of a category-vs-measure pair
    const measureChannel = category ? (category === 'x' ? 'y' : 'x') : null;
    const measure = measureChannel ? (config.encodings[measureChannel] ?? null) : null;
    if (category && measureChannel && measure) {
      const rawMeasure = !measure.aggregate && !measure.bin;
      if (rawMeasure && typeof rowCount === 'number' && rowCount > CROWDED_CATEGORY_ROWS) {
        // Aggregating the measure collapses one-mark-per-row to one-per-category; a bar
        // can also flip horizontal (Swap X/Y) where long labels stay readable. Offer
        // aggregate only for a quantitative measure (Sum is meaningless on a date).
        const fixes: BuilderWarningFix[] = [];
        if (effectiveType(measure) === 'quantitative') {
          fixes.push({
            label: 'Aggregate as Sum',
            apply: (c) => withChannelAggregate(c, measureChannel, 'sum'),
          });
        }
        if (mark === 'bar') fixes.push({ label: 'Swap X/Y', apply: withSwappedXY });
        warnings.push({
          channel: category,
          message: `This draws one mark per row (${rowCount} in this dataset), so the category-axis labels will overlap.`,
          fixes: fixes.length ? fixes : undefined,
        });
      } else {
        const stats = statsFor(config.encodings[category]?.field, columns);
        if (stats && stats.distinct > CROWDED_CATEGORY_DISTINCT) {
          // Aggregated already, so the remedy is fewer categories (filter — not yet a
          // builder control) or, for a bar, a horizontal flip where long lists fit.
          const fixes: BuilderWarningFix[] =
            mark === 'bar' ? [{ label: 'Swap X/Y', apply: withSwappedXY }] : [];
          warnings.push({
            channel: category,
            message: `This category axis has ${cardinalityText(stats)} distinct values, so its labels will overlap.`,
            fixes: fixes.length ? fixes : undefined,
          });
        }
      }
    }
  }

  // Crowded colour legend: a discrete colour series with many distinct values makes
  // the legend too long to scan and forces the palette to recycle hues. A continuous
  // colour ramp (quantitative/temporal) has no per-value legend, so this is for
  // discrete colour only (Datawrapper/FT cap categorical colour low).
  const color = config.encodings.color ?? null;
  if (color && !isMeasureMapping(color)) {
    const stats = statsFor(color.field, columns);
    if (stats && stats.distinct > CROWDED_LEGEND_DISTINCT) {
      warnings.push({
        channel: 'color',
        message: `Colour has ${cardinalityText(stats)} categories — a legend that long is hard to scan and the palette will repeat hues. Group rarer categories, or map this field to the X axis instead.`,
      });
    }
  }

  // Negative-value Size guard (Draco hard.lp:56 size_negative): symbol size encodes
  // magnitude, so negatives render as zero/clipped area and mislead. The type gate
  // (isChannelTypeAllowed) can't catch this — it needs the data — so we flag it from
  // the profiled numeric extent.
  const size = config.encodings.size ?? null;
  if (size?.field) {
    const stats = statsFor(size.field, columns);
    if (stats?.numericExtent && stats.numericExtent.min < 0) {
      warnings.push({
        channel: 'size',
        message: `Size can't show negative values, but "${size.field}" goes down to ${stats.numericExtent.min}. Negative magnitudes render as tiny or clipped symbols — encode this field with Colour (a diverging scale) instead, or filter to non-negative values.`,
      });
    }
  }

  return warnings;
}

// --- Top-level data transforms: filters + calculated fields (spec §06 → Data). ----

/**
 * The comparison operators a field of this type admits (spec §06 → Data filters).
 * A measure or a temporal field can be ordered and ranged (`lt`…`gte`, `range`);
 * a category (nominal/ordinal) offers equality and membership (`oneOf`) only —
 * ordering categories by value isn't meaningful. `equal`/`notEqual` apply to every
 * type. Mirrors Vega-Lite's field-predicate grammar (Voyager's guarded filter).
 */
export function validFilterOps(fieldType: FieldType): FilterOp[] {
  if (fieldType === 'quantitative' || fieldType === 'temporal') {
    return ['equal', 'notEqual', 'lt', 'lte', 'gt', 'gte', 'range'];
  }
  return ['equal', 'notEqual', 'oneOf'];
}

/** `range` reads two bounds; `oneOf` a membership list; the rest a single value. */
export function filterOpArity(op: FilterOp): 'single' | 'range' | 'list' {
  if (op === 'range') return 'range';
  if (op === 'oneOf') return 'list';
  return 'single';
}

/**
 * Coerce a filter's text value to the type Vega-Lite compares against: a
 * quantitative field compares as a number (so `> 10` orders numerically, not
 * lexically); every other type compares as the raw string. ISO date strings sort
 * lexically == chronologically, so temporal predicates work as strings without a
 * date parse; non-ISO dates are the expression-mode case. A non-numeric string on a
 * quantitative field falls back to the string (Vega-Lite then compares loosely).
 */
function coerceFilterValue(value: string, fieldType: FieldType): string | number {
  if (fieldType === 'quantitative') {
    const n = Number(value);
    return value.trim() !== '' && Number.isFinite(n) ? n : value;
  }
  return value;
}

/**
 * The Vega-Lite field predicate for a guarded filter, or `null` when it is still
 * incomplete (no field/op, or a value the operator needs is blank) — an incomplete
 * filter is skipped so the preview keeps rendering. `notEqual` wraps an `equal`
 * predicate in a `{ not }` (Vega-Lite has no bare inequality predicate).
 */
function predicateObject(filter: BuilderFilter): Record<string, unknown> | null {
  const { op } = filter;
  if (!filter.field || !op) return null;
  // Escape `.`/`[`/`]` so a column literally named e.g. `user.age` is read as that
  // field, not a nested-property accessor (same convention as encodingObject and the
  // renderer — docs/architecture/05 §4).
  const field = escapeVegaField(filter.field);
  const type = filter.fieldType ?? 'nominal';
  const value = filter.value ?? '';
  const coerce = (v: string) => coerceFilterValue(v, type);

  switch (op) {
    case 'equal':
      return value === '' ? null : { field, equal: coerce(value) };
    case 'notEqual':
      return value === '' ? null : { not: { field, equal: coerce(value) } };
    case 'lt':
      return value === '' ? null : { field, lt: coerce(value) };
    case 'lte':
      return value === '' ? null : { field, lte: coerce(value) };
    case 'gt':
      return value === '' ? null : { field, gt: coerce(value) };
    case 'gte':
      return value === '' ? null : { field, gte: coerce(value) };
    case 'range': {
      const upper = filter.value2 ?? '';
      if (value === '' || upper === '') return null;
      return { field, range: [coerce(value), coerce(upper)] };
    }
    case 'oneOf': {
      const items = value
        .split(',')
        .map((s) => s.trim())
        .filter((s) => s !== '');
      return items.length === 0 ? null : { field, oneOf: items.map(coerce) };
    }
    default:
      return null;
  }
}

/** One filter's `{ filter }` transform entry, or `null` when incomplete. */
function filterTransformObject(filter: BuilderFilter): Record<string, unknown> | null {
  if (filter.mode === 'expression') {
    const expr = (filter.expr ?? '').trim();
    // A syntactically-invalid expression is dropped like an empty one: a half-typed
    // `datum.x *` must not reach the renderer and blank the preview mid-edit — the
    // inline feedback already flags it. (Same mid-edit resilience as a partial predicate.)
    return expr === '' || !validateExpression(expr).valid ? null : { filter: expr };
  }
  const predicate = predicateObject(filter);
  return predicate ? { filter: predicate } : null;
}

/** One calculate's `{ calculate, as }` transform entry, or `null` when incomplete or unparseable. */
function calculateTransformObject(calc: BuilderCalculate): Record<string, unknown> | null {
  const expr = calc.expr.trim();
  const as = calc.as.trim();
  return expr === '' || as === '' || !validateExpression(expr).valid
    ? null
    : { calculate: expr, as };
}

/**
 * The complete top-level `transform` array for a configuration: every calculated
 * field first (so filters and encodings can reference the derived columns), then
 * every filter, each in the user's list order. Incomplete or unparseable entries (a
 * half-typed predicate, an unnamed calculate, an expression that doesn't parse) are
 * dropped so a configuration mid-edit still produces a renderable spec. An empty
 * result means no `transform` key is emitted.
 */
export function buildTransforms(config: BuilderConfig): Array<Record<string, unknown>> {
  const out: Array<Record<string, unknown>> = [];
  for (const calc of config.calculates ?? []) {
    const t = calculateTransformObject(calc);
    if (t) out.push(t);
  }
  for (const filter of config.filters ?? []) {
    const t = filterTransformObject(filter);
    if (t) out.push(t);
  }
  return out;
}

/** The named (`as`) derived fields a config defines, in order, ignoring unnamed ones. */
export function calculatedFieldNames(
  calculates: readonly BuilderCalculate[] | undefined,
): string[] {
  return (calculates ?? []).map((c) => c.as.trim()).filter((as) => as !== '');
}

/**
 * The dataset's columns augmented with the config's calculated fields, so the UI's
 * column dropdowns and field-type logic treat a derived field like any other. A
 * calculated field's inferred type is unknown, so it defaults to **number**
 * (quantitative — the common arithmetic case; the user can retype it within the
 * valid set on the channel). Calculated names that collide with a real column are
 * skipped (the real column wins). No cardinality stats are derived for them.
 */
export function effectiveColumns(
  base: BuilderColumns,
  calculates: readonly BuilderCalculate[] | undefined,
): BuilderColumns {
  const added = calculatedFieldNames(calculates).filter(
    (name, i, all) => !base.columns.includes(name) && all.indexOf(name) === i,
  );
  if (added.length === 0) return base;
  return {
    columns: [...base.columns, ...added],
    columnTypes: [
      ...base.columnTypes,
      ...added.map((name): { name: string; type: ColumnType } => ({ name, type: 'number' })),
    ],
    columnStats: base.columnStats,
  };
}

/**
 * Clear any channel whose mapped column no longer exists among the effective
 * columns — the cleanup after a calculated field is removed or renamed, so the
 * produced spec never encodes a dangling field (which Vega-Lite would render empty).
 * Returns the same config object when nothing changed (stable for React equality).
 */
export function pruneEncodings(config: BuilderConfig, base: BuilderColumns): BuilderConfig {
  const available = new Set(effectiveColumns(base, config.calculates).columns);
  let changed = false;
  const encodings = { ...config.encodings };
  for (const channel of CHANNELS) {
    const mapping = encodings[channel];
    if (mapping && mapping.field !== undefined && !available.has(mapping.field)) {
      encodings[channel] = null;
      changed = true;
    }
  }
  return changed ? { ...config, encodings } : config;
}

/** A built Vega-Lite spec, as a plain object (serialize with `buildSnippetSpecText`). */
export type ChartSpec = Record<string, unknown>;

/** Build one channel's Vega-Lite encoding object from its mapping + transforms. */
function encodingObject(mapping: ChannelMapping): Record<string, unknown> {
  // A field-less count: `{ aggregate: 'count', type: 'quantitative' }`.
  if (mapping.aggregate === 'count') {
    return { aggregate: 'count', type: 'quantitative' };
  }
  const enc: Record<string, unknown> = {};
  // Escape `.`/`[`/`]` so a column literally named e.g. `user.age` is read as that
  // field, not a nested-property accessor (docs/architecture/05 §4).
  if (mapping.field !== undefined) enc.field = escapeVegaField(mapping.field);
  enc.type = mapping.type;
  if (mapping.aggregate) enc.aggregate = mapping.aggregate;
  if (mapping.bin) enc.bin = true;
  if (mapping.timeUnit) enc.timeUnit = mapping.timeUnit;
  return enc;
}

/**
 * Assemble the complete Vega-Lite spec from a builder configuration (spec §06 →
 * Output). Includes the schema reference, a named data reference to the dataset,
 * any top-level `transform` (calculated fields then row filters), the mark with
 * tooltips enabled, every mapped encoding (field, type, and any aggregate/bin/
 * timeUnit transform), chart-level sort (rank a categorical axis by its measure)
 * and stack (part-to-whole), and any explicit width/height. Unmapped
 * channels are omitted; if nothing is mapped the `encoding` block is omitted
 * entirely (validation prevents saving that, but the live preview may render a bare
 * mark while the user is still configuring).
 */
export function buildChartSpec(config: BuilderConfig): ChartSpec {
  const spec: ChartSpec = {
    $schema: VEGA_LITE_SCHEMA_URL,
    data: { name: config.datasetName },
  };

  // Top-level transforms (calculated fields, then filters) sit between data and
  // mark — applied to the raw rows before encoding aggregation.
  const transform = buildTransforms(config);
  if (transform.length > 0) spec.transform = transform;

  spec.mark = { type: config.mark, tooltip: true };

  const encoding: Record<string, Record<string, unknown>> = {};
  for (const [channel, mapping] of mappedChannels(config)) {
    encoding[channel] = encodingObject(mapping);
  }

  // Sort: the categorical positional axis sorts by the value of the measure axis
  // ("-y" descending, "y" ascending) — the conventional Vega-Lite ranking idiom.
  if (config.sort) {
    const category = sortableCategoryChannel(config);
    if (category && encoding[category]) {
      const measure = category === 'x' ? 'y' : 'x';
      encoding[category].sort = config.sort === 'descending' ? `-${measure}` : measure;
    }
  }

  // Stack: part-to-whole on the quantitative positional axis of a bar/area + colour.
  if (config.stack && supportsStack(config)) {
    const measure = stackMeasureChannel(config);
    if (measure && encoding[measure]) encoding[measure].stack = config.stack;
  }

  if (Object.keys(encoding).length > 0) spec.encoding = encoding;

  if (config.width !== undefined) spec.width = config.width;
  if (config.height !== undefined) spec.height = config.height;

  return spec;
}

/** The built spec as pretty-printed JSON text, ready for a snippet's `spec`. */
export function buildSnippetSpecText(config: BuilderConfig): string {
  return JSON.stringify(buildChartSpec(config), null, 2);
}

/** Title-case a single mark type for display/naming (e.g. `bar` → `Bar`). */
function markLabel(mark: MarkType): string {
  return mark.charAt(0).toUpperCase() + mark.slice(1);
}

/** A human phrase for what a channel encodes, e.g. "sum of revenue", "count". */
function describeMapping(mapping: ChannelMapping): string {
  if (mapping.aggregate === 'count') return 'count';
  const field = mapping.field ?? '';
  if (mapping.aggregate) return `${mapping.aggregate} of ${field}`;
  return field;
}

/**
 * An auto-generated, descriptive name for the created snippet (spec §06 → Output:
 * "an auto-generated descriptive name"). When both X and Y are mapped it reads
 * "Bar chart of <y> by <x>" (using each channel's measure phrase, e.g. "count" or
 * "sum of revenue"); otherwise it falls back to naming the dataset. Deterministic —
 * no timestamp — so the name describes the chart, not when it was made.
 */
export function generateChartName(config: BuilderConfig): string {
  const mark = markLabel(config.mark);
  const x = config.encodings.x;
  const y = config.encodings.y;
  if (x && y) return `${mark} chart of ${describeMapping(y)} by ${describeMapping(x)}`;
  const only = mappedChannels(config)[0];
  if (only) return `${mark} chart of ${describeMapping(only[1])}`;
  return `${mark} chart of ${config.datasetName}`;
}