Chart builder: filter/calculate transforms, data preview, and inline expression validation

src/core/chart-builder.ts

@@ -73,6 +73,82 @@ export type SortOrder = (typeof SORT_ORDERS)[number];
 export const STACK_MODES = ['zero', 'normalize'] as const;
 export type StackMode = (typeof STACK_MODES)[number];
 
+/**
+ * Comparison operators a guarded filter predicate offers (Vega-Lite field
+ * predicates), in menu order. The set a given field admits is narrowed by
+ * `validFilterOps` — only a measure/temporal field offers ordering (`lt`…`gt`) and
+ * `range`; a category offers membership (`oneOf`). `notEqual` is expressed as a
+ * `{ not: { …equal } }` logical wrapper (Vega-Lite has no bare `!=` predicate).
+ */
+export const FILTER_OPS = [
+  'equal',
+  'notEqual',
+  'lt',
+  'lte',
+  'gt',
+  'gte',
+  'range',
+  'oneOf',
+] as const;
+export type FilterOp = (typeof FILTER_OPS)[number];
+
+/** How a filter expresses its predicate: a guarded shelf, or a raw expression. */
+export type FilterMode = 'predicate' | 'expression';
+
+/**
+ * One top-level data filter (Vega-Lite `transform: [{ filter }]`), applied to the
+ * raw rows **before** encoding aggregation — so "filter rows, then aggregate" is the
+ * natural reading. Two modes share one shape (the UI toggles between them on a single
+ * row):
+ *
+ *   - **predicate** — a guarded `field op value` triple (Voyager-style, no expression
+ *     to write for the common case). `range` carries a second bound (`value2`);
+ *     `oneOf` reads `value` as a comma-separated membership list. Values are coerced
+ *     by `fieldType` (a quantitative field compares as a number).
+ *   - **expression** — a raw Vega expression string (`datum.x > 0`), the power-form
+ *     for what the guarded shelf can't say (validated in the UI via
+ *     `expr-validate.ts`).
+ *
+ * `id` is a stable key for list editing only; it never reaches the produced spec.
+ * A partially-filled filter (no value yet, empty expression) is simply skipped by
+ * the assembler, so the live preview keeps rendering while the user types.
+ */
+export interface BuilderFilter {
+  /** Stable list key (UI-assigned); not serialized into the spec. */
+  id: string;
+  /** Which form this filter takes. */
+  mode: FilterMode;
+  /** Predicate mode: the column being filtered (a dataset or calculated field). */
+  field?: string;
+  /** Predicate mode: the field's Vega-Lite type — drives the op menu + coercion. */
+  fieldType?: FieldType;
+  /** Predicate mode: the comparison operator. */
+  op?: FilterOp;
+  /** Predicate mode: the comparison value (`range` lower bound; `oneOf` CSV list). */
+  value?: string;
+  /** Predicate mode: the upper bound, for `range` only. */
+  value2?: string;
+  /** Expression mode: a raw Vega predicate expression. */
+  expr?: string;
+}
+
+/**
+ * One derived field (Vega-Lite `transform: [{ calculate, as }]`): a row-wise Vega
+ * expression producing a new column the rest of the builder can encode like any
+ * other. Calculates are emitted **before** filters (a row-wise calculate is
+ * order-independent — it computes the same per-row value regardless of which rows
+ * survive — so running it first is equivalent and lets a filter reference the
+ * derived field). `id` is a UI list key only.
+ */
+export interface BuilderCalculate {
+  /** Stable list key (UI-assigned); not serialized into the spec. */
+  id: string;
+  /** The Vega expression, e.g. `datum.price * datum.quantity`. */
+  expr: string;
+  /** The new field's name (appears in the channel dropdowns once non-empty). */
+  as: string;
+}
+
 /**
  * One channel's mapping: a dataset column `field` plus its `type`, with optional
  * transforms. `field` is omitted only for a `count` aggregate (which counts records
@@ -108,6 +184,10 @@ export interface BuilderConfig {
   sort?: SortOrder;
   /** Stacking for bar/area + a colour series (part-to-whole). */
   stack?: StackMode;
+  /** Derived fields (`transform: [{ calculate, as }]`), emitted before `filters`. */
+  calculates?: BuilderCalculate[];
+  /** Row filters (`transform: [{ filter }]`), applied before encoding aggregation. */
+  filters?: BuilderFilter[];
 }
 
 /**
@@ -665,6 +745,182 @@ export function builderWarnings(
   return warnings;
 }
 
+// --- Top-level data transforms: filters + calculated fields (spec §06 → Data). ----
+
+/**
+ * The comparison operators a field of this type admits (spec §06 → Data filters).
+ * A measure or a temporal field can be ordered and ranged (`lt`…`gte`, `range`);
+ * a category (nominal/ordinal) offers equality and membership (`oneOf`) only —
+ * ordering categories by value isn't meaningful. `equal`/`notEqual` apply to every
+ * type. Mirrors Vega-Lite's field-predicate grammar (Voyager's guarded filter).
+ */
+export function validFilterOps(fieldType: FieldType): FilterOp[] {
+  if (fieldType === 'quantitative' || fieldType === 'temporal') {
+    return ['equal', 'notEqual', 'lt', 'lte', 'gt', 'gte', 'range'];
+  }
+  return ['equal', 'notEqual', 'oneOf'];
+}
+
+/** `range` reads two bounds; `oneOf` a membership list; the rest a single value. */
+export function filterOpArity(op: FilterOp): 'single' | 'range' | 'list' {
+  if (op === 'range') return 'range';
+  if (op === 'oneOf') return 'list';
+  return 'single';
+}
+
+/**
+ * Coerce a filter's text value to the type Vega-Lite compares against: a
+ * quantitative field compares as a number (so `> 10` orders numerically, not
+ * lexically); every other type compares as the raw string. ISO date strings sort
+ * lexically == chronologically, so temporal predicates work as strings without a
+ * date parse; non-ISO dates are the expression-mode case. A non-numeric string on a
+ * quantitative field falls back to the string (Vega-Lite then compares loosely).
+ */
+function coerceFilterValue(value: string, fieldType: FieldType): string | number {
+  if (fieldType === 'quantitative') {
+    const n = Number(value);
+    return value.trim() !== '' && Number.isFinite(n) ? n : value;
+  }
+  return value;
+}
+
+/**
+ * The Vega-Lite field predicate for a guarded filter, or `null` when it is still
+ * incomplete (no field/op, or a value the operator needs is blank) — an incomplete
+ * filter is skipped so the preview keeps rendering. `notEqual` wraps an `equal`
+ * predicate in a `{ not }` (Vega-Lite has no bare inequality predicate).
+ */
+function predicateObject(filter: BuilderFilter): Record<string, unknown> | null {
+  const { field, op } = filter;
+  if (!field || !op) return null;
+  // TODO: the data-derived column name reaches `field:` unescaped here, as it also
+  // does in encodingObject. A column whose name contains `.`/`[`/`]` is then read as
+  // a nested-property accessor rather than a literal field. core/rendering.ts exports
+  // escapeVegaField for exactly this but nothing currently wires it into the builder.
+  const type = filter.fieldType ?? 'nominal';
+  const value = filter.value ?? '';
+  const coerce = (v: string) => coerceFilterValue(v, type);
+
+  switch (op) {
+    case 'equal':
+      return value === '' ? null : { field, equal: coerce(value) };
+    case 'notEqual':
+      return value === '' ? null : { not: { field, equal: coerce(value) } };
+    case 'lt':
+      return value === '' ? null : { field, lt: coerce(value) };
+    case 'lte':
+      return value === '' ? null : { field, lte: coerce(value) };
+    case 'gt':
+      return value === '' ? null : { field, gt: coerce(value) };
+    case 'gte':
+      return value === '' ? null : { field, gte: coerce(value) };
+    case 'range': {
+      const upper = filter.value2 ?? '';
+      if (value === '' || upper === '') return null;
+      return { field, range: [coerce(value), coerce(upper)] };
+    }
+    case 'oneOf': {
+      const items = value
+        .split(',')
+        .map((s) => s.trim())
+        .filter((s) => s !== '');
+      return items.length === 0 ? null : { field, oneOf: items.map(coerce) };
+    }
+    default:
+      return null;
+  }
+}
+
+/** One filter's `{ filter }` transform entry, or `null` when incomplete. */
+function filterTransformObject(filter: BuilderFilter): Record<string, unknown> | null {
+  if (filter.mode === 'expression') {
+    const expr = (filter.expr ?? '').trim();
+    return expr === '' ? null : { filter: expr };
+  }
+  const predicate = predicateObject(filter);
+  return predicate ? { filter: predicate } : null;
+}
+
+/** One calculate's `{ calculate, as }` transform entry, or `null` when incomplete. */
+function calculateTransformObject(calc: BuilderCalculate): Record<string, unknown> | null {
+  const expr = calc.expr.trim();
+  const as = calc.as.trim();
+  return expr === '' || as === '' ? null : { calculate: expr, as };
+}
+
+/**
+ * The complete top-level `transform` array for a configuration: every calculated
+ * field first (so filters and encodings can reference the derived columns), then
+ * every filter, each in the user's list order. Incomplete entries (a half-typed
+ * filter, an unnamed calculate) are dropped so a configuration mid-edit still
+ * produces a renderable spec. An empty result means no `transform` key is emitted.
+ */
+export function buildTransforms(config: BuilderConfig): Array<Record<string, unknown>> {
+  const out: Array<Record<string, unknown>> = [];
+  for (const calc of config.calculates ?? []) {
+    const t = calculateTransformObject(calc);
+    if (t) out.push(t);
+  }
+  for (const filter of config.filters ?? []) {
+    const t = filterTransformObject(filter);
+    if (t) out.push(t);
+  }
+  return out;
+}
+
+/** The named (`as`) derived fields a config defines, in order, ignoring unnamed ones. */
+export function calculatedFieldNames(
+  calculates: readonly BuilderCalculate[] | undefined,
+): string[] {
+  return (calculates ?? []).map((c) => c.as.trim()).filter((as) => as !== '');
+}
+
+/**
+ * The dataset's columns augmented with the config's calculated fields, so the UI's
+ * column dropdowns and field-type logic treat a derived field like any other. A
+ * calculated field's inferred type is unknown, so it defaults to **number**
+ * (quantitative — the common arithmetic case; the user can retype it within the
+ * valid set on the channel). Calculated names that collide with a real column are
+ * skipped (the real column wins). No cardinality stats are derived for them.
+ */
+export function effectiveColumns(
+  base: BuilderColumns,
+  calculates: readonly BuilderCalculate[] | undefined,
+): BuilderColumns {
+  const added = calculatedFieldNames(calculates).filter(
+    (name, i, all) => !base.columns.includes(name) && all.indexOf(name) === i,
+  );
+  if (added.length === 0) return base;
+  return {
+    columns: [...base.columns, ...added],
+    columnTypes: [
+      ...base.columnTypes,
+      ...added.map((name): { name: string; type: ColumnType } => ({ name, type: 'number' })),
+    ],
+    columnStats: base.columnStats,
+  };
+}
+
+/**
+ * Clear any channel whose mapped column no longer exists among the effective
+ * columns — the cleanup after a calculated field is removed or renamed, so the
+ * produced spec never encodes a dangling field (which Vega-Lite would render empty).
+ * Returns the same config object when nothing changed (stable for React equality).
+ */
+export function pruneEncodings(config: BuilderConfig, base: BuilderColumns): BuilderConfig {
+  const available = new Set(effectiveColumns(base, config.calculates).columns);
+  let changed = false;
+  const encodings = { ...config.encodings };
+  for (const channel of CHANNELS) {
+    const mapping = encodings[channel];
+    if (mapping && mapping.field !== undefined && !available.has(mapping.field)) {
+      encodings[channel] = null;
+      changed = true;
+    }
+  }
+  return changed ? { ...config, encodings } : config;
+}
+
 /** A built Vega-Lite spec, as a plain object (serialize with `buildSnippetSpecText`). */
 export type ChartSpec = Record<string, unknown>;
 
@@ -686,9 +942,10 @@ function encodingObject(mapping: ChannelMapping): Record<string, unknown> {
 /**
  * Assemble the complete Vega-Lite spec from a builder configuration (spec §06 →
  * Output). Includes the schema reference, a named data reference to the dataset,
- * the mark with tooltips enabled, every mapped encoding (field, type, and any
- * aggregate/bin/timeUnit transform), chart-level sort (rank a categorical axis by
- * its measure) and stack (part-to-whole), and any explicit width/height. Unmapped
+ * any top-level `transform` (calculated fields then row filters), the mark with
+ * tooltips enabled, every mapped encoding (field, type, and any aggregate/bin/
+ * timeUnit transform), chart-level sort (rank a categorical axis by its measure)
+ * and stack (part-to-whole), and any explicit width/height. Unmapped
  * channels are omitted; if nothing is mapped the `encoding` block is omitted
  * entirely (validation prevents saving that, but the live preview may render a bare
  * mark while the user is still configuring).
@@ -697,9 +954,15 @@ export function buildChartSpec(config: BuilderConfig): ChartSpec {
   const spec: ChartSpec = {
     $schema: VEGA_LITE_SCHEMA_URL,
     data: { name: config.datasetName },
-    mark: { type: config.mark, tooltip: true },
   };
 
+  // Top-level transforms (calculated fields, then filters) sit between data and
+  // mark — applied to the raw rows before encoding aggregation.
+  const transform = buildTransforms(config);
+  if (transform.length > 0) spec.transform = transform;
+
+  spec.mark = { type: config.mark, tooltip: true };
+
   const encoding: Record<string, Record<string, unknown>> = {};
   for (const [channel, mapping] of mappedChannels(config)) {
     encoding[channel] = encodingObject(mapping);