Chart builder: filter/calculate transforms, data preview, and inline expression validation

src/app/components/ChartBuilderModal.tsx

@@ -24,6 +24,8 @@ import {
   TIME_UNITS,
   builderWarnings,
   defaultFieldType,
+  effectiveColumns,
+  filterOpArity,
   isBuilderConfigValid,
   isChannelTypeAllowed,
   supportsAggregate,
@@ -32,14 +34,21 @@ import {
   supportsStack,
   supportsTimeUnit,
   validFieldTypes,
+  validFilterOps,
+  type BuilderCalculate,
   type AggregateOp,
+  type BuilderColumns,
+  type BuilderFilter,
   type BuilderWarningFix,
   type ChannelMapping,
   type ChannelName,
   type FieldType,
+  type FilterOp,
   type MarkType,
   type TimeUnit,
 } from '@core/chart-builder';
+import { referencedFields, validateExpression } from '@core/expr-validate';
+import { tabularRows } from '@core/dataset';
 import type { ColumnType } from '@core/type-inference';
 import { DatasetNotFoundError, prepareSpecForRender } from '@core/rendering';
 import { chartConfigFor } from '@core/vega-themes';
@@ -131,6 +140,57 @@ const TIME_UNIT_LABELS: Record<TimeUnit, string> = {
   hours: 'Hour',
 };
 
+/** Readable labels for the filter operators, phrased to read as "<field> <op> <value>". */
+const FILTER_OP_LABELS: Record<FilterOp, string> = {
+  equal: 'is',
+  notEqual: 'is not',
+  lt: '<',
+  lte: '≤',
+  gt: '>',
+  gte: '≥',
+  range: 'is between',
+  oneOf: 'is one of',
+};
+
+/** Rows shown in the builder's data-preview table before truncating (1D, spec §06). */
+const PREVIEW_ROW_LIMIT = 50;
+
+/**
+ * The Vega expression-language reference — both expression inputs (a filter's
+ * expression mode, a calculated field) compile to a raw Vega expression, so this is
+ * the precise vocabulary. Surfaced contextually (only when an expression is in play),
+ * external so it falls outside offline scope, opened in a new tab.
+ */
+const VEGA_EXPRESSION_DOCS_URL = 'https://vega.github.io/vega/docs/expressions/';
+
+/** One preview cell's text: blank for empty, the string as-is, else JSON. */
+function cellText(value: unknown): string {
+  if (value == null) return '';
+  if (typeof value === 'string') return value;
+  return JSON.stringify(value);
+}
+
+/** A safe `datum` accessor for a column name (dot for identifiers, bracket otherwise). */
+function datumRef(name: string): string {
+  return /^[A-Za-z_$][\w$]*$/.test(name) ? `datum.${name}` : `datum[${JSON.stringify(name)}]`;
+}
+
+/**
+ * An example expression seeded as the input placeholder, drawn from the dataset's own
+ * columns — discovery for writing Vega expressions without an autocomplete popup. A
+ * filter example reads as a predicate; a calculate example as a derived value.
+ */
+function exprPlaceholder(columns: BuilderColumns, kind: 'filter' | 'calc'): string {
+  const numeric = columns.columnTypes.find((c) => c.type === 'number')?.name;
+  const anyField = columns.columns[0];
+  if (kind === 'filter') {
+    if (numeric) return `${datumRef(numeric)} > 0`;
+    return anyField ? `${datumRef(anyField)} != null` : 'datum.value > 0';
+  }
+  if (numeric) return `${datumRef(numeric)} * 2`;
+  return anyField ? datumRef(anyField) : 'datum.a + datum.b';
+}
+
 /** A compact type indicator for a column option (# / date / bool / text). */
 function typeBadge(type: ColumnType): string {
   switch (type) {
@@ -156,7 +216,14 @@ function isCount(mapping: ChannelMapping | null): boolean {
 }
 
 function ChannelBlock({ channel }: { channel: ChannelName }) {
-  const columns = useChartBuilderStore((s) => s.columns);
+  const baseColumns = useChartBuilderStore((s) => s.columns);
+  const calculates = useChartBuilderStore((s) => s.config.calculates);
+  // Effective columns = the dataset's columns plus any calculated fields, so a derived
+  // field is selectable on a channel like any real column.
+  const columns = useMemo(
+    () => effectiveColumns(baseColumns, calculates),
+    [baseColumns, calculates],
+  );
   const mapping = useChartBuilderStore((s) => s.config.encodings[channel] ?? null);
   const setChannelColumn = useChartBuilderStore((s) => s.setChannelColumn);
   const setChannelType = useChartBuilderStore((s) => s.setChannelType);
@@ -277,6 +344,356 @@ function ChannelBlock({ channel }: { channel: ChannelName }) {
   );
 }
 
+/**
+ * Inline feedback for an expression input (filter expression / calculated field):
+ * a parse error (assertive) takes priority, else a soft warning for `datum.<field>`
+ * references that don't match a known column — a typo guard before the chart renders
+ * empty (1E). Nothing renders for a valid, fully-resolved expression.
+ */
+function ExprFeedback({ expr, columns }: { expr: string; columns: BuilderColumns }) {
+  const feedback = useMemo(() => {
+    const validation = validateExpression(expr);
+    if (!validation.valid) {
+      return { kind: 'error' as const, text: validation.error ?? 'Invalid expression.' };
+    }
+    const unknown = referencedFields(expr).filter((f) => !columns.columns.includes(f));
+    if (unknown.length > 0) {
+      const plural = unknown.length > 1 ? 's' : '';
+      return { kind: 'warn' as const, text: `Unknown field${plural}: ${unknown.join(', ')}` };
+    }
+    return null;
+  }, [expr, columns]);
+
+  if (!feedback) return null;
+  return feedback.kind === 'error' ? (
+    <p className={styles.exprError} role="alert">
+      {feedback.text}
+    </p>
+  ) : (
+    <p className={styles.exprWarn} role="status">
+      {feedback.text}
+    </p>
+  );
+}
+
+/**
+ * A contextual pointer to the Vega expression vocabulary, shown only when an
+ * expression input is in play (a calculated field, or a filter in expression mode) —
+ * the place the user needs to know what functions/operators exist.
+ */
+function ExprHelp() {
+  return (
+    <p className={styles.exprHelp}>
+      Filter expressions and calculated fields use the{' '}
+      <a
+        className={styles.exprHelpLink}
+        href={VEGA_EXPRESSION_DOCS_URL}
+        target="_blank"
+        rel="noreferrer"
+      >
+        Vega expression language
+        <span aria-hidden="true"> ↗</span>
+        <span className="visually-hidden"> (opens in a new tab)</span>
+      </a>
+      .
+    </p>
+  );
+}
+
+/** One filter row — a guarded `field op value` predicate, or a raw expression. */
+function FilterRow({ filter, columns }: { filter: BuilderFilter; columns: BuilderColumns }) {
+  const setFilterField = useChartBuilderStore((s) => s.setFilterField);
+  const updateFilter = useChartBuilderStore((s) => s.updateFilter);
+  const setFilterMode = useChartBuilderStore((s) => s.setFilterMode);
+  const removeFilter = useChartBuilderStore((s) => s.removeFilter);
+
+  const expressionMode = filter.mode === 'expression';
+  const fieldType = filter.fieldType ?? 'nominal';
+  const op = filter.op ?? 'equal';
+  const arity = filterOpArity(op);
+  const hasColumns = columns.columns.length > 0;
+
+  return (
+    <div className={styles.transformBlock}>
+      <div className={styles.transformBlockTop}>
+        {expressionMode ? (
+          <input
+            className={styles.exprInput}
+            aria-label="Filter expression"
+            placeholder={exprPlaceholder(columns, 'filter')}
+            value={filter.expr ?? ''}
+            aria-invalid={!validateExpression(filter.expr ?? '').valid || undefined}
+            onChange={(e) => updateFilter(filter.id, { expr: e.target.value })}
+          />
+        ) : (
+          <select
+            className={styles.filterField}
+            aria-label="Filter field"
+            value={filter.field ?? ''}
+            onChange={(e) => setFilterField(filter.id, e.target.value)}
+          >
+            {!filter.field && <option value="">Choose a field…</option>}
+            {columns.columns.map((name) => (
+              <option key={name} value={name}>
+                {name}
+              </option>
+            ))}
+          </select>
+        )}
+        <button
+          type="button"
+          className={styles.removeRow}
+          aria-label="Remove filter"
+          onClick={() => removeFilter(filter.id)}
+        >
+          <Icon name="close" />
+        </button>
+      </div>
+
+      {!expressionMode && (
+        <div className={styles.filterPredicate}>
+          <select
+            className={styles.mini}
+            aria-label="Filter operator"
+            value={op}
+            onChange={(e) => updateFilter(filter.id, { op: e.target.value as FilterOp })}
+          >
+            {validFilterOps(fieldType).map((o) => (
+              <option key={o} value={o}>
+                {FILTER_OP_LABELS[o]}
+              </option>
+            ))}
+          </select>
+          {arity === 'range' ? (
+            <>
+              <input
+                className={styles.valueInput}
+                aria-label="Lower bound"
+                placeholder="min"
+                value={filter.value ?? ''}
+                onChange={(e) => updateFilter(filter.id, { value: e.target.value })}
+              />
+              <span className={styles.rangeDash} aria-hidden="true">
+                –
+              </span>
+              <input
+                className={styles.valueInput}
+                aria-label="Upper bound"
+                placeholder="max"
+                value={filter.value2 ?? ''}
+                onChange={(e) => updateFilter(filter.id, { value2: e.target.value })}
+              />
+            </>
+          ) : (
+            <input
+              className={styles.valueInput}
+              aria-label={arity === 'list' ? 'Values, comma-separated' : 'Filter value'}
+              placeholder={arity === 'list' ? 'A, B, C' : 'value'}
+              value={filter.value ?? ''}
+              onChange={(e) => updateFilter(filter.id, { value: e.target.value })}
+            />
+          )}
+        </div>
+      )}
+
+      {expressionMode && <ExprFeedback expr={filter.expr ?? ''} columns={columns} />}
+
+      {hasColumns && (
+        <button
+          type="button"
+          className={styles.modeToggle}
+          onClick={() => setFilterMode(filter.id, expressionMode ? 'predicate' : 'expression')}
+        >
+          {expressionMode ? 'Use the field picker' : 'Write an expression'}
+        </button>
+      )}
+    </div>
+  );
+}
+
+/** One calculated-field row — a name and a Vega expression producing a new column. */
+function CalculateRow({ calc, columns }: { calc: BuilderCalculate; columns: BuilderColumns }) {
+  const updateCalculate = useChartBuilderStore((s) => s.updateCalculate);
+  const removeCalculate = useChartBuilderStore((s) => s.removeCalculate);
+
+  return (
+    <div className={styles.transformBlock}>
+      <div className={styles.transformBlockTop}>
+        <input
+          className={styles.calcName}
+          aria-label="New field name"
+          placeholder="new field"
+          value={calc.as}
+          onChange={(e) => updateCalculate(calc.id, { as: e.target.value })}
+        />
+        <span className={styles.calcEquals} aria-hidden="true">
+          =
+        </span>
+        <input
+          className={styles.exprInput}
+          aria-label="Calculated field expression"
+          placeholder={exprPlaceholder(columns, 'calc')}
+          value={calc.expr}
+          aria-invalid={!validateExpression(calc.expr).valid || undefined}
+          onChange={(e) => updateCalculate(calc.id, { expr: e.target.value })}
+        />
+        <button
+          type="button"
+          className={styles.removeRow}
+          aria-label="Remove calculated field"
+          onClick={() => removeCalculate(calc.id)}
+        >
+          <Icon name="close" />
+        </button>
+      </div>
+      <ExprFeedback expr={calc.expr} columns={columns} />
+    </div>
+  );
+}
+
+/**
+ * A collapsible, read-only sample of the dataset's actual rows with a per-column type
+ * chip in each header (1D) — lets the user sanity-check inferred types before building,
+ * exactly when inference is most likely to surprise. Shows the base dataset columns
+ * (calculated fields don't exist in the raw rows). Non-tabular payloads have no rows.
+ */
+function DataPreview() {
+  const datasetId = useChartBuilderStore((s) => s.datasetId);
+  const baseColumns = useChartBuilderStore((s) => s.columns);
+  const dataset = useDatasetStore((s) => s.datasets.find((d) => d.id === datasetId) ?? null);
+  const [open, setOpen] = useState(false);
+
+  const rows = useMemo(
+    () => (dataset ? tabularRows(dataset.data, dataset.format, PREVIEW_ROW_LIMIT) : null),
+    [dataset],
+  );
+  if (!dataset) return null;
+
+  const typeOf = (name: string): ColumnType =>
+    baseColumns.columnTypes.find((c) => c.name === name)?.type ?? 'string';
+
+  return (
+    <div className={styles.dataPreview}>
+      <button
+        type="button"
+        className={styles.previewToggle}
+        aria-expanded={open}
+        onClick={() => setOpen((o) => !o)}
+      >
+        <span className={styles.previewCaret} aria-hidden="true">
+          {open ? '▾' : '▸'}
+        </span>
+        Preview rows
+        {dataset.rowCount != null && (
+          <span className={styles.previewMeta}>
+            {dataset.rowCount.toLocaleString()} rows · {dataset.columnCount} cols
+          </span>
+        )}
+      </button>
+
+      {open &&
+        (rows ? (
+          <div
+            className={styles.previewTableWrap}
+            tabIndex={0}
+            role="group"
+            aria-label="Data preview"
+          >
+            <table className={styles.previewTable}>
+              <thead>
+                <tr>
+                  {baseColumns.columns.map((col) => (
+                    <th key={col} scope="col">
+                      <span className={styles.previewColName}>{col}</span>{' '}
+                      <span className={styles.previewColType}>{typeBadge(typeOf(col))}</span>
+                    </th>
+                  ))}
+                </tr>
+              </thead>
+              <tbody>
+                {rows.map((row, ri) => (
+                  <tr key={ri}>
+                    {baseColumns.columns.map((col) => (
+                      <td key={col}>{cellText(row[col])}</td>
+                    ))}
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        ) : (
+          <p className={styles.previewEmptyNote}>This dataset has no tabular rows to preview.</p>
+        ))}
+
+      {open && rows && dataset.rowCount != null && dataset.rowCount > rows.length && (
+        <p className={styles.previewEmptyNote}>
+          Showing the first {rows.length} of {dataset.rowCount.toLocaleString()} rows.
+        </p>
+      )}
+    </div>
+  );
+}
+
+/**
+ * The "Data" section at the top of the config pane: row filters and calculated fields
+ * (the top-level transforms, 1C) plus a collapsible row preview (1D) — "here are your
+ * rows, shape them, then encode them". Calculated fields appear in the channel
+ * dropdowns below via the effective-columns derivation.
+ */
+function DataSection() {
+  const baseColumns = useChartBuilderStore((s) => s.columns);
+  const calculates = useChartBuilderStore((s) => s.config.calculates);
+  const filters = useChartBuilderStore((s) => s.config.filters);
+  const addFilter = useChartBuilderStore((s) => s.addFilter);
+  const addCalculate = useChartBuilderStore((s) => s.addCalculate);
+
+  const columns = useMemo(
+    () => effectiveColumns(baseColumns, calculates),
+    [baseColumns, calculates],
+  );
+
+  // The expression reference is shown only when an expression input exists — a
+  // calculated field, or a filter switched to expression mode.
+  const hasExpression =
+    (filters ?? []).some((f) => f.mode === 'expression') || (calculates ?? []).length > 0;
+
+  return (
+    <section className={styles.dataSection} aria-label="Data">
+      <span className={styles.fieldLabel}>Data</span>
+
+      {/* The source rows come first, so they read as the input — distinct from the
+          filters/calculated fields below, which shape what the chart actually draws. */}
+      <DataPreview />
+
+      <div className={styles.transformGroup}>
+        <div className={styles.transformGroupHead}>
+          <span className={styles.miniLabel}>Filters</span>
+          <button type="button" className={styles.addRow} onClick={addFilter}>
+            <Icon name="add" /> Add filter
+          </button>
+        </div>
+        {(filters ?? []).map((f) => (
+          <FilterRow key={f.id} filter={f} columns={columns} />
+        ))}
+      </div>
+
+      <div className={styles.transformGroup}>
+        <div className={styles.transformGroupHead}>
+          <span className={styles.miniLabel}>Calculated fields</span>
+          <button type="button" className={styles.addRow} onClick={addCalculate}>
+            <Icon name="add" /> Add field
+          </button>
+        </div>
+        {(calculates ?? []).map((c) => (
+          <CalculateRow key={c.id} calc={c} columns={columns} />
+        ))}
+      </div>
+
+      {hasExpression && <ExprHelp />}
+    </section>
+  );
+}
+
 /** Sort control values: 'none' maps to an unsorted config. */
 const SORT_OPTIONS: ReadonlyArray<SegmentedOption<'none' | 'ascending' | 'descending'>> = [
   { value: 'none', label: 'None' },
@@ -488,6 +905,8 @@ export function ChartBuilderModal() {
           Building from <strong>{datasetName}</strong>
         </p>
 
+        <DataSection />
+
         <div className={styles.field}>
           <span className={styles.fieldLabel}>Mark</span>
           <SegmentedControl