mirror of
https://github.com/olehomelchenko/astrolabe.git
synced 2026-08-08 02:02:33 +00:00
Add dataset library, extract-to-dataset, and render-time reference resolution
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import {
|
||||
computeDatasetProfile,
|
||||
CURRENT_DATASET_VERSION,
|
||||
createDataset,
|
||||
datasetReference,
|
||||
parseDelimited,
|
||||
} from './dataset';
|
||||
|
||||
describe('datasetReference', () => {
|
||||
test('produces the by-name reference object', () => {
|
||||
expect(datasetReference('MyDataset')).toEqual({ data: { name: 'MyDataset' } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDelimited', () => {
|
||||
test('parses CSV header + rows into objects', () => {
|
||||
const rows = parseDelimited('a,b,c\n1,2,3\n4,5,6', 'csv');
|
||||
expect(rows).toEqual([
|
||||
{ a: '1', b: '2', c: '3' },
|
||||
{ a: '4', b: '5', c: '6' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parses TSV with the tab delimiter', () => {
|
||||
const rows = parseDelimited('x\ty\n1\t2', 'tsv');
|
||||
expect(rows).toEqual([{ x: '1', y: '2' }]);
|
||||
});
|
||||
|
||||
test('drops a trailing newline (no phantom empty row) and skips blank lines', () => {
|
||||
expect(parseDelimited('a,b\n1,2\n', 'csv')).toEqual([{ a: '1', b: '2' }]);
|
||||
expect(parseDelimited('a,b\n1,2\n\n3,4', 'csv')).toEqual([
|
||||
{ a: '1', b: '2' },
|
||||
{ a: '3', b: '4' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('strips surrounding quotes from header and cells (RFC-4180)', () => {
|
||||
// The quoted-TSV case from real exports: column keys must not keep the quotes.
|
||||
const rows = parseDelimited('"email"\t"id"\n"a@b.com"\t1', 'tsv');
|
||||
expect(rows).toEqual([{ email: 'a@b.com', id: '1' }]);
|
||||
});
|
||||
|
||||
test('quoted fields keep embedded delimiters, newlines, and escaped quotes', () => {
|
||||
const rows = parseDelimited('name,note\n"Doe, John","a ""quote""\nspans"', 'csv');
|
||||
expect(rows).toEqual([{ name: 'Doe, John', note: 'a "quote"\nspans' }]);
|
||||
});
|
||||
|
||||
test('does not trim unquoted whitespace (matches d3-dsv / Vega)', () => {
|
||||
expect(parseDelimited('a, b\n1, 2', 'csv')).toEqual([{ a: '1', ' b': ' 2' }]);
|
||||
});
|
||||
|
||||
test('ragged rows: missing cells are undefined, extra cells ignored', () => {
|
||||
const rows = parseDelimited('a,b,c\n1\n4,5,6,7', 'csv');
|
||||
expect(rows).toEqual([
|
||||
{ a: '1', b: undefined, c: undefined },
|
||||
{ a: '4', b: '5', c: '6' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('header-only or empty input yields no rows', () => {
|
||||
expect(parseDelimited('a,b,c', 'csv')).toEqual([]);
|
||||
expect(parseDelimited('', 'csv')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeDatasetProfile', () => {
|
||||
test('inline JSON array-of-objects is profiled', () => {
|
||||
const data = [
|
||||
{ a: 1, b: 'x' },
|
||||
{ a: 2, b: 'y' },
|
||||
];
|
||||
const profile = computeDatasetProfile(data, 'json', 'inline');
|
||||
expect(profile.rowCount).toBe(2);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(data)).length);
|
||||
});
|
||||
|
||||
test('inline JSON that is not an array is N/A but sized', () => {
|
||||
const profile = computeDatasetProfile({ a: 1 }, 'json', 'inline');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('inline CSV is parsed and profiled; size is the raw text byte length', () => {
|
||||
const text = 'a,b\n1,2';
|
||||
const profile = computeDatasetProfile(text, 'csv', 'inline');
|
||||
expect(profile.rowCount).toBe(1);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
expect(profile.size).toBe(new TextEncoder().encode(text).length);
|
||||
});
|
||||
|
||||
test('inline TSV is parsed and profiled', () => {
|
||||
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv', 'inline');
|
||||
expect(profile.rowCount).toBe(1);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('inline TopoJSON is N/A (non-tabular) but sized', () => {
|
||||
const topo = { type: 'Topology', objects: {} };
|
||||
const profile = computeDatasetProfile(topo, 'topojson', 'inline');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(topo)).length);
|
||||
});
|
||||
|
||||
test('URL is N/A; size is the byte length of the URL string', () => {
|
||||
const url = 'https://example.com/data.csv';
|
||||
const profile = computeDatasetProfile(url, 'csv', 'url');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.size).toBe(new TextEncoder().encode(url).length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createDataset', () => {
|
||||
test('stamps version, equal timestamps, and fills the profile for inline JSON', () => {
|
||||
const now = new Date('2026-06-05T10:00:00.000Z');
|
||||
const data = [
|
||||
{ city: 'Kyiv', pop: 2900000 },
|
||||
{ city: 'Lviv', pop: 720000 },
|
||||
];
|
||||
const d = createDataset({
|
||||
id: 99,
|
||||
name: 'Cities',
|
||||
data,
|
||||
format: 'json',
|
||||
source: 'inline',
|
||||
now,
|
||||
});
|
||||
|
||||
expect(d.id).toBe(99);
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
expect(d.name).toBe('Cities');
|
||||
expect(d.created).toBe(now.toISOString());
|
||||
expect(d.modified).toBe(d.created);
|
||||
expect(d.comment).toBe('');
|
||||
expect(d.rowCount).toBe(2);
|
||||
expect(d.columnCount).toBe(2);
|
||||
expect(d.columns).toEqual(['city', 'pop']);
|
||||
expect(d.columnTypes).toEqual([
|
||||
{ name: 'city', type: 'string' },
|
||||
{ name: 'pop', type: 'number' },
|
||||
]);
|
||||
expect(d.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('fills the profile for inline CSV', () => {
|
||||
const d = createDataset({
|
||||
id: 1,
|
||||
name: 'C',
|
||||
data: 'a,b\n1,2\n3,4',
|
||||
format: 'csv',
|
||||
source: 'inline',
|
||||
});
|
||||
expect(d.rowCount).toBe(2);
|
||||
expect(d.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('URL dataset gets an N/A profile with a size', () => {
|
||||
const d = createDataset({
|
||||
id: 2,
|
||||
name: 'Remote',
|
||||
data: 'https://example.com/x.json',
|
||||
format: 'json',
|
||||
source: 'url',
|
||||
comment: 'remote source',
|
||||
});
|
||||
expect(d.rowCount).toBeNull();
|
||||
expect(d.columnCount).toBeNull();
|
||||
expect(d.columns).toEqual([]);
|
||||
expect(d.comment).toBe('remote source');
|
||||
expect(d.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('defaults the id when not injected', () => {
|
||||
const d = createDataset({ name: 'X', data: [], format: 'json', source: 'inline' });
|
||||
expect(typeof d.id).toBe('number');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Dataset — a named, reusable data source snippets reference by name
|
||||
* (spec §09B → Dataset; spec §05 → Datasets).
|
||||
*
|
||||
* Portable core: no browser APIs, no React. Defines the record shape, the current
|
||||
* record schema version, the by-name reference object (spec §05 → Copy Reference),
|
||||
* a simple delimited-text parser, the profiling orchestration, and a factory that
|
||||
* stamps timestamps/version and fills the derived summary fields.
|
||||
*
|
||||
* A dataset has one of two **sources** — `inline` (data stored in the record) or
|
||||
* `url` (only the link is stored, fetched on demand at render time) — and one of
|
||||
* four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
|
||||
* The `data` field's shape follows source/format: a URL string for `url`; raw
|
||||
* text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
|
||||
*
|
||||
* Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
|
||||
* non-tabular data get an N/A profile but are still sized (see profile.ts).
|
||||
*/
|
||||
|
||||
import type { DataFormat } from './format-detection';
|
||||
import { profileData, type DatasetProfile } from './profile';
|
||||
import type { ColumnType } from './type-inference';
|
||||
|
||||
/** Current schema version for a Dataset record (read-time migration target). */
|
||||
export const CURRENT_DATASET_VERSION = 1;
|
||||
|
||||
/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
|
||||
export type DataSource = 'inline' | 'url';
|
||||
|
||||
export interface Dataset {
|
||||
/** Unique numeric identifier. */
|
||||
id: number;
|
||||
/** Record schema version, for read-time migration. */
|
||||
version: number;
|
||||
/** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
|
||||
name: string;
|
||||
/**
|
||||
* The payload. For `source = url`: the URL string. For `source = inline`: the
|
||||
* raw CSV/TSV text, or the parsed JSON/TopoJSON value.
|
||||
*/
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** Free-form user note about the dataset. */
|
||||
comment: string;
|
||||
/** Data rows, or `null` when N/A (URL / non-tabular). */
|
||||
rowCount: number | null;
|
||||
/** Columns, or `null` when N/A. */
|
||||
columnCount: number | null;
|
||||
/** Column names, in order. */
|
||||
columns: string[];
|
||||
/** Per-column inferred type. */
|
||||
columnTypes: Array<{ name: string; type: ColumnType }>;
|
||||
/** Approximate payload size in bytes. */
|
||||
size: number;
|
||||
/** ISO timestamp — when first added. */
|
||||
created: string;
|
||||
/** ISO timestamp — when last changed. */
|
||||
modified: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The by-name reference object copied into a spec via "Copy Reference"
|
||||
* (spec §05 → Actions): `{ "data": { "name": "MyDataset" } }`.
|
||||
*/
|
||||
export function datasetReference(name: string): { data: { name: string } } {
|
||||
return { data: { name } };
|
||||
}
|
||||
|
||||
/** UTF-8 byte length of a string (`TextEncoder` is a platform global, not DOM). */
|
||||
function byteLength(str: string): number {
|
||||
return new TextEncoder().encode(str).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize delimited text into rows of string cells, honoring RFC-4180 quoting:
|
||||
* a field wrapped in `"` may contain the delimiter, newlines, and escaped quotes
|
||||
* (`""` → `"`); a quote is only special at the start of a field. This mirrors how
|
||||
* d3-dsv (the parser Vega-Lite uses at render time) reads the same text, so the
|
||||
* profile Astrolabe shows agrees with the field names the chart actually sees.
|
||||
* Whitespace is NOT trimmed — like d3, the cell is taken verbatim (type inference
|
||||
* trims internally when classifying, so numeric columns still detect).
|
||||
*/
|
||||
function tokenizeDelimited(text: string, delimiter: string): string[][] {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
let inQuotes = false;
|
||||
let started = false; // any character seen for the current record?
|
||||
let i = text.charCodeAt(0) === 0xfeff ? 1 : 0; // skip a leading BOM
|
||||
const n = text.length;
|
||||
|
||||
const endField = () => {
|
||||
row.push(field);
|
||||
field = '';
|
||||
};
|
||||
const endRow = () => {
|
||||
endField();
|
||||
rows.push(row);
|
||||
row = [];
|
||||
started = false;
|
||||
};
|
||||
|
||||
while (i < n) {
|
||||
const c = text[i];
|
||||
if (inQuotes) {
|
||||
if (c === '"') {
|
||||
if (text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
inQuotes = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
field += c;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '"' && field === '') {
|
||||
inQuotes = true;
|
||||
started = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === delimiter) {
|
||||
endField();
|
||||
started = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '\n' || c === '\r') {
|
||||
if (c === '\r' && text[i + 1] === '\n') i++;
|
||||
endRow();
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
field += c;
|
||||
started = true;
|
||||
i++;
|
||||
}
|
||||
// Flush a final field/row only if the last record had content (no phantom row
|
||||
// from a trailing newline).
|
||||
if (started || field !== '' || row.length > 0) endRow();
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse delimited text into rows-of-objects (header row → keys), RFC-4180 quoting
|
||||
* aware (see `tokenizeDelimited`). The first record is the header; each later
|
||||
* record is zipped header→cell. A record with fewer cells than the header leaves
|
||||
* the missing columns `undefined`; extra cells beyond the header are ignored.
|
||||
* Fully-empty records (e.g. a blank line) are skipped.
|
||||
*/
|
||||
export function parseDelimited(
|
||||
text: string,
|
||||
format: 'csv' | 'tsv',
|
||||
): Array<Record<string, unknown>> {
|
||||
const delimiter = format === 'tsv' ? '\t' : ',';
|
||||
const records = tokenizeDelimited(text, delimiter);
|
||||
if (records.length < 2) return [];
|
||||
|
||||
const header = records[0];
|
||||
const rows: Array<Record<string, unknown>> = [];
|
||||
for (let i = 1; i < records.length; i++) {
|
||||
const cells = records[i];
|
||||
if (cells.length === 1 && cells[0] === '') continue; // blank record
|
||||
const row: Record<string, unknown> = {};
|
||||
for (let c = 0; c < header.length; c++) {
|
||||
row[header[c]] = c < cells.length ? cells[c] : undefined;
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** A JSON value is tabular when it is a non-empty array of plain objects. */
|
||||
function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
|
||||
if (!Array.isArray(value) || value.length === 0) return null;
|
||||
const allObjects = value.every((v) => v !== null && typeof v === 'object' && !Array.isArray(v));
|
||||
return allObjects ? (value as Array<Record<string, unknown>>) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
|
||||
* tabular vs N/A by source/format, and delegate to `profileData`.
|
||||
*
|
||||
* - `url` (any format) → N/A profile; size = byte length of the URL string.
|
||||
* - inline `json` → rows when a non-empty array of objects, else N/A.
|
||||
* - inline `topojson` → N/A (non-tabular).
|
||||
* - inline `csv` / `tsv` → `parseDelimited` rows.
|
||||
*
|
||||
* `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
|
||||
* `JSON.stringify(data)` for json/topojson.
|
||||
*/
|
||||
export function computeDatasetProfile(
|
||||
data: unknown,
|
||||
format: DataFormat,
|
||||
source: DataSource,
|
||||
): DatasetProfile {
|
||||
if (source === 'url') {
|
||||
const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
|
||||
return profileData(null, byteLength(url));
|
||||
}
|
||||
|
||||
switch (format) {
|
||||
case 'csv':
|
||||
case 'tsv': {
|
||||
const text = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
|
||||
return profileData(parseDelimited(text, format), byteLength(text));
|
||||
}
|
||||
case 'json': {
|
||||
const size = byteLength(JSON.stringify(data) ?? '');
|
||||
return profileData(asObjectRows(data), size);
|
||||
}
|
||||
case 'topojson':
|
||||
default: {
|
||||
const size = byteLength(JSON.stringify(data) ?? '');
|
||||
return profileData(null, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateDatasetOptions {
|
||||
/** The dataset name (uniqueness is enforced upstream — see naming.ts). */
|
||||
name: string;
|
||||
/** The payload, shaped per source/format (see `Dataset.data`). */
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** Optional free-form note. */
|
||||
comment?: string;
|
||||
/** Clock injection for deterministic tests; defaults to the current time. */
|
||||
now?: Date;
|
||||
/** Id injection for deterministic tests; defaults to `Date.now()`. */
|
||||
id?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Dataset: stamps version/timestamps and runs `computeDatasetProfile` to
|
||||
* fill the derived summary fields. `id` defaults to `Date.now()` (numeric;
|
||||
* collision-prone for batch creation — see the DatasetStore.add TODO) and is
|
||||
* injectable for tests.
|
||||
*/
|
||||
export function createDataset(options: CreateDatasetOptions): Dataset {
|
||||
const now = options.now ?? new Date();
|
||||
const iso = now.toISOString();
|
||||
const profile = computeDatasetProfile(options.data, options.format, options.source);
|
||||
|
||||
return {
|
||||
id: options.id ?? Date.now(),
|
||||
version: CURRENT_DATASET_VERSION,
|
||||
name: options.name,
|
||||
data: options.data,
|
||||
format: options.format,
|
||||
source: options.source,
|
||||
comment: options.comment ?? '',
|
||||
rowCount: profile.rowCount,
|
||||
columnCount: profile.columnCount,
|
||||
columns: profile.columns,
|
||||
columnTypes: profile.columnTypes,
|
||||
size: profile.size,
|
||||
created: iso,
|
||||
modified: iso,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { isNameTaken, makeUniqueName } from './naming';
|
||||
|
||||
describe('isNameTaken', () => {
|
||||
const datasets = [
|
||||
{ id: 1, name: 'Sales' },
|
||||
{ id: 2, name: 'Regions' },
|
||||
];
|
||||
|
||||
test('matches case-insensitively', () => {
|
||||
expect(isNameTaken('sales', datasets)).toBe(true);
|
||||
expect(isNameTaken('SALES', datasets)).toBe(true);
|
||||
expect(isNameTaken('Unknown', datasets)).toBe(false);
|
||||
});
|
||||
|
||||
test('trims the desired name before comparing', () => {
|
||||
expect(isNameTaken(' Sales ', datasets)).toBe(true);
|
||||
});
|
||||
|
||||
test('excludeId lets a record ignore itself (e.g. a case-only rename)', () => {
|
||||
expect(isNameTaken('Sales', datasets, 1)).toBe(false);
|
||||
expect(isNameTaken('sales', datasets, 1)).toBe(false);
|
||||
// Renaming to a name owned by a *different* record is still taken.
|
||||
expect(isNameTaken('Regions', datasets, 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('makeUniqueName', () => {
|
||||
test('returns the trimmed desired name when free', () => {
|
||||
expect(makeUniqueName('Fresh', ['Sales'])).toBe('Fresh');
|
||||
expect(makeUniqueName(' Fresh ', ['Sales'])).toBe('Fresh');
|
||||
});
|
||||
|
||||
test('suffixes collisions: Name -> Name 2 -> Name 3', () => {
|
||||
expect(makeUniqueName('Name', ['Name'])).toBe('Name 2');
|
||||
expect(makeUniqueName('Name', ['Name', 'Name 2'])).toBe('Name 3');
|
||||
});
|
||||
|
||||
test('comparison is case-insensitive but casing is preserved', () => {
|
||||
expect(makeUniqueName('Name', ['name'])).toBe('Name 2');
|
||||
expect(makeUniqueName('MyData', ['mydata'])).toBe('MyData 2');
|
||||
});
|
||||
|
||||
test('within-batch reservation is the caller responsibility: a single call does not reserve', () => {
|
||||
// Two consecutive calls with the same existing set both yield "Name 2" —
|
||||
// the caller must reserve each chosen name as it goes (arch 07 §5).
|
||||
const existing = ['Name'];
|
||||
expect(makeUniqueName('Name', existing)).toBe('Name 2');
|
||||
expect(makeUniqueName('Name', existing)).toBe('Name 2');
|
||||
// Reserving manually advances to the next free slot.
|
||||
expect(makeUniqueName('Name', [...existing, 'Name 2'])).toBe('Name 3');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Name uniqueness for datasets (spec §05 → Naming & Uniqueness;
|
||||
* docs/architecture/07 §2).
|
||||
*
|
||||
* Portable core: no browser APIs, no React, no store access — plain data in,
|
||||
* plain data out. Dataset names are the key snippets reference via
|
||||
* `datasetRefs` and the contract a Vega-Lite spec uses through
|
||||
* `{ "data": { "name": "..." } }`, so they must be unique. Comparisons are
|
||||
* **case-insensitive** throughout (`Sales` and `sales` collide) so a single
|
||||
* display name maps to a single dataset regardless of how a reference is typed.
|
||||
*
|
||||
* - `isNameTaken` — reject duplicate create/rename in the UI. `excludeId` lets
|
||||
* a rename ignore the record being renamed (renaming `Sales` to `Sales`, or a
|
||||
* case-only edit, is not a self-collision).
|
||||
* - `makeUniqueName` — for non-interactive paths (import, extract, build chart)
|
||||
* where blocking the user is worse than a silent, reported rename: derive the
|
||||
* next free `${base} ${n}` (n ≥ 2). We never parse meaning out of a name; a
|
||||
* base already ending in a number still just gets a suffix (`Q1 2024 2`).
|
||||
*/
|
||||
|
||||
/** Case-insensitive: is `desired` already used by another dataset (minus `excludeId`)? */
|
||||
export function isNameTaken(
|
||||
desired: string,
|
||||
datasets: ReadonlyArray<{ id: number; name: string }>,
|
||||
excludeId?: number,
|
||||
): boolean {
|
||||
const lower = desired.trim().toLowerCase();
|
||||
return datasets.some((d) => d.id !== excludeId && d.name.toLowerCase() === lower);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `desired` (trimmed) if free, else the first available `${desired} ${n}`
|
||||
* (n ≥ 2). `existingNames` is the set of names already in the collection.
|
||||
* Comparison is case-insensitive; the returned name preserves `desired`'s casing.
|
||||
*/
|
||||
export function makeUniqueName(desired: string, existingNames: Iterable<string>): string {
|
||||
const taken = new Set<string>();
|
||||
for (const n of existingNames) taken.add(n.toLowerCase());
|
||||
|
||||
const base = desired.trim();
|
||||
if (!taken.has(base.toLowerCase())) return base;
|
||||
|
||||
let n = 2;
|
||||
while (taken.has(`${base} ${n}`.toLowerCase())) n++;
|
||||
return `${base} ${n}`;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { profileData } from './profile';
|
||||
|
||||
describe('profileData', () => {
|
||||
test('profiles a JSON-array dataset fully', () => {
|
||||
const rows = [
|
||||
{ city: 'Kyiv', pop: 2900000, capital: 'true' },
|
||||
{ city: 'Lviv', pop: 720000, capital: 'false' },
|
||||
];
|
||||
const profile = profileData(rows, 123);
|
||||
expect(profile.rowCount).toBe(2);
|
||||
expect(profile.columnCount).toBe(3);
|
||||
expect(profile.columns).toEqual(['city', 'pop', 'capital']);
|
||||
expect(profile.columnTypes).toEqual([
|
||||
{ name: 'city', type: 'string' },
|
||||
{ name: 'pop', type: 'number' },
|
||||
{ name: 'capital', type: 'boolean' },
|
||||
]);
|
||||
expect(profile.size).toBe(123);
|
||||
});
|
||||
|
||||
test('null rows (URL) return the N/A profile but keep size', () => {
|
||||
const profile = profileData(null, 42);
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.columns).toEqual([]);
|
||||
expect(profile.columnTypes).toEqual([]);
|
||||
expect(profile.size).toBe(42);
|
||||
});
|
||||
|
||||
test('an empty array returns the N/A profile but keeps size', () => {
|
||||
const profile = profileData([], 7);
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.columns).toEqual([]);
|
||||
expect(profile.size).toBe(7);
|
||||
});
|
||||
|
||||
test('column order follows first-seen key order across ragged rows', () => {
|
||||
const rows = [{ a: 1 }, { b: 2, a: 3 }, { c: 4 }];
|
||||
const profile = profileData(rows, 0);
|
||||
expect(profile.columns).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
test('respects the sampling cap: rows beyond the head do not affect type', () => {
|
||||
// First 200 rows numeric; row 201 is text. With head-only sampling the
|
||||
// column still reads as number (an accepted, documented trade-off).
|
||||
const rows: Array<Record<string, unknown>> = [];
|
||||
for (let i = 0; i < 200; i++) rows.push({ v: i });
|
||||
rows.push({ v: 'tail-text' });
|
||||
const profile = profileData(rows, 0);
|
||||
expect(profile.rowCount).toBe(201); // count reflects the whole payload
|
||||
expect(profile.columnTypes).toEqual([{ name: 'v', type: 'number' }]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Dataset profiling (spec §05 → Profiling; docs/architecture/06 §3–§4).
|
||||
*
|
||||
* Portable core: no browser APIs, no React. A **profile** is the set of derived
|
||||
* summary fields stored on a dataset record so the UI can describe it without
|
||||
* re-parsing the payload: row/column counts, column names (in order), a per-column
|
||||
* inferred type, and an approximate byte size.
|
||||
*
|
||||
* `profileData` takes already-parsed rows-of-objects (the tabular form of a
|
||||
* CSV/TSV/JSON-array payload) plus a precomputed `size`. Parsing the delimited
|
||||
* text and deciding the payload shape happen *upstream* (see dataset.ts) so this
|
||||
* function stays pure and trivially testable.
|
||||
*
|
||||
* - `null` rows (URL data) or an empty array (non-tabular) → the **N/A profile**
|
||||
* (`rowCount`/`columnCount` null, empty columns), but `size` is still carried.
|
||||
* - Columns are the union of keys across all rows, in **first-seen order**.
|
||||
* - `rowCount`/`columnCount`/`size` reflect the **whole** payload; only type
|
||||
* inference samples — capped at the first `SAMPLE_SIZE` rows for speed and
|
||||
* determinism (arch 06 §4).
|
||||
*/
|
||||
|
||||
import { inferColumnType, type ColumnType } from './type-inference';
|
||||
|
||||
export interface ColumnTypeInfo {
|
||||
/** The column name. */
|
||||
name: string;
|
||||
/** The inferred display type for the column. */
|
||||
type: ColumnType;
|
||||
}
|
||||
|
||||
export interface DatasetProfile {
|
||||
/** Data rows, or `null` when N/A (URL / non-tabular). */
|
||||
rowCount: number | null;
|
||||
/** Columns, or `null` when N/A. */
|
||||
columnCount: number | null;
|
||||
/** Column names, in first-seen order. */
|
||||
columns: string[];
|
||||
/** Per-column inferred type. */
|
||||
columnTypes: ColumnTypeInfo[];
|
||||
/** Approximate payload size in bytes. */
|
||||
size: number;
|
||||
}
|
||||
|
||||
/** The N/A profile for URL / non-tabular data — still carries the byte size. */
|
||||
const naProfile = (size: number): DatasetProfile => ({
|
||||
rowCount: null,
|
||||
columnCount: null,
|
||||
columns: [],
|
||||
columnTypes: [],
|
||||
size,
|
||||
});
|
||||
|
||||
/**
|
||||
* Cap on rows fed to type inference. Counts and size scan the whole payload; only
|
||||
* the per-value type check is bounded, sampling the head for determinism (§4).
|
||||
*/
|
||||
const SAMPLE_SIZE = 200;
|
||||
|
||||
const sampleRows = <T>(rows: ReadonlyArray<T>): ReadonlyArray<T> =>
|
||||
rows.length <= SAMPLE_SIZE ? rows : rows.slice(0, SAMPLE_SIZE);
|
||||
|
||||
/**
|
||||
* Profile a dataset payload. `rows` is the tabular form (CSV/TSV/JSON-array)
|
||||
* already parsed to rows-of-objects, or `null` for URL / non-tabular data;
|
||||
* `size` is the precomputed byte length of the stored payload.
|
||||
*/
|
||||
export function profileData(
|
||||
rows: ReadonlyArray<Record<string, unknown>> | null,
|
||||
size: number,
|
||||
): DatasetProfile {
|
||||
if (!rows || rows.length === 0) return naProfile(size);
|
||||
|
||||
// Column order = first-seen order across all rows (handles ragged rows).
|
||||
const columns: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const row of rows) {
|
||||
for (const key of Object.keys(row)) {
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
columns.push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (columns.length === 0) return naProfile(size);
|
||||
|
||||
const sample = sampleRows(rows);
|
||||
const columnTypes = columns.map((name) => ({
|
||||
name,
|
||||
type: inferColumnType(sample.map((r) => r[name])),
|
||||
}));
|
||||
|
||||
return {
|
||||
rowCount: rows.length,
|
||||
columnCount: columns.length,
|
||||
columns,
|
||||
columnTypes,
|
||||
size,
|
||||
};
|
||||
}
|
||||
+122
-1
@@ -1,5 +1,10 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { escapeVegaField, prepareSpecForRender } from './rendering';
|
||||
import {
|
||||
DatasetNotFoundError,
|
||||
escapeVegaField,
|
||||
prepareSpecForRender,
|
||||
type ResolvableDataset,
|
||||
} from './rendering';
|
||||
|
||||
describe('prepareSpecForRender', () => {
|
||||
test('returns a deep copy, never the same reference', () => {
|
||||
@@ -95,6 +100,122 @@ describe('prepareSpecForRender — fit modes (spec §04 Rendering Contract step
|
||||
});
|
||||
});
|
||||
|
||||
describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contract step 1)', () => {
|
||||
const datasets: ResolvableDataset[] = [
|
||||
{ name: 'JsonDs', data: [{ a: 1 }], format: 'json', source: 'inline' },
|
||||
{ name: 'CsvDs', data: 'a,b\n1,2', format: 'csv', source: 'inline' },
|
||||
{ name: 'TsvDs', data: 'a\tb\n1\t2', format: 'tsv', source: 'inline' },
|
||||
{
|
||||
name: 'TopoDs',
|
||||
data: { type: 'Topology', objects: {} },
|
||||
format: 'topojson',
|
||||
source: 'inline',
|
||||
},
|
||||
{ name: 'UrlDs', data: 'https://x/y.csv', format: 'csv', source: 'url' },
|
||||
];
|
||||
|
||||
test('inline JSON → values inlined', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'JsonDs' }, mark: 'bar' }, { datasets });
|
||||
expect(out.data).toEqual({ values: [{ a: 1 }] });
|
||||
});
|
||||
|
||||
test('inline CSV → raw text inlined, tagged csv', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'CsvDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ values: 'a,b\n1,2', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
test('inline TSV → raw text inlined, tagged tsv', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'TsvDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ values: 'a\tb\n1\t2', format: { type: 'tsv' } });
|
||||
});
|
||||
|
||||
test('inline TopoJSON → value inlined, tagged topojson (preserving feature)', () => {
|
||||
const out = prepareSpecForRender(
|
||||
{ data: { name: 'TopoDs', format: { feature: 'counties' } } },
|
||||
{ datasets },
|
||||
);
|
||||
expect(out.data).toEqual({
|
||||
values: { type: 'Topology', objects: {} },
|
||||
format: { feature: 'counties', type: 'topojson' },
|
||||
});
|
||||
});
|
||||
|
||||
test('URL → url reference tagged with the dataset format', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ url: 'https://x/y.csv', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
test('resolves references in nested layer / concat / child sub-specs', () => {
|
||||
const spec = {
|
||||
layer: [{ data: { name: 'JsonDs' } }],
|
||||
spec: { hconcat: [{ data: { name: 'CsvDs' } }] },
|
||||
};
|
||||
const out = prepareSpecForRender(spec, { datasets }) as unknown as {
|
||||
layer: Array<{ data: unknown }>;
|
||||
spec: { hconcat: Array<{ data: unknown }> };
|
||||
};
|
||||
expect(out.layer[0].data).toEqual({ values: [{ a: 1 }] });
|
||||
expect(out.spec.hconcat[0].data).toEqual({ values: 'a,b\n1,2', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
test('matches dataset names case-insensitively', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'jsonds' } }, { datasets });
|
||||
expect(out.data).toEqual({ values: [{ a: 1 }] });
|
||||
});
|
||||
|
||||
test('throws DatasetNotFoundError naming the missing dataset', () => {
|
||||
expect(() => prepareSpecForRender({ data: { name: 'Missing' } }, { datasets })).toThrow(
|
||||
DatasetNotFoundError,
|
||||
);
|
||||
expect(() => prepareSpecForRender({ data: { name: 'Missing' } }, { datasets })).toThrow(
|
||||
/Missing/,
|
||||
);
|
||||
});
|
||||
|
||||
test('a library reference with no datasets provided is still not-found', () => {
|
||||
expect(() => prepareSpecForRender({ data: { name: 'Anything' } })).toThrow(
|
||||
DatasetNotFoundError,
|
||||
);
|
||||
});
|
||||
|
||||
test('a spec with no named refs passes through untouched even with no datasets', () => {
|
||||
const spec = { data: { values: [{ a: 1 }] }, mark: 'bar' };
|
||||
expect(prepareSpecForRender(spec)).toEqual(spec);
|
||||
});
|
||||
|
||||
test('a self-defined top-level datasets name is left untouched and does not throw', () => {
|
||||
const spec = { datasets: { local: [{ a: 1 }] }, data: { name: 'local' }, mark: 'bar' };
|
||||
const out = prepareSpecForRender(spec, { datasets });
|
||||
expect(out.data).toEqual({ name: 'local' });
|
||||
});
|
||||
|
||||
test('resolution runs before fit-mode: a ref + fit mode produce both transforms', () => {
|
||||
const spec = { data: { name: 'JsonDs' }, mark: 'bar' };
|
||||
const out = prepareSpecForRender(spec, { datasets, fitMode: 'full' }) as unknown as {
|
||||
data: unknown;
|
||||
width: string;
|
||||
height: string;
|
||||
};
|
||||
expect(out.data).toEqual({ values: [{ a: 1 }] });
|
||||
expect(out.width).toBe('container');
|
||||
expect(out.height).toBe('container');
|
||||
});
|
||||
|
||||
test('copy-not-mutate: the input spec is untouched during resolution', () => {
|
||||
const spec = { data: { name: 'JsonDs' }, mark: 'bar' };
|
||||
const before = structuredClone(spec);
|
||||
prepareSpecForRender(spec, { datasets });
|
||||
expect(spec).toEqual(before);
|
||||
});
|
||||
|
||||
test('preserves pre-existing reference keys other than name', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'JsonDs', foo: 1 } }, { datasets }) as {
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(out.data).toEqual({ values: [{ a: 1 }], foo: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('escapeVegaField', () => {
|
||||
test('escapes dots and brackets that VL treats as accessors', () => {
|
||||
expect(escapeVegaField('user.age')).toBe('user\\.age');
|
||||
|
||||
+125
-3
@@ -7,19 +7,59 @@
|
||||
* deterministic steps, in order, **on a deep copy** so the user's stored spec is
|
||||
* never mutated by rendering:
|
||||
*
|
||||
* 1. Dataset reference resolution — arrives in M3 (no-op here).
|
||||
* 1. Dataset reference resolution — implemented in M3 (see below).
|
||||
* 2. Fit-mode sizing — implemented in M2.
|
||||
*
|
||||
* Step 1 (spec §04 → Rendering Contract): every named-data reference
|
||||
* (`{ data: { name } }`) is replaced in-place with the referenced library
|
||||
* dataset's actual contents, shaped by source and format. A name the spec defines
|
||||
* for itself via a top-level `datasets` object is left untouched (Vega-Lite
|
||||
* resolves it natively); an unknown library name throws `DatasetNotFoundError`.
|
||||
* Resolution recurses into the same nested sub-specs as fit-mode, runs before
|
||||
* sizing, and operates only on the copy.
|
||||
*
|
||||
* The copy-not-mutate invariant and the call site the renderer depends on are
|
||||
* fixed; M3 fills in step 1 without the preview pipeline changing shape.
|
||||
* fixed.
|
||||
*/
|
||||
|
||||
import type { DataFormat } from './format-detection';
|
||||
import type { DataSource } from './dataset';
|
||||
|
||||
/** Preview sizing modes (spec §04 → Fit / Sizing Modes). `default` = Original. */
|
||||
export type FitMode = 'default' | 'width' | 'height' | 'full';
|
||||
|
||||
/**
|
||||
* The minimal structural view of a dataset that reference resolution needs. The
|
||||
* DatasetStore's records are structurally compatible, so passing them works
|
||||
* without importing the full `Dataset` type (and keeps this free of any cycle).
|
||||
*/
|
||||
export interface ResolvableDataset {
|
||||
/** The library name a spec references via `{ data: { name } }`. */
|
||||
name: string;
|
||||
/** The payload — see `Dataset.data` for the per-source/format shape. */
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
}
|
||||
|
||||
/** Thrown when a spec references a library dataset name that does not exist. */
|
||||
export class DatasetNotFoundError extends Error {
|
||||
/** The missing dataset's name, so callers can build a tailored, fixable message. */
|
||||
readonly datasetName: string;
|
||||
constructor(name: string) {
|
||||
super(`Dataset not found: "${name}"`);
|
||||
this.name = 'DatasetNotFoundError';
|
||||
this.datasetName = name;
|
||||
}
|
||||
}
|
||||
|
||||
export interface PrepareOptions {
|
||||
/** Active fit mode. Defaults to `'default'` (Original — spec sizing untouched). */
|
||||
fitMode?: FitMode;
|
||||
/** The dataset library used to resolve named-data references (step 1). */
|
||||
datasets?: ReadonlyArray<ResolvableDataset>;
|
||||
}
|
||||
|
||||
/** The container/sub-spec keys the rendering contract recurses into (spec §04). */
|
||||
@@ -71,6 +111,83 @@ function applyFitMode(node: unknown, mode: FitMode): void {
|
||||
if (isSpecNode(node.spec)) applyFitMode(node.spec, mode);
|
||||
}
|
||||
|
||||
/** The set of dataset names a spec defines for itself via top-level `datasets`. */
|
||||
function selfDefinedDatasetNames(spec: unknown): Set<string> {
|
||||
const names = new Set<string>();
|
||||
if (isSpecNode(spec)) {
|
||||
const datasets = spec.datasets;
|
||||
if (isSpecNode(datasets)) for (const key of Object.keys(datasets)) names.add(key);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the replacement `data` object for one resolved reference (spec §04 →
|
||||
* Rendering Contract, step 1). `rest` is the reference's other keys (e.g. a
|
||||
* `format` carrying a TopoJSON `feature`); the incoming `name` is dropped and any
|
||||
* pre-existing `format` is merged so such keys survive.
|
||||
*/
|
||||
function resolvedData(dataset: ResolvableDataset, rest: SpecNode): SpecNode {
|
||||
const restFormat = isSpecNode(rest.format) ? rest.format : {};
|
||||
if (dataset.source === 'url') {
|
||||
return {
|
||||
...rest,
|
||||
url: typeof dataset.data === 'string' ? dataset.data : (JSON.stringify(dataset.data) ?? ''),
|
||||
format: { ...restFormat, type: dataset.format },
|
||||
};
|
||||
}
|
||||
switch (dataset.format) {
|
||||
case 'json':
|
||||
return { ...rest, values: dataset.data };
|
||||
case 'topojson':
|
||||
return { ...rest, values: dataset.data, format: { ...restFormat, type: 'topojson' } };
|
||||
case 'csv':
|
||||
case 'tsv':
|
||||
return { ...rest, values: dataset.data, format: { ...restFormat, type: dataset.format } };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace every named-data reference in `node` with its library dataset's
|
||||
* contents, recursing through the entire spec (arrays and objects) so refs
|
||||
* anywhere are resolved — matching `extractDatasetRefs`. A self-defined name is
|
||||
* left untouched; an unknown library name throws `DatasetNotFoundError`. Matching
|
||||
* is case-insensitive, mirroring naming.ts. Mutates in place; the caller already
|
||||
* works on a copy.
|
||||
*/
|
||||
function resolveDatasetRefs(
|
||||
node: unknown,
|
||||
byName: Map<string, ResolvableDataset>,
|
||||
selfDefined: Set<string>,
|
||||
): void {
|
||||
if (Array.isArray(node)) {
|
||||
for (const item of node) resolveDatasetRefs(item, byName, selfDefined);
|
||||
return;
|
||||
}
|
||||
if (!isSpecNode(node)) return;
|
||||
|
||||
const data = node.data;
|
||||
if (isSpecNode(data) && typeof data.name === 'string') {
|
||||
const name = data.name;
|
||||
if (!selfDefined.has(name)) {
|
||||
const dataset = byName.get(name.toLowerCase());
|
||||
if (!dataset) throw new DatasetNotFoundError(name);
|
||||
const { name: _drop, ...rest } = data;
|
||||
node.data = resolvedData(dataset, rest);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: this walks EVERY key, so it also descends into inlined data payloads
|
||||
// (the just-resolved `values`, a spec's `datasets`/`data.values`). That's
|
||||
// wasteful for large inline data on every debounced render, and a row with a
|
||||
// field literally named `data` holding `{ name: "x" }` would be spuriously
|
||||
// resolved or throw DatasetNotFoundError. extractDatasetRefs shares this broad
|
||||
// walk. A scoped walk (recurse only into the known sub-spec/container keys +
|
||||
// `transform[].lookup.from`, never into data payloads) would be safer and
|
||||
// faster — change deliberately, with tests for where refs may legally appear.
|
||||
for (const key of Object.keys(node)) resolveDatasetRefs(node[key], byName, selfDefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape `.`/`[`/`]` so Vega-Lite treats a string as a literal field name rather
|
||||
* than a nested-property accessor (docs/architecture/05 §4). Used wherever
|
||||
@@ -88,7 +205,12 @@ export function escapeVegaField(name: string): string {
|
||||
export function prepareSpecForRender<T>(spec: T, options: PrepareOptions = {}): T {
|
||||
const copy = structuredClone(spec);
|
||||
|
||||
// 1. M3: resolveDatasetRefs(copy, datasets)
|
||||
// 1. Dataset reference resolution — runs before sizing, on the same copy.
|
||||
const datasets = options.datasets ?? [];
|
||||
const byName = new Map<string, ResolvableDataset>();
|
||||
for (const d of datasets) byName.set(d.name.toLowerCase(), d);
|
||||
resolveDatasetRefs(copy, byName, selfDefinedDatasetNames(copy));
|
||||
|
||||
// 2. Fit-mode sizing.
|
||||
applyFitMode(copy, options.fitMode ?? 'default');
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { extractDatasetRefs, recomputeDatasetRefs, renameDatasetInSpec } from './spec-refs';
|
||||
|
||||
describe('extractDatasetRefs', () => {
|
||||
test('collects a top-level named-data reference', () => {
|
||||
expect(extractDatasetRefs({ data: { name: 'Sales' }, mark: 'bar' })).toEqual(['Sales']);
|
||||
});
|
||||
|
||||
test('collects per-layer references', () => {
|
||||
const spec = {
|
||||
layer: [
|
||||
{ data: { name: 'A' }, mark: 'bar' },
|
||||
{ data: { name: 'B' }, mark: 'line' },
|
||||
],
|
||||
};
|
||||
expect(extractDatasetRefs(spec).sort()).toEqual(['A', 'B']);
|
||||
});
|
||||
|
||||
test('collects references inside concat and a child spec (facet/repeat)', () => {
|
||||
const spec = {
|
||||
facet: { field: 'g' },
|
||||
spec: { hconcat: [{ data: { name: 'C' } }, { vconcat: [{ data: { name: 'D' } }] }] },
|
||||
};
|
||||
expect(extractDatasetRefs(spec).sort()).toEqual(['C', 'D']);
|
||||
});
|
||||
|
||||
test('accepts a JSON-text spec', () => {
|
||||
const spec = JSON.stringify({ data: { name: 'FromString' } });
|
||||
expect(extractDatasetRefs(spec)).toEqual(['FromString']);
|
||||
});
|
||||
|
||||
test('unparseable string yields no refs', () => {
|
||||
expect(extractDatasetRefs('{ not valid json')).toEqual([]);
|
||||
});
|
||||
|
||||
test('inline-data and url-data references (no name) are ignored', () => {
|
||||
expect(extractDatasetRefs({ data: { values: [{ a: 1 }] } })).toEqual([]);
|
||||
expect(extractDatasetRefs({ data: { url: 'http://x/y.csv' } })).toEqual([]);
|
||||
});
|
||||
|
||||
test('excludes names the spec defines for itself via top-level datasets', () => {
|
||||
const spec = {
|
||||
datasets: { foo: [{ a: 1 }] },
|
||||
data: { name: 'foo' },
|
||||
layer: [{ data: { name: 'Library' } }],
|
||||
};
|
||||
// "foo" is self-defined and not a library dependency; "Library" is.
|
||||
expect(extractDatasetRefs(spec)).toEqual(['Library']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('recomputeDatasetRefs', () => {
|
||||
test('returns sorted, de-duped names', () => {
|
||||
const spec = {
|
||||
layer: [{ data: { name: 'B' } }, { data: { name: 'A' } }, { data: { name: 'B' } }],
|
||||
};
|
||||
expect(recomputeDatasetRefs(spec)).toEqual(['A', 'B']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renameDatasetInSpec', () => {
|
||||
test('rewrites every occurrence of the old name', () => {
|
||||
const spec = {
|
||||
data: { name: 'Old' },
|
||||
layer: [{ data: { name: 'Old' } }, { data: { name: 'Other' } }],
|
||||
};
|
||||
const out = renameDatasetInSpec(spec, 'Old', 'New');
|
||||
expect(out.data.name).toBe('New');
|
||||
expect(out.layer[0].data.name).toBe('New');
|
||||
expect(out.layer[1].data.name).toBe('Other');
|
||||
});
|
||||
|
||||
test('preserves other keys on the data object', () => {
|
||||
const spec = { data: { name: 'Old', format: { type: 'csv' } } };
|
||||
const out = renameDatasetInSpec(spec, 'Old', 'New');
|
||||
expect(out.data).toEqual({ name: 'New', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
test('does not mutate the input', () => {
|
||||
const spec = { data: { name: 'Old' } };
|
||||
const before = structuredClone(spec);
|
||||
renameDatasetInSpec(spec, 'Old', 'New');
|
||||
expect(spec).toEqual(before);
|
||||
});
|
||||
|
||||
test('preserves object shape for an object spec', () => {
|
||||
const out = renameDatasetInSpec({ data: { name: 'Old' } }, 'Old', 'New');
|
||||
expect(typeof out).toBe('object');
|
||||
});
|
||||
|
||||
test('preserves string shape for a string spec (returns pretty JSON text)', () => {
|
||||
const out = renameDatasetInSpec(JSON.stringify({ data: { name: 'Old' } }), 'Old', 'New');
|
||||
expect(typeof out).toBe('string');
|
||||
expect(JSON.parse(out)).toEqual({ data: { name: 'New' } });
|
||||
});
|
||||
|
||||
test('returns an unparseable string spec unchanged', () => {
|
||||
const bad = '{ not json';
|
||||
expect(renameDatasetInSpec(bad, 'Old', 'New')).toBe(bad);
|
||||
});
|
||||
|
||||
test("does not rename a name defined by the spec's own top-level datasets", () => {
|
||||
const spec = { datasets: { Old: [{ a: 1 }] }, data: { name: 'Old' } };
|
||||
const out = renameDatasetInSpec(spec, 'Old', 'New');
|
||||
expect(out.data.name).toBe('Old'); // self-defined — left untouched
|
||||
expect(Object.keys(out.datasets)).toEqual(['Old']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Snippet → dataset reference extraction and rename propagation
|
||||
* (spec §09F → Cross-entity relationships; docs/architecture/07 §3.1, §6).
|
||||
*
|
||||
* Portable core: no browser APIs, no React, no store access. A Vega-Lite spec
|
||||
* references named data through `{ "data": { "name": "MyDataset" } }`, which can
|
||||
* appear at the top level, per-layer, or inside `spec`/`facet`/concat children.
|
||||
* Rather than enumerate the grammar, we walk the spec recursively and collect
|
||||
* every `{ data: { name } }` we find — the single source of truth for "what does
|
||||
* this spec reference", which the renderer's resolution must agree with.
|
||||
*
|
||||
* A spec may be stored as an **object** or as **JSON text** (see spec §09A); we
|
||||
* normalize once at the boundary (unparseable text → no refs / unchanged spec) so
|
||||
* the recursive walk never has to care.
|
||||
*
|
||||
* Refinement beyond the doc sketch: a spec may define its OWN inline named
|
||||
* datasets via a top-level `datasets` object (e.g.
|
||||
* `{ "datasets": { "foo": [...] }, "data": { "name": "foo" } }`). Names satisfied
|
||||
* by the spec's own `datasets` are NOT library dependencies, so they are excluded
|
||||
* from extraction and left untouched on rename — keeping extraction consistent
|
||||
* with the renderer's resolution, which likewise must not treat self-defined
|
||||
* names as library refs.
|
||||
*/
|
||||
|
||||
type Json = unknown;
|
||||
|
||||
/** Parse a string spec; an unparseable draft simply has no resolvable refs. */
|
||||
function safeParse(s: string): Json {
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The set of dataset names a spec defines for itself via top-level `datasets`. */
|
||||
function selfDefinedNames(spec: Json): Set<string> {
|
||||
const names = new Set<string>();
|
||||
if (spec && typeof spec === 'object' && !Array.isArray(spec)) {
|
||||
const datasets = (spec as Record<string, Json>).datasets;
|
||||
if (datasets && typeof datasets === 'object' && !Array.isArray(datasets)) {
|
||||
for (const key of Object.keys(datasets)) names.add(key);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects every **library** dataset name referenced by `{ data: { name } }`
|
||||
* anywhere in the spec, excluding names the spec defines for itself via a
|
||||
* top-level `datasets` object. Accepts an object or JSON text.
|
||||
*/
|
||||
export function extractDatasetRefs(spec: Json): string[] {
|
||||
const root = typeof spec === 'string' ? safeParse(spec) : spec;
|
||||
const selfDefined = selfDefinedNames(root);
|
||||
const names = new Set<string>();
|
||||
|
||||
const walk = (node: Json): void => {
|
||||
if (Array.isArray(node)) {
|
||||
for (const item of node) walk(item);
|
||||
return;
|
||||
}
|
||||
if (node && typeof node === 'object') {
|
||||
const obj = node as Record<string, Json>;
|
||||
const data = obj.data as Record<string, Json> | undefined;
|
||||
if (data && typeof data === 'object' && typeof data.name === 'string') {
|
||||
if (!selfDefined.has(data.name)) names.add(data.name);
|
||||
}
|
||||
// TODO: walks every key, so it also descends into data payloads
|
||||
// (`data.values`, top-level `datasets`). A row with a field named `data`
|
||||
// holding `{ name: "x" }` is falsely counted as a reference. Shared with
|
||||
// rendering.ts resolveDatasetRefs — scope both to the keys where refs can
|
||||
// legally appear, together and with tests. Benign for typical data.
|
||||
for (const key of Object.keys(obj)) walk(obj[key]);
|
||||
}
|
||||
};
|
||||
|
||||
walk(root);
|
||||
return [...names];
|
||||
}
|
||||
|
||||
/** The list stored on `snippet.datasetRefs` — sorted + de-duped for stable diffs. */
|
||||
export function recomputeDatasetRefs(spec: Json): string[] {
|
||||
return [...new Set(extractDatasetRefs(spec))].sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a copy of `spec` with every `data.name === oldName` replaced by
|
||||
* `newName`, recursing through arrays and objects. A name that is a top-level
|
||||
* `datasets` key (self-defined) is left untouched. The input's stored shape is
|
||||
* preserved: a string spec is parsed, rewritten, and re-serialized as pretty JSON
|
||||
* text; an object spec returns an object. The generic `<T>` reflects this shape
|
||||
* preservation. The input is never mutated.
|
||||
*/
|
||||
export function renameDatasetInSpec<T>(spec: T, oldName: string, newName: string): T {
|
||||
const isString = typeof spec === 'string';
|
||||
const root = isString ? safeParse(spec) : spec;
|
||||
// Unparseable text → return it unchanged (nothing resolvable to rewrite).
|
||||
if (isString && root === null) return spec;
|
||||
|
||||
const selfDefined = selfDefinedNames(root);
|
||||
|
||||
const rewrite = (node: Json): Json => {
|
||||
if (Array.isArray(node)) return node.map(rewrite);
|
||||
if (node && typeof node === 'object') {
|
||||
const out: Record<string, Json> = {};
|
||||
for (const [k, v] of Object.entries(node as Record<string, Json>)) {
|
||||
if (
|
||||
k === 'data' &&
|
||||
v &&
|
||||
typeof v === 'object' &&
|
||||
!Array.isArray(v) &&
|
||||
(v as Record<string, Json>).name === oldName &&
|
||||
!selfDefined.has(oldName)
|
||||
) {
|
||||
out[k] = { ...v, name: newName };
|
||||
} else {
|
||||
out[k] = rewrite(v);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return node;
|
||||
};
|
||||
|
||||
const rewritten = rewrite(root);
|
||||
return (isString ? JSON.stringify(rewritten, null, 2) : rewritten) as T;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { inferColumnType } from './type-inference';
|
||||
|
||||
describe('inferColumnType', () => {
|
||||
test('detects a clean number column', () => {
|
||||
expect(inferColumnType([1, 2, 3])).toBe('number');
|
||||
expect(inferColumnType(['1', '2.5', ' 3 '])).toBe('number');
|
||||
});
|
||||
|
||||
test('detects a clean string column', () => {
|
||||
expect(inferColumnType(['apple', 'banana', 'cherry'])).toBe('string');
|
||||
});
|
||||
|
||||
test('detects a clean date column', () => {
|
||||
expect(inferColumnType(['2024-01-01', '2025-12-31'])).toBe('date');
|
||||
expect(inferColumnType(['2024/01/01', '12/31/2024'])).toBe('date');
|
||||
});
|
||||
|
||||
test('detects a clean boolean column (any case)', () => {
|
||||
expect(inferColumnType(['true', 'false'])).toBe('boolean');
|
||||
expect(inferColumnType(['TRUE', 'False'])).toBe('boolean');
|
||||
expect(inferColumnType([true, false])).toBe('boolean');
|
||||
});
|
||||
|
||||
test('a mixed column falls to string', () => {
|
||||
expect(inferColumnType([1, 2, 'three'])).toBe('string');
|
||||
expect(inferColumnType(['2024-01-01', 'not-a-date'])).toBe('string');
|
||||
});
|
||||
|
||||
test('ignores empty / whitespace cells before classifying', () => {
|
||||
expect(inferColumnType([1, null, 2, undefined, ' ', 3])).toBe('number');
|
||||
expect(inferColumnType(['true', '', 'false', ' '])).toBe('boolean');
|
||||
});
|
||||
|
||||
test('an all-empty or zero-length column is string', () => {
|
||||
expect(inferColumnType([])).toBe('string');
|
||||
expect(inferColumnType([null, undefined, '', ' '])).toBe('string');
|
||||
});
|
||||
|
||||
test('precedence: a true/false column is boolean, not string', () => {
|
||||
expect(inferColumnType(['true', 'false'])).toBe('boolean');
|
||||
});
|
||||
|
||||
test('precedence: a bare-year column is number, not date', () => {
|
||||
expect(inferColumnType(['2024', '2025'])).toBe('number');
|
||||
});
|
||||
|
||||
test('date shape guard rejects bare numbers and words even if Date.parse might accept them', () => {
|
||||
expect(inferColumnType(['42'])).toBe('number');
|
||||
expect(inferColumnType(['hello'])).toBe('string');
|
||||
expect(inferColumnType(['March'])).toBe('string');
|
||||
});
|
||||
|
||||
test('0 and 1 are number, never boolean', () => {
|
||||
expect(inferColumnType([0, 1, 0, 1])).toBe('number');
|
||||
expect(inferColumnType(['0', '1'])).toBe('number');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Column type inference (spec §05 → Profiling; docs/architecture/06 §2).
|
||||
*
|
||||
* Portable core: no browser APIs, no React — usable in Node and tests. Given the
|
||||
* values of a single column, decide which of four display types it holds:
|
||||
* `number`, `boolean`, `date`, or `string`. This is a **display hint** only —
|
||||
* nothing downstream coerces values from it, and Vega-Lite does its own type
|
||||
* handling at render time — so the rules favour being simple and predictable
|
||||
* over being clever.
|
||||
*
|
||||
* The algorithm (arch 06 §2):
|
||||
* 1. Drop empties (null/undefined/whitespace-only string) — empty cells carry
|
||||
* no type signal.
|
||||
* 2. An all-empty (or zero-length) column is `string` — no evidence otherwise.
|
||||
* 3. Run the checks in precedence order **boolean → number → date → string**,
|
||||
* narrowest evidence to widest; the first for which *every* present value
|
||||
* matches wins. One stray value knocks the column down to the next candidate.
|
||||
*
|
||||
* Date detection guards with a shape regex *before* trusting `Date.parse`, which
|
||||
* on some engines accepts `"42"` or `"March"` and would swallow number/string
|
||||
* columns. `0`/`1` are numbers, never booleans.
|
||||
*/
|
||||
|
||||
export type ColumnType = 'number' | 'string' | 'date' | 'boolean';
|
||||
|
||||
/** Empty cells (null/undefined/whitespace-only string) carry no type signal. */
|
||||
const isEmpty = (v: unknown): boolean =>
|
||||
v === null || v === undefined || (typeof v === 'string' && v.trim() === '');
|
||||
|
||||
/** Native numbers pass when finite; strings must parse to a finite, non-NaN number. */
|
||||
const isNumeric = (v: unknown): boolean => {
|
||||
if (typeof v === 'number') return Number.isFinite(v);
|
||||
if (typeof v !== 'string') return false;
|
||||
const t = v.trim();
|
||||
if (t === '') return false; // reject blank so Number("") === 0 can't sneak through
|
||||
const n = Number(t);
|
||||
return !Number.isNaN(n) && Number.isFinite(n);
|
||||
};
|
||||
|
||||
/** Native booleans pass; otherwise the trimmed, lower-cased string is exactly true/false. */
|
||||
const isBoolean = (v: unknown): boolean => {
|
||||
if (typeof v === 'boolean') return true;
|
||||
if (typeof v !== 'string') return false;
|
||||
const t = v.trim().toLowerCase();
|
||||
return t === 'true' || t === 'false';
|
||||
};
|
||||
|
||||
/**
|
||||
* Shape guard for dates: a leading `YYYY-MM-DD`/`YYYY/MM/DD`, or `M/D/YYYY`.
|
||||
* Required *before* `Date.parse` — see the module note above.
|
||||
*/
|
||||
const DATE_SHAPE = /^\d{4}[-/]\d{2}[-/]\d{2}|^\d{1,2}\/\d{1,2}\/\d{4}/;
|
||||
const isDate = (v: unknown): boolean => {
|
||||
if (typeof v !== 'string') return false;
|
||||
const t = v.trim();
|
||||
return DATE_SHAPE.test(t) && !Number.isNaN(Date.parse(t));
|
||||
};
|
||||
|
||||
/**
|
||||
* Infer one of four column types from a sample of column values. Empty cells are
|
||||
* ignored; an all-empty (or zero-length) column is `string`. Precedence:
|
||||
* boolean → number → date → string.
|
||||
*/
|
||||
export function inferColumnType(values: readonly unknown[]): ColumnType {
|
||||
const present = values.filter((v) => !isEmpty(v));
|
||||
if (present.length === 0) return 'string';
|
||||
|
||||
if (present.every(isBoolean)) return 'boolean';
|
||||
if (present.every(isNumeric)) return 'number';
|
||||
if (present.every(isDate)) return 'date';
|
||||
return 'string';
|
||||
}
|
||||
Reference in New Issue
Block a user