Snapshot URL datasets locally on add; preview tabular data as a table

This commit is contained in:
2026-06-10 10:24:42 +03:00
parent eb5e7ac53a
commit 2410c6e965
23 changed files with 1239 additions and 148 deletions
+122 -21
View File
@@ -5,6 +5,8 @@ import {
createDataset,
datasetReference,
parseDelimited,
snapshotFromText,
tabularRows,
} from './dataset';
describe('datasetReference', () => {
@@ -65,51 +67,125 @@ describe('parseDelimited', () => {
});
describe('computeDatasetProfile', () => {
test('inline JSON array-of-objects is profiled', () => {
test('JSON array-of-objects is profiled', () => {
const data = [
{ a: 1, b: 'x' },
{ a: 2, b: 'y' },
];
const profile = computeDatasetProfile(data, 'json', 'inline');
const profile = computeDatasetProfile(data, 'json');
expect(profile.rowCount).toBe(2);
expect(profile.columns).toEqual(['a', 'b']);
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(data)).length);
});
test('inline JSON that is not an array is N/A but sized', () => {
const profile = computeDatasetProfile({ a: 1 }, 'json', 'inline');
test('JSON that is not an array is N/A but sized', () => {
const profile = computeDatasetProfile({ a: 1 }, 'json');
expect(profile.rowCount).toBeNull();
expect(profile.size).toBeGreaterThan(0);
});
test('inline CSV is parsed and profiled; size is the raw text byte length', () => {
test('CSV is parsed and profiled; size is the raw text byte length', () => {
const text = 'a,b\n1,2';
const profile = computeDatasetProfile(text, 'csv', 'inline');
const profile = computeDatasetProfile(text, 'csv');
expect(profile.rowCount).toBe(1);
expect(profile.columns).toEqual(['a', 'b']);
expect(profile.size).toBe(new TextEncoder().encode(text).length);
});
test('inline TSV is parsed and profiled', () => {
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv', 'inline');
test('TSV is parsed and profiled', () => {
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv');
expect(profile.rowCount).toBe(1);
expect(profile.columns).toEqual(['a', 'b']);
});
test('inline TopoJSON is N/A (non-tabular) but sized', () => {
test('TopoJSON is N/A (non-tabular) but sized', () => {
const topo = { type: 'Topology', objects: {} };
const profile = computeDatasetProfile(topo, 'topojson', 'inline');
const profile = computeDatasetProfile(topo, 'topojson');
expect(profile.rowCount).toBeNull();
expect(profile.columnCount).toBeNull();
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(topo)).length);
});
test('URL is N/A; size is the byte length of the URL string', () => {
const url = 'https://example.com/data.csv';
const profile = computeDatasetProfile(url, 'csv', 'url');
test('a fetched URL snapshot is profiled exactly like inline data', () => {
// The snapshot model: once fetched, a URL dataset carries its payload in `data`
// and profiles through the same path as inline (no source distinction here).
const profile = computeDatasetProfile('a,b\n1,2\n3,4', 'csv');
expect(profile.rowCount).toBe(2);
expect(profile.columns).toEqual(['a', 'b']);
});
test('an unfetched URL reference (null data) is N/A with size 0', () => {
const profile = computeDatasetProfile(null, 'csv');
expect(profile.rowCount).toBeNull();
expect(profile.columnCount).toBeNull();
expect(profile.size).toBe(new TextEncoder().encode(url).length);
expect(profile.columns).toEqual([]);
expect(profile.size).toBe(0);
});
});
describe('tabularRows', () => {
test('CSV parses to rows (same parsing as profiling)', () => {
expect(tabularRows('a,b\n1,2\n3,4', 'csv')).toEqual([
{ a: '1', b: '2' },
{ a: '3', b: '4' },
]);
});
test('TSV parses to rows', () => {
expect(tabularRows('x\ty\n1\t2', 'tsv')).toEqual([{ x: '1', y: '2' }]);
});
test('a JSON array of objects is returned as-is', () => {
const data = [{ a: 1 }, { a: 2 }];
expect(tabularRows(data, 'json')).toEqual(data);
});
test('limit returns only the head', () => {
const rows = tabularRows('n\n1\n2\n3\n4', 'csv', 2);
expect(rows).toEqual([{ n: '1' }, { n: '2' }]);
});
test('non-tabular payloads return null (single object, topojson, unfetched)', () => {
expect(tabularRows({ a: 1 }, 'json')).toBeNull();
expect(tabularRows({ type: 'Topology', objects: {} }, 'topojson')).toBeNull();
expect(tabularRows(null, 'csv')).toBeNull();
expect(tabularRows('a,b', 'csv')).toBeNull(); // header only — no rows
});
});
describe('snapshotFromText', () => {
test('JSON content is parsed and typed json (content beats extension)', () => {
const { data, format } = snapshotFromText('[{"a":1}]', 'https://x/data.txt');
expect(format).toBe('json');
expect(data).toEqual([{ a: 1 }]);
});
test('CSV content keeps its raw text and is typed csv', () => {
const { data, format } = snapshotFromText('a,b\n1,2', 'https://x/data.csv');
expect(format).toBe('csv');
expect(data).toBe('a,b\n1,2');
});
test('TSV content is typed tsv', () => {
expect(snapshotFromText('a\tb\n1\t2', 'https://x/d').format).toBe('tsv');
});
test('a Topology body is typed topojson and parsed', () => {
const { data, format } = snapshotFromText('{"type":"Topology","objects":{}}', 'https://x/d');
expect(format).toBe('topojson');
expect(data).toEqual({ type: 'Topology', objects: {} });
});
test('unrecognized content falls back to the URL extension', () => {
// A single token: not JSON, not delimited. The .json extension decides the
// format; the unparseable body is kept verbatim rather than throwing.
const { data, format } = snapshotFromText('not-data', 'https://x/data.json');
expect(format).toBe('json');
expect(data).toBe('not-data');
});
test('unrecognized content with no useful extension defaults to json', () => {
expect(snapshotFromText('not-data', 'https://x/feed').format).toBe('json');
});
});
@@ -157,20 +233,45 @@ describe('createDataset', () => {
expect(d.columns).toEqual(['a', 'b']);
});
test('URL dataset gets an N/A profile with a size', () => {
test('URL dataset carries url + fetchedAt and profiles the fetched snapshot', () => {
const d = createDataset({
id: 2,
name: 'Remote',
data: 'https://example.com/x.json',
format: 'json',
data: 'a,b\n1,2',
url: 'https://example.com/x.csv',
fetchedAt: '2026-06-10T00:00:00.000Z',
format: 'csv',
source: 'url',
comment: 'remote source',
});
expect(d.rowCount).toBeNull();
expect(d.columnCount).toBeNull();
expect(d.columns).toEqual([]);
expect(d.source).toBe('url');
expect(d.url).toBe('https://example.com/x.csv');
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
expect(d.rowCount).toBe(1);
expect(d.columns).toEqual(['a', 'b']);
expect(d.comment).toBe('remote source');
expect(d.size).toBeGreaterThan(0);
});
test('an unfetched URL dataset (null data) gets an N/A profile and null fetchedAt', () => {
const d = createDataset({
id: 3,
name: 'Unfetched',
data: null,
url: 'https://example.com/x.csv',
format: 'csv',
source: 'url',
});
expect(d.url).toBe('https://example.com/x.csv');
expect(d.fetchedAt).toBeNull();
expect(d.rowCount).toBeNull();
expect(d.columns).toEqual([]);
expect(d.size).toBe(0);
});
test('inline dataset carries no url/fetchedAt keys', () => {
const d = createDataset({ id: 4, name: 'I', data: [], format: 'json', source: 'inline' });
expect('url' in d).toBe(false);
expect('fetchedAt' in d).toBe(false);
});
test('defaults the id when not injected', () => {
+102 -29
View File
@@ -7,22 +7,29 @@
* a simple delimited-text parser, the profiling orchestration, and a factory that
* stamps timestamps/version and fills the derived summary fields.
*
* A dataset has one of two **sources** — `inline` (data stored in the record) or
* `url` (only the link is stored, fetched on demand at render time) — and one of
* four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
* The `data` field's shape follows source/format: a URL string for `url`; raw
* text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
* A dataset has one of two **sources** — `inline` (data pasted into the record) or
* `url` (fetched once from a remote address and **snapshotted** into the record) —
* and one of four **formats** (reused from format-detection: `json`/`csv`/`tsv`/
* `topojson`). Either way `data` holds the actual payload, shaped by format: raw
* text for CSV/TSV, a parsed value for JSON/TopoJSON. A `url` dataset additionally
* keeps its source `url` (so it can be re-fetched) and a `fetchedAt` timestamp;
* until its first successful fetch `data` is `null` (an unfetched reference).
*
* Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
* non-tabular data get an N/A profile but are still sized (see profile.ts).
* Any tabular payload (JSON array-of-objects, CSV, TSV) is profiled — including a
* fetched URL snapshot; non-tabular or not-yet-fetched data gets an N/A profile but
* is still sized (see profile.ts).
*/
import type { DataFormat } from './format-detection';
import { detectFormat, detectFormatFromUrl, type DataFormat } from './format-detection';
import { profileData, type ColumnStats, type DatasetProfile } from './profile';
import type { ColumnType } from './type-inference';
/** Current schema version for a Dataset record (read-time migration target). */
export const CURRENT_DATASET_VERSION = 1;
/**
* Current schema version for a Dataset record (read-time migration target).
* v2 moved a URL dataset's address out of `data` into its own `url` field and made
* `data` hold the fetched snapshot (see `migrateDataset`).
*/
export const CURRENT_DATASET_VERSION = 2;
/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
export type DataSource = 'inline' | 'url';
@@ -35,14 +42,25 @@ export interface Dataset {
/** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
name: string;
/**
* The payload. For `source = url`: the URL string. For `source = inline`: the
* raw CSV/TSV text, or the parsed JSON/TopoJSON value.
* The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON
* value. For `source = url` this is the fetched snapshot, or `null` before the
* first successful fetch.
*/
data: unknown;
/** One of `json`, `csv`, `tsv`, `topojson`. */
format: DataFormat;
/** One of `inline` or `url`. */
source: DataSource;
/**
* For `source = url`: the remote address the snapshot was fetched from, retained
* so the dataset can be re-fetched ("Refresh"). Absent for inline datasets.
*/
url?: string;
/**
* For `source = url`: ISO timestamp of the last successful fetch, or `null` when
* it has never been fetched. Absent for inline datasets.
*/
fetchedAt?: string | null;
/** Free-form user note about the dataset. */
comment: string;
/** Data rows, or `null` when N/A (URL / non-tabular). */
@@ -187,26 +205,52 @@ function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
}
/**
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
* tabular vs N/A by source/format, and delegate to `profileData`.
*
* - `url` (any format) → N/A profile; size = byte length of the URL string.
* - inline `json` → rows when a non-empty array of objects, else N/A.
* - inline `topojson` → N/A (non-tabular).
* - inline `csv` / `tsv` → `parseDelimited` rows.
*
* `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
* `JSON.stringify(data)` for json/topojson.
* The tabular rows of a dataset payload for a table preview, or `null` when the
* payload isn't tabular (a single JSON object, TopoJSON, or an unfetched URL). Uses
* the **same** parsing as profiling — `parseDelimited` for CSV/TSV, `asObjectRows`
* for JSON — so the previewed rows agree exactly with the profiled `columns`. A
* positive `limit` returns only the head (a preview needs a sample, not the whole
* payload). Returns non-null on precisely the inputs `computeDatasetProfile` counts
* as rows, so a caller can gate "table vs. raw text" on this alone.
*/
export function computeDatasetProfile(
export function tabularRows(
data: unknown,
format: DataFormat,
source: DataSource,
): DatasetProfile {
if (source === 'url') {
const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
return profileData(null, byteLength(url));
limit?: number,
): Array<Record<string, unknown>> | null {
if (data == null) return null;
let rows: Array<Record<string, unknown>> | null;
switch (format) {
case 'csv':
case 'tsv':
rows = parseDelimited(typeof data === 'string' ? data : '', format);
break;
case 'json':
rows = asObjectRows(data);
break;
default:
rows = null;
}
if (!rows || rows.length === 0) return null;
return limit != null && limit >= 0 && rows.length > limit ? rows.slice(0, limit) : rows;
}
/**
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
* tabular vs N/A by format, and delegate to `profileData`. Identical for inline
* data and for a fetched URL snapshot — both carry the payload in `data`.
*
* - `null` data (unfetched URL) → N/A profile, size 0.
* - `json` → rows when a non-empty array of objects, else N/A.
* - `topojson` → N/A (non-tabular).
* - `csv` / `tsv` → `parseDelimited` rows.
*
* `size` is the UTF-8 byte length of the raw string for csv/tsv, or of
* `JSON.stringify(data)` for json/topojson.
*/
export function computeDatasetProfile(data: unknown, format: DataFormat): DatasetProfile {
// An unfetched URL reference (or a genuinely absent payload): nothing to profile.
if (data == null) return profileData(null, 0);
switch (format) {
case 'csv':
@@ -226,6 +270,29 @@ export function computeDatasetProfile(
}
}
/**
* Shape a freshly-fetched URL body into the `{ data, format }` a snapshot stores
* (spec §05 → URL datasets, snapshot model). Format is sniffed from the **content**
* first — authoritative, since `detectFormat` only reports `json` when the body
* actually parses — falling back to the URL's file extension, then JSON.
* JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text — the same
* per-format shaping inline data uses, so a fetched dataset profiles and renders
* identically to an inline one (see `computeDatasetProfile`, rendering.ts).
*/
export function snapshotFromText(text: string, url: string): { data: unknown; format: DataFormat } {
const format = detectFormat(text).format ?? detectFormatFromUrl(url) ?? 'json';
if (format === 'json' || format === 'topojson') {
try {
return { data: JSON.parse(text) as unknown, format };
} catch {
// The extension promised JSON but the body isn't — keep the raw text so the
// render surfaces a readable error instead of us throwing mid-commit.
return { data: text, format };
}
}
return { data: text, format };
}
export interface CreateDatasetOptions {
/** The dataset name (uniqueness is enforced upstream — see naming.ts). */
name: string;
@@ -235,6 +302,10 @@ export interface CreateDatasetOptions {
format: DataFormat;
/** One of `inline` or `url`. */
source: DataSource;
/** For `source = url`: the remote address (retained for Refresh). */
url?: string;
/** For `source = url`: ISO timestamp of the fetch that produced `data`. */
fetchedAt?: string | null;
/** Optional free-form note. */
comment?: string;
/** Clock injection for deterministic tests; defaults to the current time. */
@@ -253,7 +324,7 @@ export interface CreateDatasetOptions {
export function createDataset(options: CreateDatasetOptions): Dataset {
const now = options.now ?? new Date();
const iso = now.toISOString();
const profile = computeDatasetProfile(options.data, options.format, options.source);
const profile = computeDatasetProfile(options.data, options.format);
return {
id: options.id ?? Date.now(),
@@ -262,6 +333,8 @@ export function createDataset(options: CreateDatasetOptions): Dataset {
data: options.data,
format: options.format,
source: options.source,
// URL datasets carry their address + fetch time; inline records stay clean.
...(options.source === 'url' ? { url: options.url, fetchedAt: options.fetchedAt ?? null } : {}),
comment: options.comment ?? '',
rowCount: profile.rowCount,
columnCount: profile.columnCount,
+36
View File
@@ -233,6 +233,42 @@ describe('normalizeImport — dataset normalization', () => {
expect(d.version).toBe(CURRENT_DATASET_VERSION);
});
it('v1→v2: a legacy URL dataset (address in data) imports as an unfetched reference', () => {
const parsed = {
version: '1.0',
snippets: [],
datasets: [{ id: 1, name: 'Remote', source: 'url', format: 'csv', data: 'https://x/y.csv' }],
};
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
expect(d.source).toBe('url');
expect(d.url).toBe('https://x/y.csv');
expect(d.data).toBeNull();
expect(d.fetchedAt).toBeNull();
expect(d.version).toBe(CURRENT_DATASET_VERSION);
});
it('a v2 URL snapshot imports with its data, url, and fetchedAt intact', () => {
const parsed = {
version: '1.0',
snippets: [],
datasets: [
{
id: 1,
name: 'Remote',
source: 'url',
format: 'csv',
data: 'a,b\n1,2',
url: 'https://x/y.csv',
fetchedAt: '2026-06-10T00:00:00.000Z',
},
],
};
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
expect(d.data).toBe('a,b\n1,2');
expect(d.url).toBe('https://x/y.csv');
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
});
it('fills gaps with defaults and coerces id to a number', () => {
const parsed = { version: '1.0', snippets: [], datasets: [{ id: '42', name: 'D' }] };
const result = normalizeImport(parsed, { now: FIXED_NOW });
+19 -2
View File
@@ -151,6 +151,11 @@ function normalizeSnippet(raw: unknown, nowIso: string, makeId: () => string): S
* datasets already carry their derived summary fields (rowCount, columns, …); we
* preserve those and only fill gaps. We do NOT re-profile here — that needs the
* profiling pipeline and would be wasteful for already-summarized records.
*
* Applies the v1→v2 URL-snapshot shaping (mirrors `migrateDataset`): a pre-v2 export
* stored a URL dataset's address in `data`, so we move it into `url` and clear `data`
* to `null` — importing such a record yields an unfetched reference, not a record
* with a URL string masquerading as its snapshot.
*/
function normalizeDataset(raw: unknown, nowIso: string): Dataset {
const r = isPlainObject(raw) ? raw : {};
@@ -158,13 +163,25 @@ function normalizeDataset(raw: unknown, nowIso: string): Dataset {
const created = asNonEmptyString(r.created) ?? nowIso;
const modified = asNonEmptyString(r.modified) ?? created;
const source = (typeof r.source === 'string' ? r.source : 'inline') as DataSource;
const legacyUrl = source === 'url' && typeof r.url !== 'string';
const url = legacyUrl
? typeof r.data === 'string'
? r.data
: undefined
: typeof r.url === 'string'
? r.url
: undefined;
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
return {
id: typeof r.id === 'number' ? r.id : Number(r.id) || 0,
version: CURRENT_DATASET_VERSION,
name: typeof r.name === 'string' ? r.name : 'Untitled',
data: r.data,
data: legacyUrl ? null : r.data,
format: (typeof r.format === 'string' ? r.format : 'json') as DataFormat,
source: (typeof r.source === 'string' ? r.source : 'inline') as DataSource,
source,
...(source === 'url' ? { url, fetchedAt } : {}),
comment: typeof r.comment === 'string' ? r.comment : '',
rowCount: typeof r.rowCount === 'number' ? r.rowCount : null,
columnCount: typeof r.columnCount === 'number' ? r.columnCount : null,
+17 -3
View File
@@ -111,7 +111,16 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
format: 'topojson',
source: 'inline',
},
{ name: 'UrlDs', data: 'https://x/y.csv', format: 'csv', source: 'url' },
// A fetched URL snapshot carries its payload in `data` (like inline); an
// unfetched reference has `data: null` and only its `url`.
{
name: 'UrlFetchedDs',
data: 'a,b\n1,2',
url: 'https://x/y.csv',
format: 'csv',
source: 'url',
},
{ name: 'UrlUnfetchedDs', data: null, url: 'https://x/y.csv', format: 'csv', source: 'url' },
];
test('inline JSON → values inlined', () => {
@@ -140,8 +149,13 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
});
});
test('URL → url reference tagged with the dataset format', () => {
const out = prepareSpecForRender({ data: { name: 'UrlDs' } }, { datasets });
test('fetched URL snapshot → its payload inlined, tagged with the format (like inline)', () => {
const out = prepareSpecForRender({ data: { name: 'UrlFetchedDs' } }, { datasets });
expect(out.data).toEqual({ values: 'a,b\n1,2', format: { type: 'csv' } });
});
test('unfetched URL reference → live url fallback tagged with the format', () => {
const out = prepareSpecForRender({ data: { name: 'UrlUnfetchedDs' } }, { datasets });
expect(out.data).toEqual({ url: 'https://x/y.csv', format: { type: 'csv' } });
});
+11 -2
View File
@@ -42,6 +42,8 @@ export interface ResolvableDataset {
format: DataFormat;
/** One of `inline` or `url`. */
source: DataSource;
/** For `source = url`: the remote address, used only as the unfetched fallback. */
url?: string;
}
/** Thrown when a spec references a library dataset name that does not exist. */
@@ -129,13 +131,20 @@ function selfDefinedDatasetNames(spec: unknown): Set<string> {
*/
function resolvedData(dataset: ResolvableDataset, rest: SpecNode): SpecNode {
const restFormat = isSpecNode(rest.format) ? rest.format : {};
if (dataset.source === 'url') {
// An unfetched URL reference (a legacy record, or one whose snapshot fetch never
// succeeded) has no local data — fall back to a live Vega-Lite URL fetch so it
// still renders until a Refresh snapshots it (snapshot model; spec §04 step 1).
if (dataset.source === 'url' && dataset.data == null) {
return {
...rest,
url: typeof dataset.data === 'string' ? dataset.data : (JSON.stringify(dataset.data) ?? ''),
url: typeof dataset.url === 'string' ? dataset.url : '',
format: { ...restFormat, type: dataset.format },
};
}
// Inline data and fetched URL snapshots resolve identically: the payload is in
// `data`, shaped by format.
switch (dataset.format) {
case 'json':
return { ...rest, values: dataset.data };