mirror of
https://github.com/olehomelchenko/astrolabe.git
synced 2026-08-08 02:02:33 +00:00
Snapshot URL datasets locally on add; preview tabular data as a table
This commit is contained in:
+122
-21
@@ -5,6 +5,8 @@ import {
|
||||
createDataset,
|
||||
datasetReference,
|
||||
parseDelimited,
|
||||
snapshotFromText,
|
||||
tabularRows,
|
||||
} from './dataset';
|
||||
|
||||
describe('datasetReference', () => {
|
||||
@@ -65,51 +67,125 @@ describe('parseDelimited', () => {
|
||||
});
|
||||
|
||||
describe('computeDatasetProfile', () => {
|
||||
test('inline JSON array-of-objects is profiled', () => {
|
||||
test('JSON array-of-objects is profiled', () => {
|
||||
const data = [
|
||||
{ a: 1, b: 'x' },
|
||||
{ a: 2, b: 'y' },
|
||||
];
|
||||
const profile = computeDatasetProfile(data, 'json', 'inline');
|
||||
const profile = computeDatasetProfile(data, 'json');
|
||||
expect(profile.rowCount).toBe(2);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(data)).length);
|
||||
});
|
||||
|
||||
test('inline JSON that is not an array is N/A but sized', () => {
|
||||
const profile = computeDatasetProfile({ a: 1 }, 'json', 'inline');
|
||||
test('JSON that is not an array is N/A but sized', () => {
|
||||
const profile = computeDatasetProfile({ a: 1 }, 'json');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('inline CSV is parsed and profiled; size is the raw text byte length', () => {
|
||||
test('CSV is parsed and profiled; size is the raw text byte length', () => {
|
||||
const text = 'a,b\n1,2';
|
||||
const profile = computeDatasetProfile(text, 'csv', 'inline');
|
||||
const profile = computeDatasetProfile(text, 'csv');
|
||||
expect(profile.rowCount).toBe(1);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
expect(profile.size).toBe(new TextEncoder().encode(text).length);
|
||||
});
|
||||
|
||||
test('inline TSV is parsed and profiled', () => {
|
||||
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv', 'inline');
|
||||
test('TSV is parsed and profiled', () => {
|
||||
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv');
|
||||
expect(profile.rowCount).toBe(1);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('inline TopoJSON is N/A (non-tabular) but sized', () => {
|
||||
test('TopoJSON is N/A (non-tabular) but sized', () => {
|
||||
const topo = { type: 'Topology', objects: {} };
|
||||
const profile = computeDatasetProfile(topo, 'topojson', 'inline');
|
||||
const profile = computeDatasetProfile(topo, 'topojson');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(topo)).length);
|
||||
});
|
||||
|
||||
test('URL is N/A; size is the byte length of the URL string', () => {
|
||||
const url = 'https://example.com/data.csv';
|
||||
const profile = computeDatasetProfile(url, 'csv', 'url');
|
||||
test('a fetched URL snapshot is profiled exactly like inline data', () => {
|
||||
// The snapshot model: once fetched, a URL dataset carries its payload in `data`
|
||||
// and profiles through the same path as inline (no source distinction here).
|
||||
const profile = computeDatasetProfile('a,b\n1,2\n3,4', 'csv');
|
||||
expect(profile.rowCount).toBe(2);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('an unfetched URL reference (null data) is N/A with size 0', () => {
|
||||
const profile = computeDatasetProfile(null, 'csv');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.size).toBe(new TextEncoder().encode(url).length);
|
||||
expect(profile.columns).toEqual([]);
|
||||
expect(profile.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('tabularRows', () => {
|
||||
test('CSV parses to rows (same parsing as profiling)', () => {
|
||||
expect(tabularRows('a,b\n1,2\n3,4', 'csv')).toEqual([
|
||||
{ a: '1', b: '2' },
|
||||
{ a: '3', b: '4' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('TSV parses to rows', () => {
|
||||
expect(tabularRows('x\ty\n1\t2', 'tsv')).toEqual([{ x: '1', y: '2' }]);
|
||||
});
|
||||
|
||||
test('a JSON array of objects is returned as-is', () => {
|
||||
const data = [{ a: 1 }, { a: 2 }];
|
||||
expect(tabularRows(data, 'json')).toEqual(data);
|
||||
});
|
||||
|
||||
test('limit returns only the head', () => {
|
||||
const rows = tabularRows('n\n1\n2\n3\n4', 'csv', 2);
|
||||
expect(rows).toEqual([{ n: '1' }, { n: '2' }]);
|
||||
});
|
||||
|
||||
test('non-tabular payloads return null (single object, topojson, unfetched)', () => {
|
||||
expect(tabularRows({ a: 1 }, 'json')).toBeNull();
|
||||
expect(tabularRows({ type: 'Topology', objects: {} }, 'topojson')).toBeNull();
|
||||
expect(tabularRows(null, 'csv')).toBeNull();
|
||||
expect(tabularRows('a,b', 'csv')).toBeNull(); // header only — no rows
|
||||
});
|
||||
});
|
||||
|
||||
describe('snapshotFromText', () => {
|
||||
test('JSON content is parsed and typed json (content beats extension)', () => {
|
||||
const { data, format } = snapshotFromText('[{"a":1}]', 'https://x/data.txt');
|
||||
expect(format).toBe('json');
|
||||
expect(data).toEqual([{ a: 1 }]);
|
||||
});
|
||||
|
||||
test('CSV content keeps its raw text and is typed csv', () => {
|
||||
const { data, format } = snapshotFromText('a,b\n1,2', 'https://x/data.csv');
|
||||
expect(format).toBe('csv');
|
||||
expect(data).toBe('a,b\n1,2');
|
||||
});
|
||||
|
||||
test('TSV content is typed tsv', () => {
|
||||
expect(snapshotFromText('a\tb\n1\t2', 'https://x/d').format).toBe('tsv');
|
||||
});
|
||||
|
||||
test('a Topology body is typed topojson and parsed', () => {
|
||||
const { data, format } = snapshotFromText('{"type":"Topology","objects":{}}', 'https://x/d');
|
||||
expect(format).toBe('topojson');
|
||||
expect(data).toEqual({ type: 'Topology', objects: {} });
|
||||
});
|
||||
|
||||
test('unrecognized content falls back to the URL extension', () => {
|
||||
// A single token: not JSON, not delimited. The .json extension decides the
|
||||
// format; the unparseable body is kept verbatim rather than throwing.
|
||||
const { data, format } = snapshotFromText('not-data', 'https://x/data.json');
|
||||
expect(format).toBe('json');
|
||||
expect(data).toBe('not-data');
|
||||
});
|
||||
|
||||
test('unrecognized content with no useful extension defaults to json', () => {
|
||||
expect(snapshotFromText('not-data', 'https://x/feed').format).toBe('json');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -157,20 +233,45 @@ describe('createDataset', () => {
|
||||
expect(d.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('URL dataset gets an N/A profile with a size', () => {
|
||||
test('URL dataset carries url + fetchedAt and profiles the fetched snapshot', () => {
|
||||
const d = createDataset({
|
||||
id: 2,
|
||||
name: 'Remote',
|
||||
data: 'https://example.com/x.json',
|
||||
format: 'json',
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://example.com/x.csv',
|
||||
fetchedAt: '2026-06-10T00:00:00.000Z',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
comment: 'remote source',
|
||||
});
|
||||
expect(d.rowCount).toBeNull();
|
||||
expect(d.columnCount).toBeNull();
|
||||
expect(d.columns).toEqual([]);
|
||||
expect(d.source).toBe('url');
|
||||
expect(d.url).toBe('https://example.com/x.csv');
|
||||
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
|
||||
expect(d.rowCount).toBe(1);
|
||||
expect(d.columns).toEqual(['a', 'b']);
|
||||
expect(d.comment).toBe('remote source');
|
||||
expect(d.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('an unfetched URL dataset (null data) gets an N/A profile and null fetchedAt', () => {
|
||||
const d = createDataset({
|
||||
id: 3,
|
||||
name: 'Unfetched',
|
||||
data: null,
|
||||
url: 'https://example.com/x.csv',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
});
|
||||
expect(d.url).toBe('https://example.com/x.csv');
|
||||
expect(d.fetchedAt).toBeNull();
|
||||
expect(d.rowCount).toBeNull();
|
||||
expect(d.columns).toEqual([]);
|
||||
expect(d.size).toBe(0);
|
||||
});
|
||||
|
||||
test('inline dataset carries no url/fetchedAt keys', () => {
|
||||
const d = createDataset({ id: 4, name: 'I', data: [], format: 'json', source: 'inline' });
|
||||
expect('url' in d).toBe(false);
|
||||
expect('fetchedAt' in d).toBe(false);
|
||||
});
|
||||
|
||||
test('defaults the id when not injected', () => {
|
||||
|
||||
+102
-29
@@ -7,22 +7,29 @@
|
||||
* a simple delimited-text parser, the profiling orchestration, and a factory that
|
||||
* stamps timestamps/version and fills the derived summary fields.
|
||||
*
|
||||
* A dataset has one of two **sources** — `inline` (data stored in the record) or
|
||||
* `url` (only the link is stored, fetched on demand at render time) — and one of
|
||||
* four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
|
||||
* The `data` field's shape follows source/format: a URL string for `url`; raw
|
||||
* text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
|
||||
* A dataset has one of two **sources** — `inline` (data pasted into the record) or
|
||||
* `url` (fetched once from a remote address and **snapshotted** into the record) —
|
||||
* and one of four **formats** (reused from format-detection: `json`/`csv`/`tsv`/
|
||||
* `topojson`). Either way `data` holds the actual payload, shaped by format: raw
|
||||
* text for CSV/TSV, a parsed value for JSON/TopoJSON. A `url` dataset additionally
|
||||
* keeps its source `url` (so it can be re-fetched) and a `fetchedAt` timestamp;
|
||||
* until its first successful fetch `data` is `null` (an unfetched reference).
|
||||
*
|
||||
* Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
|
||||
* non-tabular data get an N/A profile but are still sized (see profile.ts).
|
||||
* Any tabular payload (JSON array-of-objects, CSV, TSV) is profiled — including a
|
||||
* fetched URL snapshot; non-tabular or not-yet-fetched data gets an N/A profile but
|
||||
* is still sized (see profile.ts).
|
||||
*/
|
||||
|
||||
import type { DataFormat } from './format-detection';
|
||||
import { detectFormat, detectFormatFromUrl, type DataFormat } from './format-detection';
|
||||
import { profileData, type ColumnStats, type DatasetProfile } from './profile';
|
||||
import type { ColumnType } from './type-inference';
|
||||
|
||||
/** Current schema version for a Dataset record (read-time migration target). */
|
||||
export const CURRENT_DATASET_VERSION = 1;
|
||||
/**
|
||||
* Current schema version for a Dataset record (read-time migration target).
|
||||
* v2 moved a URL dataset's address out of `data` into its own `url` field and made
|
||||
* `data` hold the fetched snapshot (see `migrateDataset`).
|
||||
*/
|
||||
export const CURRENT_DATASET_VERSION = 2;
|
||||
|
||||
/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
|
||||
export type DataSource = 'inline' | 'url';
|
||||
@@ -35,14 +42,25 @@ export interface Dataset {
|
||||
/** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
|
||||
name: string;
|
||||
/**
|
||||
* The payload. For `source = url`: the URL string. For `source = inline`: the
|
||||
* raw CSV/TSV text, or the parsed JSON/TopoJSON value.
|
||||
* The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON
|
||||
* value. For `source = url` this is the fetched snapshot, or `null` before the
|
||||
* first successful fetch.
|
||||
*/
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/**
|
||||
* For `source = url`: the remote address the snapshot was fetched from, retained
|
||||
* so the dataset can be re-fetched ("Refresh"). Absent for inline datasets.
|
||||
*/
|
||||
url?: string;
|
||||
/**
|
||||
* For `source = url`: ISO timestamp of the last successful fetch, or `null` when
|
||||
* it has never been fetched. Absent for inline datasets.
|
||||
*/
|
||||
fetchedAt?: string | null;
|
||||
/** Free-form user note about the dataset. */
|
||||
comment: string;
|
||||
/** Data rows, or `null` when N/A (URL / non-tabular). */
|
||||
@@ -187,26 +205,52 @@ function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
|
||||
* tabular vs N/A by source/format, and delegate to `profileData`.
|
||||
*
|
||||
* - `url` (any format) → N/A profile; size = byte length of the URL string.
|
||||
* - inline `json` → rows when a non-empty array of objects, else N/A.
|
||||
* - inline `topojson` → N/A (non-tabular).
|
||||
* - inline `csv` / `tsv` → `parseDelimited` rows.
|
||||
*
|
||||
* `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
|
||||
* `JSON.stringify(data)` for json/topojson.
|
||||
* The tabular rows of a dataset payload for a table preview, or `null` when the
|
||||
* payload isn't tabular (a single JSON object, TopoJSON, or an unfetched URL). Uses
|
||||
* the **same** parsing as profiling — `parseDelimited` for CSV/TSV, `asObjectRows`
|
||||
* for JSON — so the previewed rows agree exactly with the profiled `columns`. A
|
||||
* positive `limit` returns only the head (a preview needs a sample, not the whole
|
||||
* payload). Returns non-null on precisely the inputs `computeDatasetProfile` counts
|
||||
* as rows, so a caller can gate "table vs. raw text" on this alone.
|
||||
*/
|
||||
export function computeDatasetProfile(
|
||||
export function tabularRows(
|
||||
data: unknown,
|
||||
format: DataFormat,
|
||||
source: DataSource,
|
||||
): DatasetProfile {
|
||||
if (source === 'url') {
|
||||
const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
|
||||
return profileData(null, byteLength(url));
|
||||
limit?: number,
|
||||
): Array<Record<string, unknown>> | null {
|
||||
if (data == null) return null;
|
||||
let rows: Array<Record<string, unknown>> | null;
|
||||
switch (format) {
|
||||
case 'csv':
|
||||
case 'tsv':
|
||||
rows = parseDelimited(typeof data === 'string' ? data : '', format);
|
||||
break;
|
||||
case 'json':
|
||||
rows = asObjectRows(data);
|
||||
break;
|
||||
default:
|
||||
rows = null;
|
||||
}
|
||||
if (!rows || rows.length === 0) return null;
|
||||
return limit != null && limit >= 0 && rows.length > limit ? rows.slice(0, limit) : rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
|
||||
* tabular vs N/A by format, and delegate to `profileData`. Identical for inline
|
||||
* data and for a fetched URL snapshot — both carry the payload in `data`.
|
||||
*
|
||||
* - `null` data (unfetched URL) → N/A profile, size 0.
|
||||
* - `json` → rows when a non-empty array of objects, else N/A.
|
||||
* - `topojson` → N/A (non-tabular).
|
||||
* - `csv` / `tsv` → `parseDelimited` rows.
|
||||
*
|
||||
* `size` is the UTF-8 byte length of the raw string for csv/tsv, or of
|
||||
* `JSON.stringify(data)` for json/topojson.
|
||||
*/
|
||||
export function computeDatasetProfile(data: unknown, format: DataFormat): DatasetProfile {
|
||||
// An unfetched URL reference (or a genuinely absent payload): nothing to profile.
|
||||
if (data == null) return profileData(null, 0);
|
||||
|
||||
switch (format) {
|
||||
case 'csv':
|
||||
@@ -226,6 +270,29 @@ export function computeDatasetProfile(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shape a freshly-fetched URL body into the `{ data, format }` a snapshot stores
|
||||
* (spec §05 → URL datasets, snapshot model). Format is sniffed from the **content**
|
||||
* first — authoritative, since `detectFormat` only reports `json` when the body
|
||||
* actually parses — falling back to the URL's file extension, then JSON.
|
||||
* JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text — the same
|
||||
* per-format shaping inline data uses, so a fetched dataset profiles and renders
|
||||
* identically to an inline one (see `computeDatasetProfile`, rendering.ts).
|
||||
*/
|
||||
export function snapshotFromText(text: string, url: string): { data: unknown; format: DataFormat } {
|
||||
const format = detectFormat(text).format ?? detectFormatFromUrl(url) ?? 'json';
|
||||
if (format === 'json' || format === 'topojson') {
|
||||
try {
|
||||
return { data: JSON.parse(text) as unknown, format };
|
||||
} catch {
|
||||
// The extension promised JSON but the body isn't — keep the raw text so the
|
||||
// render surfaces a readable error instead of us throwing mid-commit.
|
||||
return { data: text, format };
|
||||
}
|
||||
}
|
||||
return { data: text, format };
|
||||
}
|
||||
|
||||
export interface CreateDatasetOptions {
|
||||
/** The dataset name (uniqueness is enforced upstream — see naming.ts). */
|
||||
name: string;
|
||||
@@ -235,6 +302,10 @@ export interface CreateDatasetOptions {
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** For `source = url`: the remote address (retained for Refresh). */
|
||||
url?: string;
|
||||
/** For `source = url`: ISO timestamp of the fetch that produced `data`. */
|
||||
fetchedAt?: string | null;
|
||||
/** Optional free-form note. */
|
||||
comment?: string;
|
||||
/** Clock injection for deterministic tests; defaults to the current time. */
|
||||
@@ -253,7 +324,7 @@ export interface CreateDatasetOptions {
|
||||
export function createDataset(options: CreateDatasetOptions): Dataset {
|
||||
const now = options.now ?? new Date();
|
||||
const iso = now.toISOString();
|
||||
const profile = computeDatasetProfile(options.data, options.format, options.source);
|
||||
const profile = computeDatasetProfile(options.data, options.format);
|
||||
|
||||
return {
|
||||
id: options.id ?? Date.now(),
|
||||
@@ -262,6 +333,8 @@ export function createDataset(options: CreateDatasetOptions): Dataset {
|
||||
data: options.data,
|
||||
format: options.format,
|
||||
source: options.source,
|
||||
// URL datasets carry their address + fetch time; inline records stay clean.
|
||||
...(options.source === 'url' ? { url: options.url, fetchedAt: options.fetchedAt ?? null } : {}),
|
||||
comment: options.comment ?? '',
|
||||
rowCount: profile.rowCount,
|
||||
columnCount: profile.columnCount,
|
||||
|
||||
@@ -233,6 +233,42 @@ describe('normalizeImport — dataset normalization', () => {
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
});
|
||||
|
||||
it('v1→v2: a legacy URL dataset (address in data) imports as an unfetched reference', () => {
|
||||
const parsed = {
|
||||
version: '1.0',
|
||||
snippets: [],
|
||||
datasets: [{ id: 1, name: 'Remote', source: 'url', format: 'csv', data: 'https://x/y.csv' }],
|
||||
};
|
||||
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
|
||||
expect(d.source).toBe('url');
|
||||
expect(d.url).toBe('https://x/y.csv');
|
||||
expect(d.data).toBeNull();
|
||||
expect(d.fetchedAt).toBeNull();
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
});
|
||||
|
||||
it('a v2 URL snapshot imports with its data, url, and fetchedAt intact', () => {
|
||||
const parsed = {
|
||||
version: '1.0',
|
||||
snippets: [],
|
||||
datasets: [
|
||||
{
|
||||
id: 1,
|
||||
name: 'Remote',
|
||||
source: 'url',
|
||||
format: 'csv',
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://x/y.csv',
|
||||
fetchedAt: '2026-06-10T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
};
|
||||
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
|
||||
expect(d.data).toBe('a,b\n1,2');
|
||||
expect(d.url).toBe('https://x/y.csv');
|
||||
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('fills gaps with defaults and coerces id to a number', () => {
|
||||
const parsed = { version: '1.0', snippets: [], datasets: [{ id: '42', name: 'D' }] };
|
||||
const result = normalizeImport(parsed, { now: FIXED_NOW });
|
||||
|
||||
@@ -151,6 +151,11 @@ function normalizeSnippet(raw: unknown, nowIso: string, makeId: () => string): S
|
||||
* datasets already carry their derived summary fields (rowCount, columns, …); we
|
||||
* preserve those and only fill gaps. We do NOT re-profile here — that needs the
|
||||
* profiling pipeline and would be wasteful for already-summarized records.
|
||||
*
|
||||
* Applies the v1→v2 URL-snapshot shaping (mirrors `migrateDataset`): a pre-v2 export
|
||||
* stored a URL dataset's address in `data`, so we move it into `url` and clear `data`
|
||||
* to `null` — importing such a record yields an unfetched reference, not a record
|
||||
* with a URL string masquerading as its snapshot.
|
||||
*/
|
||||
function normalizeDataset(raw: unknown, nowIso: string): Dataset {
|
||||
const r = isPlainObject(raw) ? raw : {};
|
||||
@@ -158,13 +163,25 @@ function normalizeDataset(raw: unknown, nowIso: string): Dataset {
|
||||
const created = asNonEmptyString(r.created) ?? nowIso;
|
||||
const modified = asNonEmptyString(r.modified) ?? created;
|
||||
|
||||
const source = (typeof r.source === 'string' ? r.source : 'inline') as DataSource;
|
||||
const legacyUrl = source === 'url' && typeof r.url !== 'string';
|
||||
const url = legacyUrl
|
||||
? typeof r.data === 'string'
|
||||
? r.data
|
||||
: undefined
|
||||
: typeof r.url === 'string'
|
||||
? r.url
|
||||
: undefined;
|
||||
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
|
||||
|
||||
return {
|
||||
id: typeof r.id === 'number' ? r.id : Number(r.id) || 0,
|
||||
version: CURRENT_DATASET_VERSION,
|
||||
name: typeof r.name === 'string' ? r.name : 'Untitled',
|
||||
data: r.data,
|
||||
data: legacyUrl ? null : r.data,
|
||||
format: (typeof r.format === 'string' ? r.format : 'json') as DataFormat,
|
||||
source: (typeof r.source === 'string' ? r.source : 'inline') as DataSource,
|
||||
source,
|
||||
...(source === 'url' ? { url, fetchedAt } : {}),
|
||||
comment: typeof r.comment === 'string' ? r.comment : '',
|
||||
rowCount: typeof r.rowCount === 'number' ? r.rowCount : null,
|
||||
columnCount: typeof r.columnCount === 'number' ? r.columnCount : null,
|
||||
|
||||
@@ -111,7 +111,16 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
|
||||
format: 'topojson',
|
||||
source: 'inline',
|
||||
},
|
||||
{ name: 'UrlDs', data: 'https://x/y.csv', format: 'csv', source: 'url' },
|
||||
// A fetched URL snapshot carries its payload in `data` (like inline); an
|
||||
// unfetched reference has `data: null` and only its `url`.
|
||||
{
|
||||
name: 'UrlFetchedDs',
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://x/y.csv',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
},
|
||||
{ name: 'UrlUnfetchedDs', data: null, url: 'https://x/y.csv', format: 'csv', source: 'url' },
|
||||
];
|
||||
|
||||
test('inline JSON → values inlined', () => {
|
||||
@@ -140,8 +149,13 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
|
||||
});
|
||||
});
|
||||
|
||||
test('URL → url reference tagged with the dataset format', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlDs' } }, { datasets });
|
||||
test('fetched URL snapshot → its payload inlined, tagged with the format (like inline)', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlFetchedDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ values: 'a,b\n1,2', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
test('unfetched URL reference → live url fallback tagged with the format', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlUnfetchedDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ url: 'https://x/y.csv', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
|
||||
+11
-2
@@ -42,6 +42,8 @@ export interface ResolvableDataset {
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** For `source = url`: the remote address, used only as the unfetched fallback. */
|
||||
url?: string;
|
||||
}
|
||||
|
||||
/** Thrown when a spec references a library dataset name that does not exist. */
|
||||
@@ -129,13 +131,20 @@ function selfDefinedDatasetNames(spec: unknown): Set<string> {
|
||||
*/
|
||||
function resolvedData(dataset: ResolvableDataset, rest: SpecNode): SpecNode {
|
||||
const restFormat = isSpecNode(rest.format) ? rest.format : {};
|
||||
if (dataset.source === 'url') {
|
||||
|
||||
// An unfetched URL reference (a legacy record, or one whose snapshot fetch never
|
||||
// succeeded) has no local data — fall back to a live Vega-Lite URL fetch so it
|
||||
// still renders until a Refresh snapshots it (snapshot model; spec §04 step 1).
|
||||
if (dataset.source === 'url' && dataset.data == null) {
|
||||
return {
|
||||
...rest,
|
||||
url: typeof dataset.data === 'string' ? dataset.data : (JSON.stringify(dataset.data) ?? ''),
|
||||
url: typeof dataset.url === 'string' ? dataset.url : '',
|
||||
format: { ...restFormat, type: dataset.format },
|
||||
};
|
||||
}
|
||||
|
||||
// Inline data and fetched URL snapshots resolve identically: the payload is in
|
||||
// `data`, shaped by format.
|
||||
switch (dataset.format) {
|
||||
case 'json':
|
||||
return { ...rest, values: dataset.data };
|
||||
|
||||
Reference in New Issue
Block a user