Snapshot URL datasets locally on add; preview tabular data as a table

This commit is contained in:
2026-06-10 10:24:42 +03:00
parent eb5e7ac53a
commit 2410c6e965
23 changed files with 1239 additions and 148 deletions
@@ -235,6 +235,32 @@
color: var(--text-secondary);
}
/* URL dataset provenance: the source address + last-fetched time (spec §05). */
.sourceMeta {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: var(--space-2) var(--space-4);
margin: 0;
font-size: 12px;
color: var(--text-secondary);
}
.sourceLink {
color: var(--accent);
text-decoration: none;
overflow-wrap: anywhere;
}
.sourceLink:hover {
text-decoration: underline;
}
.sourceLink:focus-visible {
outline: 2px solid var(--focus);
outline-offset: 2px;
}
.section {
display: flex;
flex-direction: column;
@@ -335,6 +361,46 @@
white-space: pre;
}
/* Tabular preview — a scrollable grid with a sticky header (spec §05 → Preview). */
.previewTableWrap {
max-height: 260px;
overflow: auto;
border: var(--border-width) solid var(--border);
border-radius: var(--radius);
}
.previewTable {
border-collapse: collapse;
width: 100%;
font-family: var(--font-mono);
font-size: 12px;
line-height: 1.4;
}
.previewTable th,
.previewTable td {
text-align: left;
padding: var(--space-2) var(--space-3);
border-bottom: var(--border-width) solid var(--border);
white-space: nowrap;
max-width: 240px;
overflow: hidden;
text-overflow: ellipsis;
}
.previewTable thead th {
position: sticky;
top: 0;
z-index: 1;
background: var(--layer-01);
font-weight: 600;
color: var(--text-secondary);
}
.previewTable tbody tr:last-child td {
border-bottom: none;
}
.muted {
margin: 0;
font-size: 13px;
@@ -439,3 +505,10 @@
font-size: 13px;
color: var(--support-error);
}
/* The inline-fallback recovery under a failed URL fetch — left-aligned, not
stretched, so it reads as a secondary recovery beneath the error message. */
.fallback {
align-self: flex-start;
margin-top: calc(-1 * var(--space-2));
}
+187 -22
View File
@@ -12,12 +12,14 @@
* Snippets stay reactive without a stored back-pointer.
*/
import { useState } from 'react';
import { useMemo, useState } from 'react';
import { useShallow } from 'zustand/react/shallow';
import { datasetReference, type DataSource, type Dataset } from '@core/dataset';
import { datasetReference, tabularRows, type DataSource, type Dataset } from '@core/dataset';
import { detectFormat, detectFormatFromUrl, type DataFormat } from '@core/format-detection';
import { datasetUsageCounts, snippetsReferencingDataset } from '@core/relationships';
import { closeModal, openModal, resnapshot } from '../modals/ModalCoordinator';
import { fetchRemoteData } from '../infrastructure/remote-data';
import { remoteFetchErrorMessage } from '../services/remote-data-errors';
import { confirm } from '../stores/ConfirmStore';
import { notify } from '../stores/NotificationStore';
import {
@@ -50,6 +52,17 @@ const SOURCE_OPTIONS: ReadonlyArray<SegmentedOption<DataSource>> = [
{ value: 'url', label: 'URL' },
];
/** Rows shown in the tabular preview before truncating (spec §05 → Detail Panel). */
const PREVIEW_ROW_LIMIT = 50;
/** One table cell's text: blank for empty, the string as-is, else JSON (numbers,
* booleans, nested values). Avoids `String()` on objects ("[object Object]"). */
function cellText(value: unknown): string {
if (value == null) return '';
if (typeof value === 'string') return value;
return JSON.stringify(value);
}
export function DatasetsModal() {
const datasets = useDatasetStore(useShallow((s) => s.datasets));
const view = useDatasetStore((s) => s.view);
@@ -116,11 +129,15 @@ function DatasetListItem({
onSelect: () => void;
}) {
// Meta line: source ("URL" prefix), row count when known, format label, size.
// A fetched URL snapshot reads like an inline dataset (rows + size); an unfetched
// URL reference shows "not fetched" in place of figures it doesn't have yet.
const unfetched = dataset.source === 'url' && dataset.data == null;
const parts: string[] = [];
if (dataset.source === 'url') parts.push('URL');
if (dataset.rowCount !== null) parts.push(`${dataset.rowCount} rows`);
if (unfetched) parts.push('not fetched');
else if (dataset.rowCount !== null) parts.push(`${dataset.rowCount} rows`);
parts.push(formatLabel(dataset.format));
if (dataset.source !== 'url') parts.push(humanBytes(dataset.size));
if (!unfetched) parts.push(humanBytes(dataset.size));
return (
<li className={`${styles.item} ${active ? styles.itemActive : ''}`}>
@@ -151,16 +168,46 @@ function DatasetDetail({
}) {
const startEdit = useDatasetStore((s) => s.startEdit);
const remove = useDatasetStore((s) => s.remove);
const refreshDataset = useDatasetStore((s) => s.refreshDataset);
const selectSnippet = useSnippetStore((s) => s.selectSnippet);
const [copied, setCopied] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const linked = snippetsReferencingDataset(snippets, dataset.name);
// Tabular data (CSV/TSV/JSON-array, inline or fetched) previews as a table of the
// first rows under the profiled columns; non-tabular payloads fall back to text.
// Memoized on the record so a large CSV isn't re-parsed on unrelated re-renders.
const previewRows = useMemo(
() => tabularRows(dataset.data, dataset.format, PREVIEW_ROW_LIMIT),
[dataset],
);
const handleEdit = () => {
startEdit();
resnapshot();
};
// Re-fetch a URL dataset's source and re-snapshot it. The visible result (updated
// rows + "Fetched" time) is the confirmation, so success raises no toast; only a
// failure surfaces one (spec §05 → Actions; docs/architecture/10 → Toast copy).
const handleRefresh = async () => {
if (!dataset.url) return;
setRefreshing(true);
try {
const { text } = await fetchRemoteData(dataset.url);
refreshDataset(dataset.id, { text });
} catch (err) {
notify({
kind: 'error',
title: "Couldn't refresh dataset",
message: remoteFetchErrorMessage(err, 'retry'),
});
} finally {
setRefreshing(false);
}
};
const handleCopy = async () => {
const text = JSON.stringify(datasetReference(dataset.name), null, 2);
try {
@@ -212,6 +259,16 @@ function DatasetDetail({
<span role="status" className="visually-hidden">
{copied ? 'Reference copied to clipboard' : ''}
</span>
{dataset.source === 'url' && (
<button
type="button"
className={styles.action}
onClick={() => void handleRefresh()}
disabled={refreshing}
>
{refreshing ? 'Refreshing…' : 'Refresh'}
</button>
)}
<button type="button" className={styles.action} onClick={handleEdit}>
Edit
</button>
@@ -237,6 +294,19 @@ function DatasetDetail({
{dataset.comment && <p className={styles.comment}>{dataset.comment}</p>}
{dataset.source === 'url' && (
<p className={styles.sourceMeta}>
<a className={styles.sourceLink} href={dataset.url} target="_blank" rel="noreferrer">
{dataset.url}
</a>
<span>
{dataset.fetchedAt
? `Fetched ${new Date(dataset.fetchedAt).toLocaleString()}`
: 'Not fetched yet'}
</span>
</p>
)}
<section className={styles.section}>
<h4 className={styles.sectionTitle}>Overview</h4>
<dl className={styles.stats}>
@@ -254,7 +324,7 @@ function DatasetDetail({
</div>
<div>
<dt>Size</dt>
<dd>{dataset.source === 'url' ? 'N/A' : humanBytes(dataset.size)}</dd>
<dd>{dataset.data == null ? 'N/A' : humanBytes(dataset.size)}</dd>
</div>
</dl>
@@ -277,7 +347,43 @@ function DatasetDetail({
<section className={styles.section}>
<h4 className={styles.sectionTitle}>Preview</h4>
<pre className={styles.preview}>{previewText(dataset)}</pre>
{previewRows ? (
<>
{/* TODO: a11y — this scroll container isn't keyboard-reachable; when the
preview overflows, keyboard-only users can't scroll it (WCAG 2.1.1).
A council pass should decide the pattern (tabindex=0 + aria-label region,
or a different overflow treatment) before we lean on it more. */}
<div className={styles.previewTableWrap}>
<table className={styles.previewTable}>
<thead>
<tr>
{dataset.columns.map((col, ci) => (
<th key={ci} scope="col">
{col}
</th>
))}
</tr>
</thead>
<tbody>
{previewRows.map((row, ri) => (
<tr key={ri}>
{dataset.columns.map((col, ci) => (
<td key={ci}>{cellText(row[col])}</td>
))}
</tr>
))}
</tbody>
</table>
</div>
{dataset.rowCount != null && dataset.rowCount > previewRows.length && (
<p className={styles.muted}>
Showing the first {previewRows.length} of {dataset.rowCount} rows.
</p>
)}
</>
) : (
<pre className={styles.preview}>{previewText(dataset)}</pre>
)}
</section>
<section className={styles.section}>
@@ -307,19 +413,22 @@ function DatasetDetail({
);
}
/** A truncated rendering of the data: raw for csv/tsv/url, pretty JSON otherwise. */
/** A truncated rendering of the data: raw for csv/tsv, pretty JSON otherwise. */
function previewText(dataset: Dataset): string {
const MAX = 2000;
// No snapshot yet (an unfetched URL reference) — there is nothing to preview.
if (dataset.data == null) {
return dataset.source === 'url' ? 'Not fetched yet — use Refresh to load the data.' : '';
}
let text: string;
if (dataset.source === 'url') {
text = String(dataset.data);
} else if (dataset.format === 'csv' || dataset.format === 'tsv') {
text = String(dataset.data);
if (dataset.format === 'csv' || dataset.format === 'tsv') {
// CSV/TSV payloads are raw text; fall back to JSON for any non-string value.
text = typeof dataset.data === 'string' ? dataset.data : JSON.stringify(dataset.data);
} else {
try {
text = JSON.stringify(dataset.data, null, 2);
} catch {
text = String(dataset.data);
text = typeof dataset.data === 'string' ? dataset.data : '[unserializable data]';
}
}
return text.length > MAX ? `${text.slice(0, MAX)}\n…` : text;
@@ -328,20 +437,59 @@ function previewText(dataset: Dataset): string {
function DatasetFormView({ editing }: { editing: boolean }) {
const form = useDatasetStore((s) => s.form);
const formError = useDatasetStore((s) => s.formError);
const selected = useDatasetStore(selectSelectedDataset);
const updateForm = useDatasetStore((s) => s.updateForm);
const cancelForm = useDatasetStore((s) => s.cancelForm);
const save = useDatasetStore((s) => s.save);
const commitUrlSnapshot = useDatasetStore((s) => s.commitUrlSnapshot);
// Save stays disabled until a name and valid data/URL are present (spec §05).
const canSave = useDatasetStore(selectCanSave);
// URL datasets fetch on save (snapshot model): `fetching` drives the button's
// busy state; `fetchError` holds a failed fetch's message + the inline fallback.
const [fetching, setFetching] = useState(false);
const [fetchError, setFetchError] = useState<string | null>(null);
// Live format/source hint from the current input (spec §05 → Auto-detection).
const detected =
form.source === 'url'
? { format: detectFormatFromUrl(form.input.trim()), confidence: 'url' as const }
: detectFormat(form.input);
const handleSave = () => {
if (save()) resnapshot(); // committed — re-baseline so a later close won't prompt
const handleSave = async () => {
// Inline saves are synchronous; only URL datasets touch the network.
if (form.source !== 'url') {
if (save()) resnapshot(); // committed — re-baseline so a later close won't prompt
return;
}
const url = form.input.trim();
// A metadata-only edit (same URL, snapshot already present) needs no re-fetch.
const metaOnly =
editing &&
selected?.source === 'url' &&
selected.data != null &&
url === (selected.url ?? '');
if (metaOnly) {
if (save()) resnapshot();
return;
}
// Create, changed URL, or inline→URL: fetch once, then snapshot + commit.
setFetchError(null);
setFetching(true);
try {
const { text } = await fetchRemoteData(url);
if (commitUrlSnapshot({ text })) resnapshot();
} catch (err) {
setFetchError(remoteFetchErrorMessage(err));
} finally {
setFetching(false);
}
};
// Recovery from a failed fetch: switch to an inline paste, keeping name + comment.
const handlePasteInline = () => {
setFetchError(null);
updateForm({ source: 'inline', input: '' });
};
const handleCancel = () => {
@@ -370,7 +518,10 @@ function DatasetFormView({ editing }: { editing: boolean }) {
label="Dataset source"
options={SOURCE_OPTIONS}
value={form.source}
onChange={(source) => updateForm({ source })}
onChange={(source) => {
setFetchError(null);
updateForm({ source });
}}
/>
</div>
@@ -381,7 +532,10 @@ function DatasetFormView({ editing }: { editing: boolean }) {
type="url"
className={styles.input}
value={form.input}
onChange={(e) => updateForm({ input: e.target.value })}
onChange={(e) => {
setFetchError(null);
updateForm({ input: e.target.value });
}}
placeholder="https://example.com/data.csv"
/>
) : (
@@ -418,23 +572,34 @@ function DatasetFormView({ editing }: { editing: boolean }) {
/>
</label>
{formError && (
{(formError || fetchError) && (
<p className={styles.formError} role="alert">
{formError}
{formError ?? fetchError}
</p>
)}
{/* A failed fetch is recoverable by pasting the data inline (the user's choice
when CORS or offline blocks the URL) — offered as a direct one-click path. */}
{fetchError && (
<button
type="button"
className={`${styles.action} ${styles.fallback}`}
onClick={handlePasteInline}
>
Paste data inline instead
</button>
)}
<div className={styles.formActions}>
<button type="button" className={styles.action} onClick={handleCancel}>
<button type="button" className={styles.action} onClick={handleCancel} disabled={fetching}>
Cancel
</button>
<button
type="button"
className={`${styles.action} ${styles.primary}`}
disabled={!canSave}
onClick={handleSave}
disabled={!canSave || fetching}
onClick={() => void handleSave()}
>
{editing ? 'Save changes' : 'Create dataset'}
{fetching ? 'Fetching…' : editing ? 'Save changes' : 'Create dataset'}
</button>
</div>
</div>
@@ -0,0 +1,73 @@
import { describe, expect, test } from 'vitest';
import { CURRENT_DATASET_VERSION } from '@core/dataset';
import { migrateDataset } from './dataset-migrations';
describe('migrateDataset', () => {
test('stamps the current version and fills missing fields with defaults', () => {
const d = migrateDataset({ id: 7, data: 'a,b\n1,2', format: 'csv', source: 'inline' });
expect(d.id).toBe(7);
expect(d.version).toBe(CURRENT_DATASET_VERSION);
expect(d.name).toBe('Untitled');
expect(d.comment).toBe('');
expect(d.columns).toEqual([]);
expect(d.columnStats).toEqual([]);
});
test('an inline record carries no url/fetchedAt keys', () => {
const d = migrateDataset({ id: 1, data: [{ a: 1 }], format: 'json', source: 'inline' });
expect(d.source).toBe('inline');
expect('url' in d).toBe(false);
expect('fetchedAt' in d).toBe(false);
});
test('v1→v2: a legacy URL record moves its address out of data into url', () => {
// Pre-v2 shape: the URL lived in `data`, there was no `url`/`fetchedAt`, and the
// profile was N/A. It becomes an unfetched reference: url set, data cleared.
const d = migrateDataset({
id: 2,
data: 'https://example.com/data.csv',
format: 'csv',
source: 'url',
rowCount: null,
columns: [],
});
expect(d.version).toBe(CURRENT_DATASET_VERSION);
expect(d.source).toBe('url');
expect(d.url).toBe('https://example.com/data.csv');
expect(d.data).toBeNull();
expect(d.fetchedAt).toBeNull();
expect(d.rowCount).toBeNull();
});
test('a v2 URL snapshot is preserved (data, url, and fetchedAt all kept)', () => {
const d = migrateDataset({
id: 3,
version: 2,
data: 'a,b\n1,2',
url: 'https://example.com/data.csv',
fetchedAt: '2026-06-10T00:00:00.000Z',
format: 'csv',
source: 'url',
rowCount: 1,
columns: ['a', 'b'],
});
expect(d.data).toBe('a,b\n1,2');
expect(d.url).toBe('https://example.com/data.csv');
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
expect(d.rowCount).toBe(1);
});
test('preserves unknown fields so a newer build round-trips without loss', () => {
const d = migrateDataset({
id: 1,
data: [],
format: 'json',
source: 'inline',
futureField: 42,
});
expect((d as unknown as Record<string, unknown>).futureField).toBe(42);
});
});
+21 -2
View File
@@ -30,14 +30,33 @@ function asCount(v: unknown): number | null {
/** Upgrade a raw stored record to the current Dataset shape. */
export function migrateDataset(raw: unknown): Dataset {
const r = { ...(raw as Record<string, unknown>) };
const source = asSource(r.source);
// v1→v2: a URL dataset used to store its address in `data`; it now stores the
// fetched snapshot in `data` and keeps the address in `url` (+ a `fetchedAt`).
// A legacy record (url source, no `url` field) becomes an *unfetched* reference:
// move the address to `url` and clear `data`. The N/A profile carries over as-is;
// rendering falls back to the live URL until a Refresh fetches + profiles it.
const legacyUrl = source === 'url' && typeof r.url !== 'string';
const url = legacyUrl
? typeof r.data === 'string'
? r.data
: undefined
: typeof r.url === 'string'
? r.url
: undefined;
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
return {
...r,
id: typeof r.id === 'number' ? r.id : Number(r.id),
version: CURRENT_DATASET_VERSION,
name: typeof r.name === 'string' ? r.name : 'Untitled',
data: r.data,
data: legacyUrl ? null : r.data,
format: asFormat(r.format),
source: asSource(r.source),
source,
// Only URL datasets carry url/fetchedAt; inline records stay clean.
...(source === 'url' ? { url, fetchedAt } : {}),
comment: typeof r.comment === 'string' ? r.comment : '',
rowCount: asCount(r.rowCount),
columnCount: asCount(r.columnCount),
@@ -0,0 +1,94 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { fetchRemoteData, RemoteFetchError } from './remote-data';
/** A minimal Response stand-in for the bits the adapter reads. */
function fakeResponse(opts: {
ok: boolean;
status?: number;
statusText?: string;
body?: string;
contentType?: string | null;
}): Response {
return {
ok: opts.ok,
status: opts.status ?? (opts.ok ? 200 : 500),
statusText: opts.statusText ?? '',
text: () => Promise.resolve(opts.body ?? ''),
headers: {
get: (k: string) => (k.toLowerCase() === 'content-type' ? (opts.contentType ?? null) : null),
},
} as unknown as Response;
}
afterEach(() => vi.unstubAllGlobals());
describe('fetchRemoteData', () => {
test('returns the body and content-type on success', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() =>
Promise.resolve(fakeResponse({ ok: true, body: 'a,b\n1,2', contentType: 'text/csv' })),
),
);
const result = await fetchRemoteData('https://example.com/data.csv');
expect(result.text).toBe('a,b\n1,2');
expect(result.contentType).toBe('text/csv');
});
test('classifies a non-2xx response as http with its status', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() =>
Promise.resolve(fakeResponse({ ok: false, status: 404, statusText: 'Not Found' })),
),
);
await expect(fetchRemoteData('https://example.com/missing.csv')).rejects.toMatchObject({
reason: 'http',
status: 404,
});
});
test('classifies a rejected fetch (CORS / offline) as network', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
);
await expect(fetchRemoteData('https://blocked.example/data.csv')).rejects.toMatchObject({
reason: 'network',
});
});
test('classifies an empty body as empty', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => Promise.resolve(fakeResponse({ ok: true, body: ' \n' }))),
);
await expect(fetchRemoteData('https://example.com/blank.csv')).rejects.toMatchObject({
reason: 'empty',
});
});
test('classifies an aborted fetch as timeout', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => {
const e = new Error('aborted');
e.name = 'AbortError';
return Promise.reject(e);
}),
);
await expect(
fetchRemoteData('https://slow.example/data.csv', { timeoutMs: 1 }),
).rejects.toMatchObject({
reason: 'timeout',
});
});
test('the thrown error is a RemoteFetchError', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
);
await expect(fetchRemoteData('https://x.example')).rejects.toBeInstanceOf(RemoteFetchError);
});
});
+99
View File
@@ -0,0 +1,99 @@
/**
* Remote-data fetch adapter (snapshot model — spec §05 → URL datasets).
*
* The ONLY place Astrolabe touches the network. Adding a URL dataset fetches the
* resource once here; core then profiles the returned text exactly like inline
* data (see dataset.ts). Keeping the fetch in infrastructure preserves the rule
* that `src/core` stays pure and that the rest of the app never calls `fetch`
* directly (AGENTS → Architecture).
*
* Failures are classified into a small set of machine-readable `reason`s and
* thrown as `RemoteFetchError`; the user-facing wording (and the "paste inline
* instead" fallback) lives in the UI layer, where it is council-reviewed — the
* same split as `storage-errors`. A browser cross-origin block and an
* offline/DNS failure are indistinguishable here — both reject with a `TypeError`
* and no status — so they share the `network` reason and the UI hedges the cause.
*/
/** Why a remote fetch failed, in terms the UI maps to copy + a recovery path. */
export type RemoteFetchReason = 'network' | 'http' | 'empty' | 'timeout';
/** A classified remote-fetch failure. `status` is set only for `http`. */
export class RemoteFetchError extends Error {
readonly reason: RemoteFetchReason;
readonly status?: number;
constructor(reason: RemoteFetchReason, message: string, status?: number) {
super(message);
this.name = 'RemoteFetchError';
this.reason = reason;
this.status = status;
}
}
/** A successful fetch: the raw body plus a weak format hint from the server. */
export interface RemoteData {
/** The response body, ready for format detection + profiling. */
text: string;
/** The server's `Content-Type`, when provided — a tertiary format hint. */
contentType: string | null;
}
/** Default ceiling on a single fetch before it is aborted as a timeout. */
const DEFAULT_TIMEOUT_MS = 30_000;
export interface FetchRemoteDataOptions {
/** Abort the fetch after this many ms (default 30 s). */
timeoutMs?: number;
}
function describeNetworkError(err: unknown): string {
const reason = err instanceof Error ? err.message : String(err);
return `Couldn't reach the URL (${reason}).`;
}
/**
* Fetch a remote dataset resource once and return its raw body. Throws
* `RemoteFetchError` on any failure — the caller distinguishes recovery by
* `reason`. The body is returned verbatim; format detection and profiling happen
* downstream so this adapter stays free of core logic.
*/
export async function fetchRemoteData(
url: string,
options: FetchRemoteDataOptions = {},
): Promise<RemoteData> {
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
let response: Response;
try {
response = await fetch(url, { redirect: 'follow', signal: controller.signal });
} catch (err) {
// A timeout aborts via the controller (AbortError); everything else here is a
// CORS block, offline, DNS failure, or malformed URL — all opaque TypeErrors.
if ((err as { name?: string }).name === 'AbortError') {
throw new RemoteFetchError('timeout', `The URL took longer than ${timeoutMs} ms to respond.`);
}
throw new RemoteFetchError('network', describeNetworkError(err));
} finally {
clearTimeout(timer);
}
if (!response.ok) {
throw new RemoteFetchError(
'http',
`The server responded ${response.status} ${response.statusText}.`.trim(),
response.status,
);
}
const text = await response.text();
if (text.trim() === '') {
throw new RemoteFetchError('empty', 'The URL returned no data.');
}
// TODO: guard very large responses (cap or warn) before snapshotting into
// IndexedDB — a multi-hundred-MB file would load fully into memory + storage.
return { text, contentType: response.headers.get('content-type') };
}
@@ -0,0 +1,38 @@
import { describe, expect, test } from 'vitest';
import { RemoteFetchError } from '../infrastructure/remote-data';
import { remoteFetchErrorMessage } from './remote-data-errors';
describe('remoteFetchErrorMessage', () => {
test('a network failure hedges offline vs cross-origin and offers the inline fallback', () => {
const msg = remoteFetchErrorMessage(new RemoteFetchError('network', 'x'));
expect(msg).toMatch(/offline/i);
expect(msg).toMatch(/cross-origin/i);
expect(msg).toMatch(/paste the data inline/i);
});
test('an HTTP 404 explains "not found" without a raw error code', () => {
const msg = remoteFetchErrorMessage(new RemoteFetchError('http', 'x', 404));
expect(msg).toMatch(/couldn't find/i);
expect(msg).not.toMatch(/404/);
});
test('an HTTP 403 explains refused access', () => {
expect(remoteFetchErrorMessage(new RemoteFetchError('http', 'x', 403))).toMatch(
/refused access/i,
);
});
test('an empty body says it returned no data', () => {
expect(remoteFetchErrorMessage(new RemoteFetchError('empty', 'x'))).toMatch(/no data/i);
});
test('the retry recovery points at refreshing, not pasting inline', () => {
const msg = remoteFetchErrorMessage(new RemoteFetchError('timeout', 'x'), 'retry');
expect(msg).toMatch(/refreshing again/i);
expect(msg).not.toMatch(/inline/i);
});
test('an unknown error still produces a sensible fallback message', () => {
expect(remoteFetchErrorMessage(new Error('boom'))).toMatch(/couldn't fetch this url/i);
});
});
+51
View File
@@ -0,0 +1,51 @@
/**
* Translate a remote-data fetch failure (snapshot model — spec §05 → URL datasets)
* into the message shown when adding or refreshing a URL dataset.
*
* Pure and framework-free so the wording is unit-tested directly — the same split
* as `storage-errors`. Follows the recorded error-copy resolution (docs/architecture
* /10 → error copy): plain language, name what stopped, and — because every one of
* these is user-fixable — point at the next step. No error codes in the user-facing
* line (GOV.UK / NN/g #9). The recovery clause adapts to where the error shows: the
* create/edit form can fall back to pasting inline; Refresh can only retry.
*/
import { RemoteFetchError } from '../infrastructure/remote-data';
/** Where the error surfaces, which decides the recovery the copy points at. */
export type FetchRecovery = 'inline' | 'retry';
function nextStep(recovery: FetchRecovery): string {
return recovery === 'inline'
? 'Check the address, or paste the data inline instead.'
: 'Check the address and try refreshing again.';
}
function httpCause(status?: number): string {
if (status === 404) return "The server couldn't find anything at this URL.";
if (status === 401 || status === 403) return 'The server refused access to this URL.';
if (status && status >= 500) return 'The server hit an error serving this URL.';
return "The server couldn't return this URL.";
}
/**
* The user-facing message for a fetch failure. `recovery` defaults to `inline`
* (the add/edit form); pass `retry` for the Refresh action, which has no inline
* fallback.
*/
export function remoteFetchErrorMessage(err: unknown, recovery: FetchRecovery = 'inline'): string {
const next = nextStep(recovery);
if (err instanceof RemoteFetchError) {
switch (err.reason) {
case 'network':
return `Couldn't fetch this URL. Your device may be offline, or the host may block cross-origin requests. ${next}`;
case 'http':
return `${httpCause(err.status)} ${next}`;
case 'empty':
return `This URL returned no data. Make sure it points directly at a data file. ${next}`;
case 'timeout':
return `This URL took too long to respond. ${next}`;
}
}
return `Couldn't fetch this URL. ${next}`;
}
+70 -3
View File
@@ -42,15 +42,82 @@ describe('save — create', () => {
expect(ds?.rowCount).toBe(2);
});
test('a valid URL infers format from the extension', () => {
test('save() does not commit a URL create — it must go through commitUrlSnapshot', () => {
store().startCreate();
store().updateForm({ name: 'Remote', source: 'url', input: 'https://example.com/data.csv' });
// The component fetches first; save() never touches the network, so a URL create
// through save() is a no-op rather than an unfetched record.
expect(store().save(T)).toBe(false);
expect(store().datasets).toHaveLength(0);
});
});
describe('commitUrlSnapshot — create from a fetched body', () => {
test('snapshots the body, infers format from content, and profiles it', () => {
store().startCreate();
store().updateForm({ name: 'Remote', source: 'url', input: 'https://example.com/data.csv' });
expect(store().save(T)).toBe(true);
expect(store().commitUrlSnapshot({ text: 'a,b\n1,2\n3,4' }, T)).toBe(true);
const ds = selectSelectedDataset(store());
expect(ds?.source).toBe('url');
expect(ds?.url).toBe('https://example.com/data.csv');
expect(ds?.format).toBe('csv');
expect(ds?.rowCount).toBeNull(); // URL datasets aren't profiled
expect(ds?.data).toBe('a,b\n1,2\n3,4');
expect(ds?.rowCount).toBe(2);
expect(ds?.columns).toEqual(['a', 'b']);
expect(ds?.fetchedAt).toBe(T.toISOString());
expect(store().view).toBe('detail');
});
test('a duplicate name is rejected even with a successful fetch', () => {
store().add(
createDataset({ name: 'Dupe', data: [{ a: 1 }], format: 'json', source: 'inline', now: T }),
);
store().startCreate();
store().updateForm({ name: 'dupe', source: 'url', input: 'https://x/y.csv' });
expect(store().commitUrlSnapshot({ text: 'a\n1' }, T)).toBe(false);
expect(store().formError).toMatch(/already exists/i);
});
});
describe('URL dataset edits & refresh', () => {
const seedUrl = () => {
store().startCreate();
store().updateForm({ name: 'Remote', source: 'url', input: 'https://x/y.csv' });
store().commitUrlSnapshot({ text: 'a,b\n1,2' }, T);
return store().selectedId!;
};
test('a metadata-only edit (same URL) updates name without re-fetching', () => {
const id = seedUrl();
store().startEdit();
store().updateForm({ name: 'Renamed' }); // URL field stays https://x/y.csv
const later = new Date('2026-07-01T00:00:00Z');
expect(store().save(later)).toBe(true);
const ds = store().datasets.find((d) => d.id === id)!;
expect(ds.name).toBe('Renamed');
expect(ds.data).toBe('a,b\n1,2'); // snapshot untouched
expect(ds.fetchedAt).toBe(T.toISOString()); // not re-fetched
expect(ds.modified).toBe(later.toISOString());
});
test('refreshDataset re-snapshots, re-profiles, and advances fetchedAt', () => {
const id = seedUrl();
const later = new Date('2026-07-01T00:00:00Z');
expect(store().refreshDataset(id, { text: 'a,b\n1,2\n3,4\n5,6' }, later)).toBe(true);
const ds = store().datasets.find((d) => d.id === id)!;
expect(ds.rowCount).toBe(3);
expect(ds.fetchedAt).toBe(later.toISOString());
expect(ds.modified).toBe(later.toISOString());
expect(ds.name).toBe('Remote'); // preserved
});
test('refreshDataset is a no-op for an inline dataset', () => {
store().add(
createDataset({ name: 'Inline', data: [{ a: 1 }], format: 'json', source: 'inline', now: T }),
);
const id = store().selectedId!;
expect(store().refreshDataset(id, { text: 'a\n1' }, T)).toBe(false);
});
});
+141 -19
View File
@@ -19,8 +19,14 @@
*/
import { create } from 'zustand';
import { detectFormat, detectFormatFromUrl, type DataFormat } from '@core/format-detection';
import { createDataset, computeDatasetProfile, type DataSource, type Dataset } from '@core/dataset';
import { detectFormat, type DataFormat } from '@core/format-detection';
import {
createDataset,
computeDatasetProfile,
snapshotFromText,
type DataSource,
type Dataset,
} from '@core/dataset';
import { isNameTaken } from '@core/naming';
import { useSnippetStore } from './SnippetStore';
@@ -67,6 +73,21 @@ export interface DatasetState {
*/
save: (now?: Date) => boolean;
/**
* Commit a URL dataset from an already-fetched body. The component performs the
* network fetch via the remote-data adapter and passes the result here, keeping
* this store browser-free. Validates the form, snapshots + profiles the body, and
* creates (view `new`) or updates (view `edit`) the dataset — including renaming
* referencing snippets on an edit. Returns whether it committed.
*/
commitUrlSnapshot: (fetched: { text: string }, now?: Date) => boolean;
/**
* Re-snapshot an existing URL dataset from a freshly-fetched body ("Refresh"):
* re-detect the format, re-profile, and advance `fetchedAt`/`modified`. Name,
* comment, and URL are preserved. Returns false for a non-URL dataset.
*/
refreshDataset: (id: number, fetched: { text: string }, now?: Date) => boolean;
/**
* Low-level: add a fully-formed dataset and select it. The id is (re)assigned
* via `nextDatasetId`, so a `createDataset` default id (`Date.now()`) can never
@@ -144,7 +165,12 @@ function validateForm(
return null;
}
/** Validate a form into a saveable shape, or return an error message. */
/**
* Validate an **inline** form into a saveable shape, or return an error message.
* URL datasets never reach here — they are fetched first and committed through
* `commitUrlSnapshot` (which snapshots the fetched body), so this only shapes the
* pasted-inline payload.
*/
function resolveForm(
form: DatasetForm,
datasets: Dataset[],
@@ -153,20 +179,12 @@ function resolveForm(
const error = validateForm(form, datasets, excludeId);
if (error) return { error };
const name = form.name.trim();
const input = form.input.trim();
if (form.source === 'url') {
// Format is inferred from the extension; default to JSON when unknown (§05).
return { name, data: input, format: detectFormatFromUrl(input) ?? 'json', source: 'url' };
}
// validateForm guaranteed a detectable format above.
const format = detectFormat(form.input).format as DataFormat;
// JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text (data model §09B).
const data =
format === 'json' || format === 'topojson' ? (JSON.parse(form.input) as unknown) : form.input;
return { name, data, format, source: 'inline' };
return { name: form.name.trim(), data, format, source: 'inline' };
}
export const useDatasetStore = create<DatasetState>((set, get) => ({
@@ -201,12 +219,14 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
name: ds.name,
source: ds.source,
comment: ds.comment,
// Re-render the stored payload as editable text: raw for csv/tsv/url,
// pretty-printed JSON for json/topojson.
// Re-render the editable text: the URL for url datasets, raw text for
// csv/tsv, pretty-printed JSON for json/topojson.
input:
ds.source === 'url' || ds.format === 'csv' || ds.format === 'tsv'
? String(ds.data)
: JSON.stringify(ds.data, null, 2),
ds.source === 'url'
? (ds.url ?? '')
: ds.format === 'csv' || ds.format === 'tsv'
? String(ds.data)
: JSON.stringify(ds.data, null, 2),
},
});
},
@@ -225,6 +245,30 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
const editing = view === 'edit';
const excludeId = editing ? (selectedId ?? undefined) : undefined;
// URL datasets that need the network — a create, a URL change, or an inline→URL
// conversion — are routed through `commitUrlSnapshot` after the component
// fetches; save() never fetches. The one URL case it commits is a metadata-only
// edit of an existing snapshot (same URL): just update name/comment, no re-fetch.
if (form.source === 'url') {
const error = validateForm(form, datasets, excludeId);
if (error) {
set({ formError: error });
return false;
}
if (!editing || selectedId === null) return false;
const existing = datasets.find((d) => d.id === selectedId);
if (!existing || existing.source !== 'url' || form.input.trim() !== (existing.url ?? '')) {
return false;
}
const name = form.name.trim();
get().update(selectedId, { name, comment: form.comment }, now);
if (name !== existing.name) {
useSnippetStore.getState().renameDatasetRefs(existing.name, name, now);
}
set({ view: 'detail', form: EMPTY_FORM, formError: null });
return true;
}
const resolved = resolveForm(form, datasets, excludeId);
if ('error' in resolved) {
set({ formError: resolved.error });
@@ -234,11 +278,12 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
if (editing && selectedId !== null) {
const existing = datasets.find((d) => d.id === selectedId);
if (!existing) return false;
const profile = computeDatasetProfile(resolved.data, resolved.format, resolved.source);
const profile = computeDatasetProfile(resolved.data, resolved.format);
// Re-profile and update the record (including any new name), then propagate
// the rename across referencing snippets so each spec and its datasetRefs
// stay consistent (docs/architecture/07 §6). SnippetStore never imports this
// store, so the direct call is cycle-free.
// store, so the direct call is cycle-free. Clear any url/fetchedAt left over
// from a URL→inline conversion so the record carries no stale remote origin.
get().update(
selectedId,
{
@@ -246,6 +291,13 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
data: resolved.data,
format: resolved.format,
source: resolved.source,
// TODO: `update` shallow-merges, so these set the keys to `undefined`
// rather than removing them — a URL→inline record keeps `url`/`fetchedAt`
// present-but-undefined. Invisible today (rendering/profiling key off
// `source`/`data == null`, and JSON export drops undefined), but it breaks
// the "inline records carry no url/fetchedAt keys" invariant on this path.
url: undefined,
fetchedAt: undefined,
comment: form.comment,
...profile,
},
@@ -275,6 +327,76 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
return true;
},
commitUrlSnapshot: (fetched, now) => {
const { view, form, datasets, selectedId } = get();
const editing = view === 'edit';
const excludeId = editing ? (selectedId ?? undefined) : undefined;
const error = validateForm(form, datasets, excludeId);
if (error) {
set({ formError: error });
return false;
}
const name = form.name.trim();
const url = form.input.trim();
const iso = (now ?? new Date()).toISOString();
// The fetched body snapshots + profiles exactly like inline data (snapshot
// model): detect format from content, shape, and profile through core.
const { data, format } = snapshotFromText(fetched.text, url);
if (editing && selectedId !== null) {
const existing = datasets.find((d) => d.id === selectedId);
if (!existing) return false;
const profile = computeDatasetProfile(data, format);
get().update(
selectedId,
{
name,
data,
format,
source: 'url',
url,
fetchedAt: iso,
comment: form.comment,
...profile,
},
now,
);
if (name !== existing.name) {
useSnippetStore.getState().renameDatasetRefs(existing.name, name, now);
}
set({ view: 'detail', form: EMPTY_FORM, formError: null });
return true;
}
const dataset = createDataset({
name,
data,
format,
source: 'url',
url,
fetchedAt: iso,
comment: form.comment,
now,
});
get().add(dataset);
set({ view: 'detail', form: EMPTY_FORM, formError: null });
return true;
},
refreshDataset: (id, fetched, now) => {
const dataset = get().datasets.find((d) => d.id === id);
if (!dataset || dataset.source !== 'url' || !dataset.url) return false;
const iso = (now ?? new Date()).toISOString();
const { data, format } = snapshotFromText(fetched.text, dataset.url);
const profile = computeDatasetProfile(data, format);
// Name, comment, and url are preserved; only the snapshot + its profile and the
// fetch time change. `update` advances `modified`.
get().update(id, { data, format, fetchedAt: iso, ...profile }, now);
return true;
},
add: (dataset) =>
set((s) => {
const withId = { ...dataset, id: nextDatasetId(s.datasets) };
+122 -21
View File
@@ -5,6 +5,8 @@ import {
createDataset,
datasetReference,
parseDelimited,
snapshotFromText,
tabularRows,
} from './dataset';
describe('datasetReference', () => {
@@ -65,51 +67,125 @@ describe('parseDelimited', () => {
});
describe('computeDatasetProfile', () => {
test('inline JSON array-of-objects is profiled', () => {
test('JSON array-of-objects is profiled', () => {
const data = [
{ a: 1, b: 'x' },
{ a: 2, b: 'y' },
];
const profile = computeDatasetProfile(data, 'json', 'inline');
const profile = computeDatasetProfile(data, 'json');
expect(profile.rowCount).toBe(2);
expect(profile.columns).toEqual(['a', 'b']);
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(data)).length);
});
test('inline JSON that is not an array is N/A but sized', () => {
const profile = computeDatasetProfile({ a: 1 }, 'json', 'inline');
test('JSON that is not an array is N/A but sized', () => {
const profile = computeDatasetProfile({ a: 1 }, 'json');
expect(profile.rowCount).toBeNull();
expect(profile.size).toBeGreaterThan(0);
});
test('inline CSV is parsed and profiled; size is the raw text byte length', () => {
test('CSV is parsed and profiled; size is the raw text byte length', () => {
const text = 'a,b\n1,2';
const profile = computeDatasetProfile(text, 'csv', 'inline');
const profile = computeDatasetProfile(text, 'csv');
expect(profile.rowCount).toBe(1);
expect(profile.columns).toEqual(['a', 'b']);
expect(profile.size).toBe(new TextEncoder().encode(text).length);
});
test('inline TSV is parsed and profiled', () => {
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv', 'inline');
test('TSV is parsed and profiled', () => {
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv');
expect(profile.rowCount).toBe(1);
expect(profile.columns).toEqual(['a', 'b']);
});
test('inline TopoJSON is N/A (non-tabular) but sized', () => {
test('TopoJSON is N/A (non-tabular) but sized', () => {
const topo = { type: 'Topology', objects: {} };
const profile = computeDatasetProfile(topo, 'topojson', 'inline');
const profile = computeDatasetProfile(topo, 'topojson');
expect(profile.rowCount).toBeNull();
expect(profile.columnCount).toBeNull();
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(topo)).length);
});
test('URL is N/A; size is the byte length of the URL string', () => {
const url = 'https://example.com/data.csv';
const profile = computeDatasetProfile(url, 'csv', 'url');
test('a fetched URL snapshot is profiled exactly like inline data', () => {
// The snapshot model: once fetched, a URL dataset carries its payload in `data`
// and profiles through the same path as inline (no source distinction here).
const profile = computeDatasetProfile('a,b\n1,2\n3,4', 'csv');
expect(profile.rowCount).toBe(2);
expect(profile.columns).toEqual(['a', 'b']);
});
test('an unfetched URL reference (null data) is N/A with size 0', () => {
const profile = computeDatasetProfile(null, 'csv');
expect(profile.rowCount).toBeNull();
expect(profile.columnCount).toBeNull();
expect(profile.size).toBe(new TextEncoder().encode(url).length);
expect(profile.columns).toEqual([]);
expect(profile.size).toBe(0);
});
});
describe('tabularRows', () => {
test('CSV parses to rows (same parsing as profiling)', () => {
expect(tabularRows('a,b\n1,2\n3,4', 'csv')).toEqual([
{ a: '1', b: '2' },
{ a: '3', b: '4' },
]);
});
test('TSV parses to rows', () => {
expect(tabularRows('x\ty\n1\t2', 'tsv')).toEqual([{ x: '1', y: '2' }]);
});
test('a JSON array of objects is returned as-is', () => {
const data = [{ a: 1 }, { a: 2 }];
expect(tabularRows(data, 'json')).toEqual(data);
});
test('limit returns only the head', () => {
const rows = tabularRows('n\n1\n2\n3\n4', 'csv', 2);
expect(rows).toEqual([{ n: '1' }, { n: '2' }]);
});
test('non-tabular payloads return null (single object, topojson, unfetched)', () => {
expect(tabularRows({ a: 1 }, 'json')).toBeNull();
expect(tabularRows({ type: 'Topology', objects: {} }, 'topojson')).toBeNull();
expect(tabularRows(null, 'csv')).toBeNull();
expect(tabularRows('a,b', 'csv')).toBeNull(); // header only — no rows
});
});
describe('snapshotFromText', () => {
test('JSON content is parsed and typed json (content beats extension)', () => {
const { data, format } = snapshotFromText('[{"a":1}]', 'https://x/data.txt');
expect(format).toBe('json');
expect(data).toEqual([{ a: 1 }]);
});
test('CSV content keeps its raw text and is typed csv', () => {
const { data, format } = snapshotFromText('a,b\n1,2', 'https://x/data.csv');
expect(format).toBe('csv');
expect(data).toBe('a,b\n1,2');
});
test('TSV content is typed tsv', () => {
expect(snapshotFromText('a\tb\n1\t2', 'https://x/d').format).toBe('tsv');
});
test('a Topology body is typed topojson and parsed', () => {
const { data, format } = snapshotFromText('{"type":"Topology","objects":{}}', 'https://x/d');
expect(format).toBe('topojson');
expect(data).toEqual({ type: 'Topology', objects: {} });
});
test('unrecognized content falls back to the URL extension', () => {
// A single token: not JSON, not delimited. The .json extension decides the
// format; the unparseable body is kept verbatim rather than throwing.
const { data, format } = snapshotFromText('not-data', 'https://x/data.json');
expect(format).toBe('json');
expect(data).toBe('not-data');
});
test('unrecognized content with no useful extension defaults to json', () => {
expect(snapshotFromText('not-data', 'https://x/feed').format).toBe('json');
});
});
@@ -157,20 +233,45 @@ describe('createDataset', () => {
expect(d.columns).toEqual(['a', 'b']);
});
test('URL dataset gets an N/A profile with a size', () => {
test('URL dataset carries url + fetchedAt and profiles the fetched snapshot', () => {
const d = createDataset({
id: 2,
name: 'Remote',
data: 'https://example.com/x.json',
format: 'json',
data: 'a,b\n1,2',
url: 'https://example.com/x.csv',
fetchedAt: '2026-06-10T00:00:00.000Z',
format: 'csv',
source: 'url',
comment: 'remote source',
});
expect(d.rowCount).toBeNull();
expect(d.columnCount).toBeNull();
expect(d.columns).toEqual([]);
expect(d.source).toBe('url');
expect(d.url).toBe('https://example.com/x.csv');
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
expect(d.rowCount).toBe(1);
expect(d.columns).toEqual(['a', 'b']);
expect(d.comment).toBe('remote source');
expect(d.size).toBeGreaterThan(0);
});
test('an unfetched URL dataset (null data) gets an N/A profile and null fetchedAt', () => {
const d = createDataset({
id: 3,
name: 'Unfetched',
data: null,
url: 'https://example.com/x.csv',
format: 'csv',
source: 'url',
});
expect(d.url).toBe('https://example.com/x.csv');
expect(d.fetchedAt).toBeNull();
expect(d.rowCount).toBeNull();
expect(d.columns).toEqual([]);
expect(d.size).toBe(0);
});
test('inline dataset carries no url/fetchedAt keys', () => {
const d = createDataset({ id: 4, name: 'I', data: [], format: 'json', source: 'inline' });
expect('url' in d).toBe(false);
expect('fetchedAt' in d).toBe(false);
});
test('defaults the id when not injected', () => {
+102 -29
View File
@@ -7,22 +7,29 @@
* a simple delimited-text parser, the profiling orchestration, and a factory that
* stamps timestamps/version and fills the derived summary fields.
*
* A dataset has one of two **sources** — `inline` (data stored in the record) or
* `url` (only the link is stored, fetched on demand at render time) — and one of
* four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
* The `data` field's shape follows source/format: a URL string for `url`; raw
* text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
* A dataset has one of two **sources** — `inline` (data pasted into the record) or
* `url` (fetched once from a remote address and **snapshotted** into the record) —
* and one of four **formats** (reused from format-detection: `json`/`csv`/`tsv`/
* `topojson`). Either way `data` holds the actual payload, shaped by format: raw
* text for CSV/TSV, a parsed value for JSON/TopoJSON. A `url` dataset additionally
* keeps its source `url` (so it can be re-fetched) and a `fetchedAt` timestamp;
* until its first successful fetch `data` is `null` (an unfetched reference).
*
* Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
* non-tabular data get an N/A profile but are still sized (see profile.ts).
* Any tabular payload (JSON array-of-objects, CSV, TSV) is profiled — including a
* fetched URL snapshot; non-tabular or not-yet-fetched data gets an N/A profile but
* is still sized (see profile.ts).
*/
import type { DataFormat } from './format-detection';
import { detectFormat, detectFormatFromUrl, type DataFormat } from './format-detection';
import { profileData, type ColumnStats, type DatasetProfile } from './profile';
import type { ColumnType } from './type-inference';
/** Current schema version for a Dataset record (read-time migration target). */
export const CURRENT_DATASET_VERSION = 1;
/**
* Current schema version for a Dataset record (read-time migration target).
* v2 moved a URL dataset's address out of `data` into its own `url` field and made
* `data` hold the fetched snapshot (see `migrateDataset`).
*/
export const CURRENT_DATASET_VERSION = 2;
/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
export type DataSource = 'inline' | 'url';
@@ -35,14 +42,25 @@ export interface Dataset {
/** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
name: string;
/**
* The payload. For `source = url`: the URL string. For `source = inline`: the
* raw CSV/TSV text, or the parsed JSON/TopoJSON value.
* The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON
* value. For `source = url` this is the fetched snapshot, or `null` before the
* first successful fetch.
*/
data: unknown;
/** One of `json`, `csv`, `tsv`, `topojson`. */
format: DataFormat;
/** One of `inline` or `url`. */
source: DataSource;
/**
* For `source = url`: the remote address the snapshot was fetched from, retained
* so the dataset can be re-fetched ("Refresh"). Absent for inline datasets.
*/
url?: string;
/**
* For `source = url`: ISO timestamp of the last successful fetch, or `null` when
* it has never been fetched. Absent for inline datasets.
*/
fetchedAt?: string | null;
/** Free-form user note about the dataset. */
comment: string;
/** Data rows, or `null` when N/A (URL / non-tabular). */
@@ -187,26 +205,52 @@ function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
}
/**
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
* tabular vs N/A by source/format, and delegate to `profileData`.
*
* - `url` (any format) → N/A profile; size = byte length of the URL string.
* - inline `json` → rows when a non-empty array of objects, else N/A.
* - inline `topojson` → N/A (non-tabular).
* - inline `csv` / `tsv` → `parseDelimited` rows.
*
* `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
* `JSON.stringify(data)` for json/topojson.
* The tabular rows of a dataset payload for a table preview, or `null` when the
* payload isn't tabular (a single JSON object, TopoJSON, or an unfetched URL). Uses
* the **same** parsing as profiling — `parseDelimited` for CSV/TSV, `asObjectRows`
* for JSON — so the previewed rows agree exactly with the profiled `columns`. A
* positive `limit` returns only the head (a preview needs a sample, not the whole
* payload). Returns non-null on precisely the inputs `computeDatasetProfile` counts
* as rows, so a caller can gate "table vs. raw text" on this alone.
*/
export function computeDatasetProfile(
export function tabularRows(
data: unknown,
format: DataFormat,
source: DataSource,
): DatasetProfile {
if (source === 'url') {
const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
return profileData(null, byteLength(url));
limit?: number,
): Array<Record<string, unknown>> | null {
if (data == null) return null;
let rows: Array<Record<string, unknown>> | null;
switch (format) {
case 'csv':
case 'tsv':
rows = parseDelimited(typeof data === 'string' ? data : '', format);
break;
case 'json':
rows = asObjectRows(data);
break;
default:
rows = null;
}
if (!rows || rows.length === 0) return null;
return limit != null && limit >= 0 && rows.length > limit ? rows.slice(0, limit) : rows;
}
/**
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
* tabular vs N/A by format, and delegate to `profileData`. Identical for inline
* data and for a fetched URL snapshot — both carry the payload in `data`.
*
* - `null` data (unfetched URL) → N/A profile, size 0.
* - `json` → rows when a non-empty array of objects, else N/A.
* - `topojson` → N/A (non-tabular).
* - `csv` / `tsv` → `parseDelimited` rows.
*
* `size` is the UTF-8 byte length of the raw string for csv/tsv, or of
* `JSON.stringify(data)` for json/topojson.
*/
export function computeDatasetProfile(data: unknown, format: DataFormat): DatasetProfile {
// An unfetched URL reference (or a genuinely absent payload): nothing to profile.
if (data == null) return profileData(null, 0);
switch (format) {
case 'csv':
@@ -226,6 +270,29 @@ export function computeDatasetProfile(
}
}
/**
* Shape a freshly-fetched URL body into the `{ data, format }` a snapshot stores
* (spec §05 → URL datasets, snapshot model). Format is sniffed from the **content**
* first — authoritative, since `detectFormat` only reports `json` when the body
* actually parses — falling back to the URL's file extension, then JSON.
* JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text — the same
* per-format shaping inline data uses, so a fetched dataset profiles and renders
* identically to an inline one (see `computeDatasetProfile`, rendering.ts).
*/
export function snapshotFromText(text: string, url: string): { data: unknown; format: DataFormat } {
const format = detectFormat(text).format ?? detectFormatFromUrl(url) ?? 'json';
if (format === 'json' || format === 'topojson') {
try {
return { data: JSON.parse(text) as unknown, format };
} catch {
// The extension promised JSON but the body isn't — keep the raw text so the
// render surfaces a readable error instead of us throwing mid-commit.
return { data: text, format };
}
}
return { data: text, format };
}
export interface CreateDatasetOptions {
/** The dataset name (uniqueness is enforced upstream — see naming.ts). */
name: string;
@@ -235,6 +302,10 @@ export interface CreateDatasetOptions {
format: DataFormat;
/** One of `inline` or `url`. */
source: DataSource;
/** For `source = url`: the remote address (retained for Refresh). */
url?: string;
/** For `source = url`: ISO timestamp of the fetch that produced `data`. */
fetchedAt?: string | null;
/** Optional free-form note. */
comment?: string;
/** Clock injection for deterministic tests; defaults to the current time. */
@@ -253,7 +324,7 @@ export interface CreateDatasetOptions {
export function createDataset(options: CreateDatasetOptions): Dataset {
const now = options.now ?? new Date();
const iso = now.toISOString();
const profile = computeDatasetProfile(options.data, options.format, options.source);
const profile = computeDatasetProfile(options.data, options.format);
return {
id: options.id ?? Date.now(),
@@ -262,6 +333,8 @@ export function createDataset(options: CreateDatasetOptions): Dataset {
data: options.data,
format: options.format,
source: options.source,
// URL datasets carry their address + fetch time; inline records stay clean.
...(options.source === 'url' ? { url: options.url, fetchedAt: options.fetchedAt ?? null } : {}),
comment: options.comment ?? '',
rowCount: profile.rowCount,
columnCount: profile.columnCount,
+36
View File
@@ -233,6 +233,42 @@ describe('normalizeImport — dataset normalization', () => {
expect(d.version).toBe(CURRENT_DATASET_VERSION);
});
it('v1→v2: a legacy URL dataset (address in data) imports as an unfetched reference', () => {
const parsed = {
version: '1.0',
snippets: [],
datasets: [{ id: 1, name: 'Remote', source: 'url', format: 'csv', data: 'https://x/y.csv' }],
};
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
expect(d.source).toBe('url');
expect(d.url).toBe('https://x/y.csv');
expect(d.data).toBeNull();
expect(d.fetchedAt).toBeNull();
expect(d.version).toBe(CURRENT_DATASET_VERSION);
});
it('a v2 URL snapshot imports with its data, url, and fetchedAt intact', () => {
const parsed = {
version: '1.0',
snippets: [],
datasets: [
{
id: 1,
name: 'Remote',
source: 'url',
format: 'csv',
data: 'a,b\n1,2',
url: 'https://x/y.csv',
fetchedAt: '2026-06-10T00:00:00.000Z',
},
],
};
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
expect(d.data).toBe('a,b\n1,2');
expect(d.url).toBe('https://x/y.csv');
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
});
it('fills gaps with defaults and coerces id to a number', () => {
const parsed = { version: '1.0', snippets: [], datasets: [{ id: '42', name: 'D' }] };
const result = normalizeImport(parsed, { now: FIXED_NOW });
+19 -2
View File
@@ -151,6 +151,11 @@ function normalizeSnippet(raw: unknown, nowIso: string, makeId: () => string): S
* datasets already carry their derived summary fields (rowCount, columns, …); we
* preserve those and only fill gaps. We do NOT re-profile here — that needs the
* profiling pipeline and would be wasteful for already-summarized records.
*
* Applies the v1→v2 URL-snapshot shaping (mirrors `migrateDataset`): a pre-v2 export
* stored a URL dataset's address in `data`, so we move it into `url` and clear `data`
* to `null` — importing such a record yields an unfetched reference, not a record
* with a URL string masquerading as its snapshot.
*/
function normalizeDataset(raw: unknown, nowIso: string): Dataset {
const r = isPlainObject(raw) ? raw : {};
@@ -158,13 +163,25 @@ function normalizeDataset(raw: unknown, nowIso: string): Dataset {
const created = asNonEmptyString(r.created) ?? nowIso;
const modified = asNonEmptyString(r.modified) ?? created;
const source = (typeof r.source === 'string' ? r.source : 'inline') as DataSource;
const legacyUrl = source === 'url' && typeof r.url !== 'string';
const url = legacyUrl
? typeof r.data === 'string'
? r.data
: undefined
: typeof r.url === 'string'
? r.url
: undefined;
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
return {
id: typeof r.id === 'number' ? r.id : Number(r.id) || 0,
version: CURRENT_DATASET_VERSION,
name: typeof r.name === 'string' ? r.name : 'Untitled',
data: r.data,
data: legacyUrl ? null : r.data,
format: (typeof r.format === 'string' ? r.format : 'json') as DataFormat,
source: (typeof r.source === 'string' ? r.source : 'inline') as DataSource,
source,
...(source === 'url' ? { url, fetchedAt } : {}),
comment: typeof r.comment === 'string' ? r.comment : '',
rowCount: typeof r.rowCount === 'number' ? r.rowCount : null,
columnCount: typeof r.columnCount === 'number' ? r.columnCount : null,
+17 -3
View File
@@ -111,7 +111,16 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
format: 'topojson',
source: 'inline',
},
{ name: 'UrlDs', data: 'https://x/y.csv', format: 'csv', source: 'url' },
// A fetched URL snapshot carries its payload in `data` (like inline); an
// unfetched reference has `data: null` and only its `url`.
{
name: 'UrlFetchedDs',
data: 'a,b\n1,2',
url: 'https://x/y.csv',
format: 'csv',
source: 'url',
},
{ name: 'UrlUnfetchedDs', data: null, url: 'https://x/y.csv', format: 'csv', source: 'url' },
];
test('inline JSON → values inlined', () => {
@@ -140,8 +149,13 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
});
});
test('URL → url reference tagged with the dataset format', () => {
const out = prepareSpecForRender({ data: { name: 'UrlDs' } }, { datasets });
test('fetched URL snapshot → its payload inlined, tagged with the format (like inline)', () => {
const out = prepareSpecForRender({ data: { name: 'UrlFetchedDs' } }, { datasets });
expect(out.data).toEqual({ values: 'a,b\n1,2', format: { type: 'csv' } });
});
test('unfetched URL reference → live url fallback tagged with the format', () => {
const out = prepareSpecForRender({ data: { name: 'UrlUnfetchedDs' } }, { datasets });
expect(out.data).toEqual({ url: 'https://x/y.csv', format: { type: 'csv' } });
});
+11 -2
View File
@@ -42,6 +42,8 @@ export interface ResolvableDataset {
format: DataFormat;
/** One of `inline` or `url`. */
source: DataSource;
/** For `source = url`: the remote address, used only as the unfetched fallback. */
url?: string;
}
/** Thrown when a spec references a library dataset name that does not exist. */
@@ -129,13 +131,20 @@ function selfDefinedDatasetNames(spec: unknown): Set<string> {
*/
function resolvedData(dataset: ResolvableDataset, rest: SpecNode): SpecNode {
const restFormat = isSpecNode(rest.format) ? rest.format : {};
if (dataset.source === 'url') {
// An unfetched URL reference (a legacy record, or one whose snapshot fetch never
// succeeded) has no local data — fall back to a live Vega-Lite URL fetch so it
// still renders until a Refresh snapshots it (snapshot model; spec §04 step 1).
if (dataset.source === 'url' && dataset.data == null) {
return {
...rest,
url: typeof dataset.data === 'string' ? dataset.data : (JSON.stringify(dataset.data) ?? ''),
url: typeof dataset.url === 'string' ? dataset.url : '',
format: { ...restFormat, type: dataset.format },
};
}
// Inline data and fetched URL snapshots resolve identically: the payload is in
// `data`, shaped by format.
switch (dataset.format) {
case 'json':
return { ...rest, values: dataset.data };