Snapshot URL datasets locally on add; preview tabular data as a table

This commit is contained in:
2026-06-10 10:24:42 +03:00
parent eb5e7ac53a
commit 2410c6e965
23 changed files with 1239 additions and 148 deletions
@@ -235,6 +235,32 @@
color: var(--text-secondary);
}
/* URL dataset provenance: the source address + last-fetched time (spec §05). */
.sourceMeta {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: var(--space-2) var(--space-4);
margin: 0;
font-size: 12px;
color: var(--text-secondary);
}
.sourceLink {
color: var(--accent);
text-decoration: none;
overflow-wrap: anywhere;
}
.sourceLink:hover {
text-decoration: underline;
}
.sourceLink:focus-visible {
outline: 2px solid var(--focus);
outline-offset: 2px;
}
.section {
display: flex;
flex-direction: column;
@@ -335,6 +361,46 @@
white-space: pre;
}
/* Tabular preview — a scrollable grid with a sticky header (spec §05 → Preview). */
.previewTableWrap {
max-height: 260px;
overflow: auto;
border: var(--border-width) solid var(--border);
border-radius: var(--radius);
}
.previewTable {
border-collapse: collapse;
width: 100%;
font-family: var(--font-mono);
font-size: 12px;
line-height: 1.4;
}
.previewTable th,
.previewTable td {
text-align: left;
padding: var(--space-2) var(--space-3);
border-bottom: var(--border-width) solid var(--border);
white-space: nowrap;
max-width: 240px;
overflow: hidden;
text-overflow: ellipsis;
}
.previewTable thead th {
position: sticky;
top: 0;
z-index: 1;
background: var(--layer-01);
font-weight: 600;
color: var(--text-secondary);
}
.previewTable tbody tr:last-child td {
border-bottom: none;
}
.muted {
margin: 0;
font-size: 13px;
@@ -439,3 +505,10 @@
font-size: 13px;
color: var(--support-error);
}
/* The inline-fallback recovery under a failed URL fetch — left-aligned, not
stretched, so it reads as a secondary recovery beneath the error message. */
.fallback {
align-self: flex-start;
margin-top: calc(-1 * var(--space-2));
}
+187 -22
View File
@@ -12,12 +12,14 @@
* Snippets stay reactive without a stored back-pointer.
*/
import { useState } from 'react';
import { useMemo, useState } from 'react';
import { useShallow } from 'zustand/react/shallow';
import { datasetReference, type DataSource, type Dataset } from '@core/dataset';
import { datasetReference, tabularRows, type DataSource, type Dataset } from '@core/dataset';
import { detectFormat, detectFormatFromUrl, type DataFormat } from '@core/format-detection';
import { datasetUsageCounts, snippetsReferencingDataset } from '@core/relationships';
import { closeModal, openModal, resnapshot } from '../modals/ModalCoordinator';
import { fetchRemoteData } from '../infrastructure/remote-data';
import { remoteFetchErrorMessage } from '../services/remote-data-errors';
import { confirm } from '../stores/ConfirmStore';
import { notify } from '../stores/NotificationStore';
import {
@@ -50,6 +52,17 @@ const SOURCE_OPTIONS: ReadonlyArray<SegmentedOption<DataSource>> = [
{ value: 'url', label: 'URL' },
];
/** Rows shown in the tabular preview before truncating (spec §05 → Detail Panel). */
const PREVIEW_ROW_LIMIT = 50;
/** One table cell's text: blank for empty, the string as-is, else JSON (numbers,
* booleans, nested values). Avoids `String()` on objects ("[object Object]"). */
function cellText(value: unknown): string {
if (value == null) return '';
if (typeof value === 'string') return value;
return JSON.stringify(value);
}
export function DatasetsModal() {
const datasets = useDatasetStore(useShallow((s) => s.datasets));
const view = useDatasetStore((s) => s.view);
@@ -116,11 +129,15 @@ function DatasetListItem({
onSelect: () => void;
}) {
// Meta line: source ("URL" prefix), row count when known, format label, size.
// A fetched URL snapshot reads like an inline dataset (rows + size); an unfetched
// URL reference shows "not fetched" in place of figures it doesn't have yet.
const unfetched = dataset.source === 'url' && dataset.data == null;
const parts: string[] = [];
if (dataset.source === 'url') parts.push('URL');
if (dataset.rowCount !== null) parts.push(`${dataset.rowCount} rows`);
if (unfetched) parts.push('not fetched');
else if (dataset.rowCount !== null) parts.push(`${dataset.rowCount} rows`);
parts.push(formatLabel(dataset.format));
if (dataset.source !== 'url') parts.push(humanBytes(dataset.size));
if (!unfetched) parts.push(humanBytes(dataset.size));
return (
<li className={`${styles.item} ${active ? styles.itemActive : ''}`}>
@@ -151,16 +168,46 @@ function DatasetDetail({
}) {
const startEdit = useDatasetStore((s) => s.startEdit);
const remove = useDatasetStore((s) => s.remove);
const refreshDataset = useDatasetStore((s) => s.refreshDataset);
const selectSnippet = useSnippetStore((s) => s.selectSnippet);
const [copied, setCopied] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const linked = snippetsReferencingDataset(snippets, dataset.name);
// Tabular data (CSV/TSV/JSON-array, inline or fetched) previews as a table of the
// first rows under the profiled columns; non-tabular payloads fall back to text.
// Memoized on the record so a large CSV isn't re-parsed on unrelated re-renders.
const previewRows = useMemo(
() => tabularRows(dataset.data, dataset.format, PREVIEW_ROW_LIMIT),
[dataset],
);
const handleEdit = () => {
startEdit();
resnapshot();
};
// Re-fetch a URL dataset's source and re-snapshot it. The visible result (updated
// rows + "Fetched" time) is the confirmation, so success raises no toast; only a
// failure surfaces one (spec §05 → Actions; docs/architecture/10 → Toast copy).
const handleRefresh = async () => {
if (!dataset.url) return;
setRefreshing(true);
try {
const { text } = await fetchRemoteData(dataset.url);
refreshDataset(dataset.id, { text });
} catch (err) {
notify({
kind: 'error',
title: "Couldn't refresh dataset",
message: remoteFetchErrorMessage(err, 'retry'),
});
} finally {
setRefreshing(false);
}
};
const handleCopy = async () => {
const text = JSON.stringify(datasetReference(dataset.name), null, 2);
try {
@@ -212,6 +259,16 @@ function DatasetDetail({
<span role="status" className="visually-hidden">
{copied ? 'Reference copied to clipboard' : ''}
</span>
{dataset.source === 'url' && (
<button
type="button"
className={styles.action}
onClick={() => void handleRefresh()}
disabled={refreshing}
>
{refreshing ? 'Refreshing…' : 'Refresh'}
</button>
)}
<button type="button" className={styles.action} onClick={handleEdit}>
Edit
</button>
@@ -237,6 +294,19 @@ function DatasetDetail({
{dataset.comment && <p className={styles.comment}>{dataset.comment}</p>}
{dataset.source === 'url' && (
<p className={styles.sourceMeta}>
<a className={styles.sourceLink} href={dataset.url} target="_blank" rel="noreferrer">
{dataset.url}
</a>
<span>
{dataset.fetchedAt
? `Fetched ${new Date(dataset.fetchedAt).toLocaleString()}`
: 'Not fetched yet'}
</span>
</p>
)}
<section className={styles.section}>
<h4 className={styles.sectionTitle}>Overview</h4>
<dl className={styles.stats}>
@@ -254,7 +324,7 @@ function DatasetDetail({
</div>
<div>
<dt>Size</dt>
<dd>{dataset.source === 'url' ? 'N/A' : humanBytes(dataset.size)}</dd>
<dd>{dataset.data == null ? 'N/A' : humanBytes(dataset.size)}</dd>
</div>
</dl>
@@ -277,7 +347,43 @@ function DatasetDetail({
<section className={styles.section}>
<h4 className={styles.sectionTitle}>Preview</h4>
<pre className={styles.preview}>{previewText(dataset)}</pre>
{previewRows ? (
<>
{/* TODO: a11y — this scroll container isn't keyboard-reachable; when the
preview overflows, keyboard-only users can't scroll it (WCAG 2.1.1).
A council pass should decide the pattern (tabindex=0 + aria-label region,
or a different overflow treatment) before we lean on it more. */}
<div className={styles.previewTableWrap}>
<table className={styles.previewTable}>
<thead>
<tr>
{dataset.columns.map((col, ci) => (
<th key={ci} scope="col">
{col}
</th>
))}
</tr>
</thead>
<tbody>
{previewRows.map((row, ri) => (
<tr key={ri}>
{dataset.columns.map((col, ci) => (
<td key={ci}>{cellText(row[col])}</td>
))}
</tr>
))}
</tbody>
</table>
</div>
{dataset.rowCount != null && dataset.rowCount > previewRows.length && (
<p className={styles.muted}>
Showing the first {previewRows.length} of {dataset.rowCount} rows.
</p>
)}
</>
) : (
<pre className={styles.preview}>{previewText(dataset)}</pre>
)}
</section>
<section className={styles.section}>
@@ -307,19 +413,22 @@ function DatasetDetail({
);
}
/** A truncated rendering of the data: raw for csv/tsv/url, pretty JSON otherwise. */
/** A truncated rendering of the data: raw for csv/tsv, pretty JSON otherwise. */
function previewText(dataset: Dataset): string {
const MAX = 2000;
// No snapshot yet (an unfetched URL reference) — there is nothing to preview.
if (dataset.data == null) {
return dataset.source === 'url' ? 'Not fetched yet — use Refresh to load the data.' : '';
}
let text: string;
if (dataset.source === 'url') {
text = String(dataset.data);
} else if (dataset.format === 'csv' || dataset.format === 'tsv') {
text = String(dataset.data);
if (dataset.format === 'csv' || dataset.format === 'tsv') {
// CSV/TSV payloads are raw text; fall back to JSON for any non-string value.
text = typeof dataset.data === 'string' ? dataset.data : JSON.stringify(dataset.data);
} else {
try {
text = JSON.stringify(dataset.data, null, 2);
} catch {
text = String(dataset.data);
text = typeof dataset.data === 'string' ? dataset.data : '[unserializable data]';
}
}
return text.length > MAX ? `${text.slice(0, MAX)}\n…` : text;
@@ -328,20 +437,59 @@ function previewText(dataset: Dataset): string {
function DatasetFormView({ editing }: { editing: boolean }) {
const form = useDatasetStore((s) => s.form);
const formError = useDatasetStore((s) => s.formError);
const selected = useDatasetStore(selectSelectedDataset);
const updateForm = useDatasetStore((s) => s.updateForm);
const cancelForm = useDatasetStore((s) => s.cancelForm);
const save = useDatasetStore((s) => s.save);
const commitUrlSnapshot = useDatasetStore((s) => s.commitUrlSnapshot);
// Save stays disabled until a name and valid data/URL are present (spec §05).
const canSave = useDatasetStore(selectCanSave);
// URL datasets fetch on save (snapshot model): `fetching` drives the button's
// busy state; `fetchError` holds a failed fetch's message + the inline fallback.
const [fetching, setFetching] = useState(false);
const [fetchError, setFetchError] = useState<string | null>(null);
// Live format/source hint from the current input (spec §05 → Auto-detection).
const detected =
form.source === 'url'
? { format: detectFormatFromUrl(form.input.trim()), confidence: 'url' as const }
: detectFormat(form.input);
const handleSave = () => {
if (save()) resnapshot(); // committed — re-baseline so a later close won't prompt
const handleSave = async () => {
// Inline saves are synchronous; only URL datasets touch the network.
if (form.source !== 'url') {
if (save()) resnapshot(); // committed — re-baseline so a later close won't prompt
return;
}
const url = form.input.trim();
// A metadata-only edit (same URL, snapshot already present) needs no re-fetch.
const metaOnly =
editing &&
selected?.source === 'url' &&
selected.data != null &&
url === (selected.url ?? '');
if (metaOnly) {
if (save()) resnapshot();
return;
}
// Create, changed URL, or inline→URL: fetch once, then snapshot + commit.
setFetchError(null);
setFetching(true);
try {
const { text } = await fetchRemoteData(url);
if (commitUrlSnapshot({ text })) resnapshot();
} catch (err) {
setFetchError(remoteFetchErrorMessage(err));
} finally {
setFetching(false);
}
};
// Recovery from a failed fetch: switch to an inline paste, keeping name + comment.
const handlePasteInline = () => {
setFetchError(null);
updateForm({ source: 'inline', input: '' });
};
const handleCancel = () => {
@@ -370,7 +518,10 @@ function DatasetFormView({ editing }: { editing: boolean }) {
label="Dataset source"
options={SOURCE_OPTIONS}
value={form.source}
onChange={(source) => updateForm({ source })}
onChange={(source) => {
setFetchError(null);
updateForm({ source });
}}
/>
</div>
@@ -381,7 +532,10 @@ function DatasetFormView({ editing }: { editing: boolean }) {
type="url"
className={styles.input}
value={form.input}
onChange={(e) => updateForm({ input: e.target.value })}
onChange={(e) => {
setFetchError(null);
updateForm({ input: e.target.value });
}}
placeholder="https://example.com/data.csv"
/>
) : (
@@ -418,23 +572,34 @@ function DatasetFormView({ editing }: { editing: boolean }) {
/>
</label>
{formError && (
{(formError || fetchError) && (
<p className={styles.formError} role="alert">
{formError}
{formError ?? fetchError}
</p>
)}
{/* A failed fetch is recoverable by pasting the data inline (the user's choice
when CORS or offline blocks the URL) — offered as a direct one-click path. */}
{fetchError && (
<button
type="button"
className={`${styles.action} ${styles.fallback}`}
onClick={handlePasteInline}
>
Paste data inline instead
</button>
)}
<div className={styles.formActions}>
<button type="button" className={styles.action} onClick={handleCancel}>
<button type="button" className={styles.action} onClick={handleCancel} disabled={fetching}>
Cancel
</button>
<button
type="button"
className={`${styles.action} ${styles.primary}`}
disabled={!canSave}
onClick={handleSave}
disabled={!canSave || fetching}
onClick={() => void handleSave()}
>
{editing ? 'Save changes' : 'Create dataset'}
{fetching ? 'Fetching…' : editing ? 'Save changes' : 'Create dataset'}
</button>
</div>
</div>
@@ -0,0 +1,73 @@
import { describe, expect, test } from 'vitest';
import { CURRENT_DATASET_VERSION } from '@core/dataset';
import { migrateDataset } from './dataset-migrations';
describe('migrateDataset', () => {
test('stamps the current version and fills missing fields with defaults', () => {
const d = migrateDataset({ id: 7, data: 'a,b\n1,2', format: 'csv', source: 'inline' });
expect(d.id).toBe(7);
expect(d.version).toBe(CURRENT_DATASET_VERSION);
expect(d.name).toBe('Untitled');
expect(d.comment).toBe('');
expect(d.columns).toEqual([]);
expect(d.columnStats).toEqual([]);
});
test('an inline record carries no url/fetchedAt keys', () => {
const d = migrateDataset({ id: 1, data: [{ a: 1 }], format: 'json', source: 'inline' });
expect(d.source).toBe('inline');
expect('url' in d).toBe(false);
expect('fetchedAt' in d).toBe(false);
});
test('v1→v2: a legacy URL record moves its address out of data into url', () => {
// Pre-v2 shape: the URL lived in `data`, there was no `url`/`fetchedAt`, and the
// profile was N/A. It becomes an unfetched reference: url set, data cleared.
const d = migrateDataset({
id: 2,
data: 'https://example.com/data.csv',
format: 'csv',
source: 'url',
rowCount: null,
columns: [],
});
expect(d.version).toBe(CURRENT_DATASET_VERSION);
expect(d.source).toBe('url');
expect(d.url).toBe('https://example.com/data.csv');
expect(d.data).toBeNull();
expect(d.fetchedAt).toBeNull();
expect(d.rowCount).toBeNull();
});
test('a v2 URL snapshot is preserved (data, url, and fetchedAt all kept)', () => {
const d = migrateDataset({
id: 3,
version: 2,
data: 'a,b\n1,2',
url: 'https://example.com/data.csv',
fetchedAt: '2026-06-10T00:00:00.000Z',
format: 'csv',
source: 'url',
rowCount: 1,
columns: ['a', 'b'],
});
expect(d.data).toBe('a,b\n1,2');
expect(d.url).toBe('https://example.com/data.csv');
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
expect(d.rowCount).toBe(1);
});
test('preserves unknown fields so a newer build round-trips without loss', () => {
const d = migrateDataset({
id: 1,
data: [],
format: 'json',
source: 'inline',
futureField: 42,
});
expect((d as unknown as Record<string, unknown>).futureField).toBe(42);
});
});
+21 -2
View File
@@ -30,14 +30,33 @@ function asCount(v: unknown): number | null {
/** Upgrade a raw stored record to the current Dataset shape. */
export function migrateDataset(raw: unknown): Dataset {
const r = { ...(raw as Record<string, unknown>) };
const source = asSource(r.source);
// v1→v2: a URL dataset used to store its address in `data`; it now stores the
// fetched snapshot in `data` and keeps the address in `url` (+ a `fetchedAt`).
// A legacy record (url source, no `url` field) becomes an *unfetched* reference:
// move the address to `url` and clear `data`. The N/A profile carries over as-is;
// rendering falls back to the live URL until a Refresh fetches + profiles it.
const legacyUrl = source === 'url' && typeof r.url !== 'string';
const url = legacyUrl
? typeof r.data === 'string'
? r.data
: undefined
: typeof r.url === 'string'
? r.url
: undefined;
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
return {
...r,
id: typeof r.id === 'number' ? r.id : Number(r.id),
version: CURRENT_DATASET_VERSION,
name: typeof r.name === 'string' ? r.name : 'Untitled',
data: r.data,
data: legacyUrl ? null : r.data,
format: asFormat(r.format),
source: asSource(r.source),
source,
// Only URL datasets carry url/fetchedAt; inline records stay clean.
...(source === 'url' ? { url, fetchedAt } : {}),
comment: typeof r.comment === 'string' ? r.comment : '',
rowCount: asCount(r.rowCount),
columnCount: asCount(r.columnCount),
@@ -0,0 +1,94 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { fetchRemoteData, RemoteFetchError } from './remote-data';
/** A minimal Response stand-in for the bits the adapter reads. */
function fakeResponse(opts: {
ok: boolean;
status?: number;
statusText?: string;
body?: string;
contentType?: string | null;
}): Response {
return {
ok: opts.ok,
status: opts.status ?? (opts.ok ? 200 : 500),
statusText: opts.statusText ?? '',
text: () => Promise.resolve(opts.body ?? ''),
headers: {
get: (k: string) => (k.toLowerCase() === 'content-type' ? (opts.contentType ?? null) : null),
},
} as unknown as Response;
}
afterEach(() => vi.unstubAllGlobals());
describe('fetchRemoteData', () => {
test('returns the body and content-type on success', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() =>
Promise.resolve(fakeResponse({ ok: true, body: 'a,b\n1,2', contentType: 'text/csv' })),
),
);
const result = await fetchRemoteData('https://example.com/data.csv');
expect(result.text).toBe('a,b\n1,2');
expect(result.contentType).toBe('text/csv');
});
test('classifies a non-2xx response as http with its status', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() =>
Promise.resolve(fakeResponse({ ok: false, status: 404, statusText: 'Not Found' })),
),
);
await expect(fetchRemoteData('https://example.com/missing.csv')).rejects.toMatchObject({
reason: 'http',
status: 404,
});
});
test('classifies a rejected fetch (CORS / offline) as network', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
);
await expect(fetchRemoteData('https://blocked.example/data.csv')).rejects.toMatchObject({
reason: 'network',
});
});
test('classifies an empty body as empty', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => Promise.resolve(fakeResponse({ ok: true, body: ' \n' }))),
);
await expect(fetchRemoteData('https://example.com/blank.csv')).rejects.toMatchObject({
reason: 'empty',
});
});
test('classifies an aborted fetch as timeout', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => {
const e = new Error('aborted');
e.name = 'AbortError';
return Promise.reject(e);
}),
);
await expect(
fetchRemoteData('https://slow.example/data.csv', { timeoutMs: 1 }),
).rejects.toMatchObject({
reason: 'timeout',
});
});
test('the thrown error is a RemoteFetchError', async () => {
vi.stubGlobal(
'fetch',
vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
);
await expect(fetchRemoteData('https://x.example')).rejects.toBeInstanceOf(RemoteFetchError);
});
});
+99
View File
@@ -0,0 +1,99 @@
/**
* Remote-data fetch adapter (snapshot model — spec §05 → URL datasets).
*
* The ONLY place Astrolabe touches the network. Adding a URL dataset fetches the
* resource once here; core then profiles the returned text exactly like inline
* data (see dataset.ts). Keeping the fetch in infrastructure preserves the rule
* that `src/core` stays pure and that the rest of the app never calls `fetch`
* directly (AGENTS → Architecture).
*
* Failures are classified into a small set of machine-readable `reason`s and
* thrown as `RemoteFetchError`; the user-facing wording (and the "paste inline
* instead" fallback) lives in the UI layer, where it is council-reviewed — the
* same split as `storage-errors`. A browser cross-origin block and an
* offline/DNS failure are indistinguishable here — both reject with a `TypeError`
* and no status — so they share the `network` reason and the UI hedges the cause.
*/
/** Why a remote fetch failed, in terms the UI maps to copy + a recovery path. */
export type RemoteFetchReason = 'network' | 'http' | 'empty' | 'timeout';
/** A classified remote-fetch failure. `status` is set only for `http`. */
export class RemoteFetchError extends Error {
readonly reason: RemoteFetchReason;
readonly status?: number;
constructor(reason: RemoteFetchReason, message: string, status?: number) {
super(message);
this.name = 'RemoteFetchError';
this.reason = reason;
this.status = status;
}
}
/** A successful fetch: the raw body plus a weak format hint from the server. */
export interface RemoteData {
/** The response body, ready for format detection + profiling. */
text: string;
/** The server's `Content-Type`, when provided — a tertiary format hint. */
contentType: string | null;
}
/** Default ceiling on a single fetch before it is aborted as a timeout. */
const DEFAULT_TIMEOUT_MS = 30_000;
export interface FetchRemoteDataOptions {
/** Abort the fetch after this many ms (default 30 s). */
timeoutMs?: number;
}
function describeNetworkError(err: unknown): string {
const reason = err instanceof Error ? err.message : String(err);
return `Couldn't reach the URL (${reason}).`;
}
/**
* Fetch a remote dataset resource once and return its raw body. Throws
* `RemoteFetchError` on any failure — the caller distinguishes recovery by
* `reason`. The body is returned verbatim; format detection and profiling happen
* downstream so this adapter stays free of core logic.
*/
export async function fetchRemoteData(
url: string,
options: FetchRemoteDataOptions = {},
): Promise<RemoteData> {
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
let response: Response;
try {
response = await fetch(url, { redirect: 'follow', signal: controller.signal });
} catch (err) {
// A timeout aborts via the controller (AbortError); everything else here is a
// CORS block, offline, DNS failure, or malformed URL — all opaque TypeErrors.
if ((err as { name?: string }).name === 'AbortError') {
throw new RemoteFetchError('timeout', `The URL took longer than ${timeoutMs} ms to respond.`);
}
throw new RemoteFetchError('network', describeNetworkError(err));
} finally {
clearTimeout(timer);
}
if (!response.ok) {
throw new RemoteFetchError(
'http',
`The server responded ${response.status} ${response.statusText}.`.trim(),
response.status,
);
}
const text = await response.text();
if (text.trim() === '') {
throw new RemoteFetchError('empty', 'The URL returned no data.');
}
// TODO: guard very large responses (cap or warn) before snapshotting into
// IndexedDB — a multi-hundred-MB file would load fully into memory + storage.
return { text, contentType: response.headers.get('content-type') };
}
@@ -0,0 +1,38 @@
import { describe, expect, test } from 'vitest';
import { RemoteFetchError } from '../infrastructure/remote-data';
import { remoteFetchErrorMessage } from './remote-data-errors';
describe('remoteFetchErrorMessage', () => {
test('a network failure hedges offline vs cross-origin and offers the inline fallback', () => {
const msg = remoteFetchErrorMessage(new RemoteFetchError('network', 'x'));
expect(msg).toMatch(/offline/i);
expect(msg).toMatch(/cross-origin/i);
expect(msg).toMatch(/paste the data inline/i);
});
test('an HTTP 404 explains "not found" without a raw error code', () => {
const msg = remoteFetchErrorMessage(new RemoteFetchError('http', 'x', 404));
expect(msg).toMatch(/couldn't find/i);
expect(msg).not.toMatch(/404/);
});
test('an HTTP 403 explains refused access', () => {
expect(remoteFetchErrorMessage(new RemoteFetchError('http', 'x', 403))).toMatch(
/refused access/i,
);
});
test('an empty body says it returned no data', () => {
expect(remoteFetchErrorMessage(new RemoteFetchError('empty', 'x'))).toMatch(/no data/i);
});
test('the retry recovery points at refreshing, not pasting inline', () => {
const msg = remoteFetchErrorMessage(new RemoteFetchError('timeout', 'x'), 'retry');
expect(msg).toMatch(/refreshing again/i);
expect(msg).not.toMatch(/inline/i);
});
test('an unknown error still produces a sensible fallback message', () => {
expect(remoteFetchErrorMessage(new Error('boom'))).toMatch(/couldn't fetch this url/i);
});
});
+51
View File
@@ -0,0 +1,51 @@
/**
* Translate a remote-data fetch failure (snapshot model — spec §05 → URL datasets)
* into the message shown when adding or refreshing a URL dataset.
*
* Pure and framework-free so the wording is unit-tested directly — the same split
* as `storage-errors`. Follows the recorded error-copy resolution (docs/architecture
* /10 → error copy): plain language, name what stopped, and — because every one of
* these is user-fixable — point at the next step. No error codes in the user-facing
* line (GOV.UK / NN/g #9). The recovery clause adapts to where the error shows: the
* create/edit form can fall back to pasting inline; Refresh can only retry.
*/
import { RemoteFetchError } from '../infrastructure/remote-data';
/** Where the error surfaces, which decides the recovery the copy points at. */
export type FetchRecovery = 'inline' | 'retry';
function nextStep(recovery: FetchRecovery): string {
return recovery === 'inline'
? 'Check the address, or paste the data inline instead.'
: 'Check the address and try refreshing again.';
}
function httpCause(status?: number): string {
if (status === 404) return "The server couldn't find anything at this URL.";
if (status === 401 || status === 403) return 'The server refused access to this URL.';
if (status && status >= 500) return 'The server hit an error serving this URL.';
return "The server couldn't return this URL.";
}
/**
* The user-facing message for a fetch failure. `recovery` defaults to `inline`
* (the add/edit form); pass `retry` for the Refresh action, which has no inline
* fallback.
*/
export function remoteFetchErrorMessage(err: unknown, recovery: FetchRecovery = 'inline'): string {
const next = nextStep(recovery);
if (err instanceof RemoteFetchError) {
switch (err.reason) {
case 'network':
return `Couldn't fetch this URL. Your device may be offline, or the host may block cross-origin requests. ${next}`;
case 'http':
return `${httpCause(err.status)} ${next}`;
case 'empty':
return `This URL returned no data. Make sure it points directly at a data file. ${next}`;
case 'timeout':
return `This URL took too long to respond. ${next}`;
}
}
return `Couldn't fetch this URL. ${next}`;
}
+70 -3
View File
@@ -42,15 +42,82 @@ describe('save — create', () => {
expect(ds?.rowCount).toBe(2);
});
test('a valid URL infers format from the extension', () => {
test('save() does not commit a URL create — it must go through commitUrlSnapshot', () => {
store().startCreate();
store().updateForm({ name: 'Remote', source: 'url', input: 'https://example.com/data.csv' });
// The component fetches first; save() never touches the network, so a URL create
// through save() is a no-op rather than an unfetched record.
expect(store().save(T)).toBe(false);
expect(store().datasets).toHaveLength(0);
});
});
describe('commitUrlSnapshot — create from a fetched body', () => {
test('snapshots the body, infers format from content, and profiles it', () => {
store().startCreate();
store().updateForm({ name: 'Remote', source: 'url', input: 'https://example.com/data.csv' });
expect(store().save(T)).toBe(true);
expect(store().commitUrlSnapshot({ text: 'a,b\n1,2\n3,4' }, T)).toBe(true);
const ds = selectSelectedDataset(store());
expect(ds?.source).toBe('url');
expect(ds?.url).toBe('https://example.com/data.csv');
expect(ds?.format).toBe('csv');
expect(ds?.rowCount).toBeNull(); // URL datasets aren't profiled
expect(ds?.data).toBe('a,b\n1,2\n3,4');
expect(ds?.rowCount).toBe(2);
expect(ds?.columns).toEqual(['a', 'b']);
expect(ds?.fetchedAt).toBe(T.toISOString());
expect(store().view).toBe('detail');
});
test('a duplicate name is rejected even with a successful fetch', () => {
store().add(
createDataset({ name: 'Dupe', data: [{ a: 1 }], format: 'json', source: 'inline', now: T }),
);
store().startCreate();
store().updateForm({ name: 'dupe', source: 'url', input: 'https://x/y.csv' });
expect(store().commitUrlSnapshot({ text: 'a\n1' }, T)).toBe(false);
expect(store().formError).toMatch(/already exists/i);
});
});
describe('URL dataset edits & refresh', () => {
const seedUrl = () => {
store().startCreate();
store().updateForm({ name: 'Remote', source: 'url', input: 'https://x/y.csv' });
store().commitUrlSnapshot({ text: 'a,b\n1,2' }, T);
return store().selectedId!;
};
test('a metadata-only edit (same URL) updates name without re-fetching', () => {
const id = seedUrl();
store().startEdit();
store().updateForm({ name: 'Renamed' }); // URL field stays https://x/y.csv
const later = new Date('2026-07-01T00:00:00Z');
expect(store().save(later)).toBe(true);
const ds = store().datasets.find((d) => d.id === id)!;
expect(ds.name).toBe('Renamed');
expect(ds.data).toBe('a,b\n1,2'); // snapshot untouched
expect(ds.fetchedAt).toBe(T.toISOString()); // not re-fetched
expect(ds.modified).toBe(later.toISOString());
});
test('refreshDataset re-snapshots, re-profiles, and advances fetchedAt', () => {
const id = seedUrl();
const later = new Date('2026-07-01T00:00:00Z');
expect(store().refreshDataset(id, { text: 'a,b\n1,2\n3,4\n5,6' }, later)).toBe(true);
const ds = store().datasets.find((d) => d.id === id)!;
expect(ds.rowCount).toBe(3);
expect(ds.fetchedAt).toBe(later.toISOString());
expect(ds.modified).toBe(later.toISOString());
expect(ds.name).toBe('Remote'); // preserved
});
test('refreshDataset is a no-op for an inline dataset', () => {
store().add(
createDataset({ name: 'Inline', data: [{ a: 1 }], format: 'json', source: 'inline', now: T }),
);
const id = store().selectedId!;
expect(store().refreshDataset(id, { text: 'a\n1' }, T)).toBe(false);
});
});
+141 -19
View File
@@ -19,8 +19,14 @@
*/
import { create } from 'zustand';
import { detectFormat, detectFormatFromUrl, type DataFormat } from '@core/format-detection';
import { createDataset, computeDatasetProfile, type DataSource, type Dataset } from '@core/dataset';
import { detectFormat, type DataFormat } from '@core/format-detection';
import {
createDataset,
computeDatasetProfile,
snapshotFromText,
type DataSource,
type Dataset,
} from '@core/dataset';
import { isNameTaken } from '@core/naming';
import { useSnippetStore } from './SnippetStore';
@@ -67,6 +73,21 @@ export interface DatasetState {
*/
save: (now?: Date) => boolean;
/**
* Commit a URL dataset from an already-fetched body. The component performs the
* network fetch via the remote-data adapter and passes the result here, keeping
* this store browser-free. Validates the form, snapshots + profiles the body, and
* creates (view `new`) or updates (view `edit`) the dataset — including renaming
* referencing snippets on an edit. Returns whether it committed.
*/
commitUrlSnapshot: (fetched: { text: string }, now?: Date) => boolean;
/**
* Re-snapshot an existing URL dataset from a freshly-fetched body ("Refresh"):
* re-detect the format, re-profile, and advance `fetchedAt`/`modified`. Name,
* comment, and URL are preserved. Returns false for a non-URL dataset.
*/
refreshDataset: (id: number, fetched: { text: string }, now?: Date) => boolean;
/**
* Low-level: add a fully-formed dataset and select it. The id is (re)assigned
* via `nextDatasetId`, so a `createDataset` default id (`Date.now()`) can never
@@ -144,7 +165,12 @@ function validateForm(
return null;
}
/** Validate a form into a saveable shape, or return an error message. */
/**
* Validate an **inline** form into a saveable shape, or return an error message.
* URL datasets never reach here — they are fetched first and committed through
* `commitUrlSnapshot` (which snapshots the fetched body), so this only shapes the
* pasted-inline payload.
*/
function resolveForm(
form: DatasetForm,
datasets: Dataset[],
@@ -153,20 +179,12 @@ function resolveForm(
const error = validateForm(form, datasets, excludeId);
if (error) return { error };
const name = form.name.trim();
const input = form.input.trim();
if (form.source === 'url') {
// Format is inferred from the extension; default to JSON when unknown (§05).
return { name, data: input, format: detectFormatFromUrl(input) ?? 'json', source: 'url' };
}
// validateForm guaranteed a detectable format above.
const format = detectFormat(form.input).format as DataFormat;
// JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text (data model §09B).
const data =
format === 'json' || format === 'topojson' ? (JSON.parse(form.input) as unknown) : form.input;
return { name, data, format, source: 'inline' };
return { name: form.name.trim(), data, format, source: 'inline' };
}
export const useDatasetStore = create<DatasetState>((set, get) => ({
@@ -201,12 +219,14 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
name: ds.name,
source: ds.source,
comment: ds.comment,
// Re-render the stored payload as editable text: raw for csv/tsv/url,
// pretty-printed JSON for json/topojson.
// Re-render the editable text: the URL for url datasets, raw text for
// csv/tsv, pretty-printed JSON for json/topojson.
input:
ds.source === 'url' || ds.format === 'csv' || ds.format === 'tsv'
? String(ds.data)
: JSON.stringify(ds.data, null, 2),
ds.source === 'url'
? (ds.url ?? '')
: ds.format === 'csv' || ds.format === 'tsv'
? String(ds.data)
: JSON.stringify(ds.data, null, 2),
},
});
},
@@ -225,6 +245,30 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
const editing = view === 'edit';
const excludeId = editing ? (selectedId ?? undefined) : undefined;
// URL datasets that need the network — a create, a URL change, or an inline→URL
// conversion — are routed through `commitUrlSnapshot` after the component
// fetches; save() never fetches. The one URL case it commits is a metadata-only
// edit of an existing snapshot (same URL): just update name/comment, no re-fetch.
if (form.source === 'url') {
const error = validateForm(form, datasets, excludeId);
if (error) {
set({ formError: error });
return false;
}
if (!editing || selectedId === null) return false;
const existing = datasets.find((d) => d.id === selectedId);
if (!existing || existing.source !== 'url' || form.input.trim() !== (existing.url ?? '')) {
return false;
}
const name = form.name.trim();
get().update(selectedId, { name, comment: form.comment }, now);
if (name !== existing.name) {
useSnippetStore.getState().renameDatasetRefs(existing.name, name, now);
}
set({ view: 'detail', form: EMPTY_FORM, formError: null });
return true;
}
const resolved = resolveForm(form, datasets, excludeId);
if ('error' in resolved) {
set({ formError: resolved.error });
@@ -234,11 +278,12 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
if (editing && selectedId !== null) {
const existing = datasets.find((d) => d.id === selectedId);
if (!existing) return false;
const profile = computeDatasetProfile(resolved.data, resolved.format, resolved.source);
const profile = computeDatasetProfile(resolved.data, resolved.format);
// Re-profile and update the record (including any new name), then propagate
// the rename across referencing snippets so each spec and its datasetRefs
// stay consistent (docs/architecture/07 §6). SnippetStore never imports this
// store, so the direct call is cycle-free.
// store, so the direct call is cycle-free. Clear any url/fetchedAt left over
// from a URL→inline conversion so the record carries no stale remote origin.
get().update(
selectedId,
{
@@ -246,6 +291,13 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
data: resolved.data,
format: resolved.format,
source: resolved.source,
// TODO: `update` shallow-merges, so these set the keys to `undefined`
// rather than removing them — a URL→inline record keeps `url`/`fetchedAt`
// present-but-undefined. Invisible today (rendering/profiling key off
// `source`/`data == null`, and JSON export drops undefined), but it breaks
// the "inline records carry no url/fetchedAt keys" invariant on this path.
url: undefined,
fetchedAt: undefined,
comment: form.comment,
...profile,
},
@@ -275,6 +327,76 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
return true;
},
commitUrlSnapshot: (fetched, now) => {
const { view, form, datasets, selectedId } = get();
const editing = view === 'edit';
const excludeId = editing ? (selectedId ?? undefined) : undefined;
const error = validateForm(form, datasets, excludeId);
if (error) {
set({ formError: error });
return false;
}
const name = form.name.trim();
const url = form.input.trim();
const iso = (now ?? new Date()).toISOString();
// The fetched body snapshots + profiles exactly like inline data (snapshot
// model): detect format from content, shape, and profile through core.
const { data, format } = snapshotFromText(fetched.text, url);
if (editing && selectedId !== null) {
const existing = datasets.find((d) => d.id === selectedId);
if (!existing) return false;
const profile = computeDatasetProfile(data, format);
get().update(
selectedId,
{
name,
data,
format,
source: 'url',
url,
fetchedAt: iso,
comment: form.comment,
...profile,
},
now,
);
if (name !== existing.name) {
useSnippetStore.getState().renameDatasetRefs(existing.name, name, now);
}
set({ view: 'detail', form: EMPTY_FORM, formError: null });
return true;
}
const dataset = createDataset({
name,
data,
format,
source: 'url',
url,
fetchedAt: iso,
comment: form.comment,
now,
});
get().add(dataset);
set({ view: 'detail', form: EMPTY_FORM, formError: null });
return true;
},
refreshDataset: (id, fetched, now) => {
const dataset = get().datasets.find((d) => d.id === id);
if (!dataset || dataset.source !== 'url' || !dataset.url) return false;
const iso = (now ?? new Date()).toISOString();
const { data, format } = snapshotFromText(fetched.text, dataset.url);
const profile = computeDatasetProfile(data, format);
// Name, comment, and url are preserved; only the snapshot + its profile and the
// fetch time change. `update` advances `modified`.
get().update(id, { data, format, fetchedAt: iso, ...profile }, now);
return true;
},
add: (dataset) =>
set((s) => {
const withId = { ...dataset, id: nextDatasetId(s.datasets) };