mirror of
https://github.com/olehomelchenko/astrolabe.git
synced 2026-08-08 02:02:33 +00:00
Snapshot URL datasets locally on add; preview tabular data as a table
This commit is contained in:
@@ -268,7 +268,8 @@ the reference.
|
||||
- Snippet `datasetRefs` maintained on publish; library shows dataset icon +
|
||||
Linked Datasets; dataset detail shows Linked Snippets (bidirectional name link, §09F).
|
||||
- **Extract-to-Dataset** flow from the editor (§03F).
|
||||
- URL-sourced datasets fetched at render time.
|
||||
- URL datasets fetched once on add and **snapshotted** locally (profiled like inline;
|
||||
render from the snapshot, refreshable on demand — see §05 / §04 rendering contract).
|
||||
|
||||
**Tests**
|
||||
|
||||
|
||||
@@ -6,19 +6,20 @@ How Astrolabe stores data in the browser, and the rules that keep that storage t
|
||||
|
||||
## 1. The Infrastructure-Adapter Principle
|
||||
|
||||
**Rule: nothing outside `src/app/infrastructure/` ever touches `indexedDB`, `localStorage`, `window`, or `location` directly.** Every browser-storage interaction goes through a typed adapter module that exposes plain async functions returning domain objects.
|
||||
**Rule: nothing outside `src/app/infrastructure/` ever touches `indexedDB`, `localStorage`, `window`, `location`, or `fetch` directly.** Every browser interaction (storage _and_ network) goes through a typed adapter module that exposes plain async functions returning domain objects.
|
||||
|
||||
```
|
||||
src/
|
||||
├── core/ # portable engine — NO browser APIs, NO React
|
||||
├── app/
|
||||
│ ├── stores/ # Zustand stores; calls infrastructure, never IDB
|
||||
│ ├── services/ # business logic; calls infrastructure, never IDB
|
||||
│ └── infrastructure/ # the ONLY place that imports indexedDB/localStorage
|
||||
│ ├── stores/ # Zustand stores; calls infrastructure, never IDB/fetch
|
||||
│ ├── services/ # business logic; calls infrastructure, never IDB/fetch
|
||||
│ └── infrastructure/ # the ONLY place that imports indexedDB/localStorage/fetch
|
||||
│ ├── snippet-store.ts # IndexedDB: snippets (metadata + drafts)
|
||||
│ ├── dataset-store.ts # IndexedDB: datasets (heavy payloads)
|
||||
│ ├── settings-store.ts # localStorage: UserSettings
|
||||
│ └── ux-prefs.ts # localStorage: app/UI prefs (sort, panel layout)
|
||||
│ ├── ux-prefs.ts # localStorage: app/UI prefs (sort, panel layout)
|
||||
│ └── remote-data.ts # network: fetch a URL dataset's body (the ONLY fetch)
|
||||
```
|
||||
|
||||
### Why this boundary exists
|
||||
@@ -29,7 +30,15 @@ src/
|
||||
- **Failure containment.** Quota errors, corrupt JSON, and missing keys are handled at the boundary and converted into typed results (or sane fallbacks), so the rest of the app never sees a raw `DOMException`.
|
||||
|
||||
> **Do:** `import { saveSnippet } from '@/app/infrastructure/snippet-store'`
|
||||
> **Don't:** `indexedDB.open(...)` or `localStorage.getItem(...)` anywhere in a component, store, or service.
|
||||
> **Don't:** `indexedDB.open(...)`, `localStorage.getItem(...)`, or `fetch(...)` anywhere in a component, store, or service.
|
||||
|
||||
### Background vs. interactive adapters
|
||||
|
||||
Most adapters are driven by **background subscribers** (arch 01 §5, _Effects_): a store changes, a startup subscriber writes it through to IndexedDB — the store never calls the adapter itself. The **network** adapter is the exception. Fetching a URL dataset is a user-initiated action with its own pending/error UI, so the **component** calls `remote-data.ts` directly (components may call adapters — cf. `navigator.clipboard`) and hands the fetched body to **pure** store actions (`DatasetStore.commitUrlSnapshot` / `refreshDataset`). The store never fetches, so it stays browser-free and unit-testable on already-fetched text.
|
||||
|
||||
> **Rule:** keep `fetch` behind `remote-data.ts`; orchestrate the URL-dataset fetch in the _component_ (busy state + the "paste data inline instead" recovery), not the store. Store commit actions only ever receive already-fetched text.
|
||||
|
||||
**URL-dataset snapshot lifecycle** (the files a change to it touches): add / Refresh → component fetches (`infrastructure/remote-data.ts`) → `core/dataset.snapshotFromText` shapes the body by sniffed format → `DatasetStore.commitUrlSnapshot` / `refreshDataset` snapshots + profiles it _exactly like inline data_ → render resolves it cached-first in `core/rendering.resolvedData` (a live-URL fallback applies only while a URL dataset is still unfetched). See spec §05 for the behavior.
|
||||
|
||||
---
|
||||
|
||||
@@ -221,6 +230,8 @@ export async function saveSnippet(s: Snippet): Promise<void> {
|
||||
> **Do:** default `version` to the earliest shape (`1`) when the field is absent.
|
||||
> **Don't:** branch on the presence of individual fields scattered through the app to detect "old data." Centralize that knowledge in the migration function.
|
||||
|
||||
> **Mirror shape changes in the import normalizer.** Imported records are built from a file, not read from IndexedDB, so they **never pass through `migrate<Entity>`** — `core/import-normalize.ts` upgrades them independently. A migration that changes a field's _shape_ must be applied in both places or import produces a malformed record. (E.g. the dataset v1→v2 URL-snapshot reshaping — address moves from `data` into `url`, `data` cleared — lives in **both** `migrateDataset` and `normalizeDataset`.)
|
||||
|
||||
---
|
||||
|
||||
## 5. localStorage Preferences (Settings & App/UI Prefs)
|
||||
|
||||
@@ -183,7 +183,10 @@ how the UI shows **"N/A"** — see §3.2.
|
||||
|
||||
### 3.1 What gets profiled
|
||||
|
||||
Profiling applies only to **tabular inline data**:
|
||||
Profiling applies to any **tabular payload**, whether pasted inline or fetched
|
||||
from a URL (the snapshot model stores a URL dataset's data locally, so it profiles
|
||||
through the same path as inline data — `snapshotFromText` shapes the fetched body,
|
||||
then `computeDatasetProfile` runs):
|
||||
|
||||
- **JSON** that is an array of objects.
|
||||
- **CSV** (comma-separated, header row).
|
||||
@@ -191,8 +194,9 @@ Profiling applies only to **tabular inline data**:
|
||||
|
||||
Everything else is **not profiled**:
|
||||
|
||||
- **URL datasets** — the library holds only the link, not the data, so there is
|
||||
nothing to scan. Counts are `null` / N/A.
|
||||
- **Unfetched URL datasets** — a URL reference with no snapshot yet (e.g. one
|
||||
migrated from an older record), so there is nothing to scan. Counts are `null` /
|
||||
N/A until it is refreshed.
|
||||
- **Non-tabular data** — a single JSON object, TopoJSON, or anything we can't
|
||||
read as rows-of-columns. Counts are `null` / N/A.
|
||||
|
||||
|
||||
@@ -44,6 +44,23 @@
|
||||
- [ ] Toasts stack, auto-dismiss, and fade as specified; the live-preview busy
|
||||
indicator appears for slow (>~1s) renders and clears after.
|
||||
|
||||
## URL datasets (remote data snapshot)
|
||||
|
||||
> Needs a real network, real CORS, and real offline — tests mock the fetch.
|
||||
|
||||
- [ ] Add a dataset by URL from a CORS-friendly host (e.g. a GitHub raw `.csv` or a
|
||||
vega-datasets URL) — Save shows "Fetching…", then the dataset appears profiled
|
||||
(rows, columns, size) with a "Fetched <time>" line and the source address.
|
||||
- [ ] Go offline (DevTools → Network → Offline) and reload — a snippet that references
|
||||
that URL dataset still renders from the local snapshot (no fetch at render time).
|
||||
- [ ] Add a URL that blocks cross-origin requests (or while offline) — the form shows a
|
||||
cause-specific error and a "Paste data inline instead" button that switches to an
|
||||
inline paste keeping the name/comment; no broken record is saved.
|
||||
- [ ] Refresh a URL dataset whose source changed — rows/size and the "Fetched" time
|
||||
update; refreshing while offline raises an error toast and leaves the snapshot intact.
|
||||
- [ ] Build Chart from a fetched URL dataset works (columns are known); from an
|
||||
unfetched URL reference the builder has no schema until Refresh.
|
||||
|
||||
## Visual sweep of the M6 surfaces
|
||||
|
||||
- [ ] Library search / sort / empty states, the storage monitor, About & Donate
|
||||
|
||||
@@ -16,8 +16,8 @@ When a spec uses inline data, the preview renders it directly. When a spec inste
|
||||
|
||||
- A spec may point at a dataset from the library by name instead of embedding the data inline.
|
||||
- Before rendering, the preview substitutes the referenced dataset's stored contents into the spec.
|
||||
- URL-sourced datasets are fetched as needed at render time.
|
||||
- If a referenced dataset cannot be found or fetched, the preview shows a readable error (see _Error Display_) rather than a broken chart.
|
||||
- URL-sourced datasets render from their **local snapshot** (fetched when the dataset was created or last refreshed; see _Datasets_), so the preview does not fetch at render time and works offline. A URL dataset that has never been fetched has no snapshot and instead renders against its **live URL** as a fallback, until it is refreshed.
|
||||
- If a referenced dataset cannot be found, the preview shows a readable error (see _Error Display_) rather than a broken chart.
|
||||
|
||||
## Fit / Sizing Modes
|
||||
|
||||
@@ -42,12 +42,13 @@ Before the chart is drawn, the spec shown in the editor is transformed into the
|
||||
|
||||
**1. Dataset reference resolution.** Any named-data reference (`data` with a `name`) is replaced in-place with the referenced dataset's actual contents, shaped by the dataset's source and format (see _Datasets_):
|
||||
|
||||
| Dataset source / format | The reference's `data` becomes |
|
||||
| ----------------------- | ---------------------------------------------------------------- |
|
||||
| URL (any format) | a URL reference to the dataset's address, tagged with its format |
|
||||
| Inline JSON | the parsed values, inlined |
|
||||
| Inline CSV / TSV | the raw text, inlined, tagged with its format (CSV or TSV) |
|
||||
| Inline TopoJSON | the value, inlined, tagged as TopoJSON |
|
||||
| Dataset source / format | The reference's `data` becomes |
|
||||
| ------------------------- | -------------------------------------------------------------------------------- |
|
||||
| URL, fetched (any format) | its snapshot, inlined and tagged exactly like the inline rows below |
|
||||
| URL, not yet fetched | a live URL reference to the dataset's address, tagged with its format (fallback) |
|
||||
| Inline JSON | the parsed values, inlined |
|
||||
| Inline CSV / TSV | the raw text, inlined, tagged with its format (CSV or TSV) |
|
||||
| Inline TopoJSON | the value, inlined, tagged as TopoJSON |
|
||||
|
||||
- Resolution recurses into nested sub-specs (layered and concatenated specs, and a parent spec's child `spec`), so references anywhere in the spec are resolved.
|
||||
- If a referenced dataset does not exist, rendering fails with a "dataset not found" error (see _Error Display_).
|
||||
|
||||
+12
-10
@@ -28,7 +28,7 @@ A two-pane modal:
|
||||
Each list item shows:
|
||||
|
||||
- The dataset **name**.
|
||||
- A **meta line** combining: source ("URL" prefix for URL datasets), row count when known, the **format label** (JSON / CSV / TSV / TOPOJSON), and **size** (human-readable, e.g. B / KB / MB). For URL datasets where counts are not yet known, only the source and format label are shown.
|
||||
- A **meta line** combining: source ("URL" prefix for URL datasets), row count when known, the **format label** (JSON / CSV / TSV / TOPOJSON), and **size** (human-readable, e.g. B / KB / MB). A URL dataset that has not yet been fetched shows "not fetched" in place of the figures it does not have.
|
||||
- A **usage badge** when one or more snippets reference the dataset, indicating how many.
|
||||
|
||||
Clicking an item selects it and shows its detail. Per-item actions (delete, plus copy-reference and build-chart) live in the detail pane for the selected dataset.
|
||||
@@ -38,9 +38,9 @@ Clicking an item selects it and shows its detail. Per-item actions (delete, plus
|
||||
A dataset has one of two source types, chosen when creating it:
|
||||
|
||||
- **Inline** — the data itself is pasted in and stored directly in the library.
|
||||
- **URL** — the dataset stores a remote URL (http/https). The data is not copied locally; it is fetched on demand when a referencing spec is rendered (see _Live Preview_).
|
||||
- **URL** — the dataset is fetched once from a remote http/https address when it is created, and the fetched data is **snapshotted** into the library, along with the address (kept so the snapshot can be re-fetched). A URL dataset then works offline and renders from its local copy; it is never re-fetched at render time, only on demand (see _Refresh_, below).
|
||||
|
||||
For inline datasets the library holds the full data and can profile it. For URL datasets the library holds only the link, so row/column/size figures are typically not computed up front and show as "N/A".
|
||||
Either way the library holds the full data and profiles it. A URL dataset additionally records the address it was fetched from and when it was last fetched. Because the fetch happens in the browser, an address the browser cannot reach — offline, or one that blocks cross-origin requests — cannot be snapshotted; see _Actions → New / Create New_ for how that failure is handled.
|
||||
|
||||
## Supported Formats
|
||||
|
||||
@@ -59,18 +59,18 @@ When the user pastes inline data, the app auto-detects the format and reports a
|
||||
- Otherwise, multi-line text with a header row is detected as TSV (when tab-separated) or CSV (when comma-separated) — medium confidence.
|
||||
- Unrecognized input yields no format (low confidence); saving is blocked with a message asking the user to check the input.
|
||||
|
||||
The detected format and source are shown as badges in the create form so the user can confirm or override the source (Inline/URL) before saving. For URL datasets the format is inferred from the URL's file extension (`.csv`, `.tsv`, `.json`, `.topojson`) and shown as a hint.
|
||||
The detected format and source are shown as badges in the create form so the user can confirm or override the source (Inline/URL) before saving. For URL datasets the create form shows a format hint inferred from the URL's file extension (`.csv`, `.tsv`, `.json`, `.topojson`); the dataset's actual format is determined from the **fetched content** when it is created, falling back to the extension when the content is ambiguous.
|
||||
|
||||
## Profiling
|
||||
|
||||
For tabular inline data (JSON array-of-objects, CSV, TSV) the app computes and stores a profile:
|
||||
For tabular data (JSON array-of-objects, CSV, TSV) — whether pasted inline or fetched from a URL — the app computes and stores a profile:
|
||||
|
||||
- **Row count** and **column count**.
|
||||
- The list of **column names**.
|
||||
- An **inferred type per column**: number, text/string, date, or boolean. Type inference looks at the column's values: all-numeric becomes number, all `true`/`false` becomes boolean, otherwise string; empty cells are ignored.
|
||||
- **Size** in bytes of the stored data.
|
||||
|
||||
A **truncated data preview** of the raw data is also retained for display. URL datasets and non-tabular data are not profiled (counts show "N/A").
|
||||
A **truncated data preview** of the raw data is also retained for display. A fetched URL dataset is profiled exactly like inline data; only non-tabular data and a URL dataset that has **not yet been fetched** are unprofiled (counts show "N/A").
|
||||
|
||||
## Detail Panel
|
||||
|
||||
@@ -78,8 +78,9 @@ The detail pane for a selected dataset shows:
|
||||
|
||||
- **Name**.
|
||||
- **Comment** (optional free-text notes), when present.
|
||||
- **Source** (URL datasets only): the address the snapshot was fetched from, and when it was last fetched (or "Not fetched yet"), alongside the **Refresh** action.
|
||||
- **Overview**: statistics (rows, columns, size), the **column list** with each column's name and inferred type shown with a simple type indicator, and created/modified timestamps.
|
||||
- **Preview**: a truncated rendering of the data (raw text for CSV/TSV/URL, pretty-printed for JSON/TopoJSON).
|
||||
- **Preview**: a sample of the data. Tabular datasets (CSV, TSV, or a JSON array-of-objects — inline or fetched) render as a **table** of the first rows under the profiled column names, with a note when more rows exist than are shown. Non-tabular payloads (a single JSON object, TopoJSON) render as pretty-printed JSON, and a URL dataset that has not yet been fetched shows a short placeholder.
|
||||
- **Linked Snippets**: the list of snippets that reference this dataset by name. This is the dataset side of bidirectional dataset↔snippet linking (see _Snippet Library_).
|
||||
|
||||
## Actions
|
||||
@@ -89,13 +90,14 @@ A destructive or off-screen outcome raises a confirming toast; an action whose r
|
||||
- **Copy Reference** — copies the by-name reference object to the clipboard, ready to paste into a spec:
|
||||
`{ "data": { "name": "MyDataset" } }`
|
||||
The clipboard write is invisible, so it is confirmed _inline on the control_ ("Copied"), announced politely to assistive technology — not a toast.
|
||||
- **New / Create New** — opens the create form in the detail pane with fields: **name** (required, unique), **source** toggle (Inline / URL), the **data** (a paste area for inline, a URL field for URL source), and an optional **comment**. Save is disabled until a name and valid data/URL are present. On success the new dataset is shown selected in the detail pane — that visible result is the confirmation, so no toast is raised (a dataset created _off-screen_ via Extract does toast; see _Spec Editor_).
|
||||
- **Edit** — rename, edit the comment, and update the data (re-paste inline data or refresh the URL). Updating inline data re-profiles it; the modified timestamp advances.
|
||||
- **New / Create New** — opens the create form in the detail pane with fields: **name** (required, unique), **source** toggle (Inline / URL), the **data** (a paste area for inline, a URL field for URL source), and an optional **comment**. Save is disabled until a name and valid data/URL are present. For a **URL** dataset, saving fetches and snapshots the address; the Save control shows a busy state while fetching. If the fetch fails — offline, blocked by the host (cross-origin), not found, empty, or timed out — the form shows a readable, cause-specific error and offers a one-click **"Paste data inline instead"** that switches the form to an inline paste (keeping the name and comment), rather than saving a broken record. On success the new dataset is shown selected in the detail pane — that visible result is the confirmation, so no toast is raised (a dataset created _off-screen_ via Extract does toast; see _Spec Editor_).
|
||||
- **Refresh** (URL datasets) — re-fetches the dataset's address and re-snapshots it, re-profiling the data and advancing the last-fetched time. The updated figures are the visible confirmation, so success raises no toast; a failed refresh raises an error toast. Refresh is also how a URL dataset that has not yet been fetched (e.g. one migrated from an older version) acquires its snapshot.
|
||||
- **Edit** — rename, edit the comment, change the source data, and (for URL datasets) change the address. Updating inline data re-profiles it; changing a URL dataset's address re-fetches and re-snapshots it; editing only a URL dataset's name or comment does **not** re-fetch. The modified timestamp advances.
|
||||
- **Delete** — asks for confirmation ("Delete \"Name\"? This cannot be undone."), then removes the dataset and clears the selection.
|
||||
|
||||
## Build Chart From Dataset
|
||||
|
||||
From a selected dataset the user can launch the visual _Chart Builder_ (see _Chart Builder_) pre-targeted at that dataset, producing a new snippet whose spec references the dataset by name.
|
||||
From a selected dataset the user can launch the visual _Chart Builder_ (see _Chart Builder_) pre-targeted at that dataset, producing a new snippet whose spec references the dataset by name. This relies on the dataset's profiled columns, so a URL dataset must have been fetched first — an unfetched URL reference has no schema to map (use _Refresh_ to fetch it).
|
||||
|
||||
## Extract Inline Data → Dataset
|
||||
|
||||
|
||||
+21
-17
@@ -34,29 +34,33 @@ A snippet carries two specs at once. `draftSpec` is the editable working copy; `
|
||||
|
||||
A **Dataset** is a named, reusable data source that snippets can reference by name instead of inlining data. Datasets are managed in the _Datasets_ manager and support multiple formats and two source kinds.
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| ------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `id` | number | Unique numeric identifier. |
|
||||
| `version` | number | Schema version of this record, used for read-time migration (see _Schema versioning_ below). |
|
||||
| `name` | string | Unique, human-readable name; the key snippets reference via `datasetRefs`. |
|
||||
| `data` | JSON value | The payload. For `source = url`: the URL string. For `source = inline`: the raw CSV/TSV text, or the parsed JSON/TopoJSON value. |
|
||||
| `format` | string | One of `json`, `csv`, `tsv`, `topojson`. |
|
||||
| `source` | string | One of `inline` (data embedded in the record) or `url` (data fetched from a remote address). |
|
||||
| `comment` | string | Free-form user note about the dataset. |
|
||||
| `rowCount` | number or null | Number of data rows, or null when unknown/not applicable. |
|
||||
| `columnCount` | number or null | Number of columns, or null when unknown/not applicable. |
|
||||
| `columns` | string[] | Column names, in order. |
|
||||
| `columnTypes` | array of `{ name, type }` | Per-column inferred type. `name` is the column; `type` is one of `number`, `string`, `date`, `boolean`. |
|
||||
| `size` | number | Approximate payload size in bytes. |
|
||||
| `created` | ISO-timestamp string | When the dataset was first added. |
|
||||
| `modified` | ISO-timestamp string | When the dataset was last changed. |
|
||||
| Field | Type | Meaning |
|
||||
| ------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `id` | number | Unique numeric identifier. |
|
||||
| `version` | number | Schema version of this record, used for read-time migration (see _Schema versioning_ below). |
|
||||
| `name` | string | Unique, human-readable name; the key snippets reference via `datasetRefs`. |
|
||||
| `data` | JSON value | The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON value. For `source = url` this is the **fetched snapshot**, or `null` before the first successful fetch. |
|
||||
| `format` | string | One of `json`, `csv`, `tsv`, `topojson`. |
|
||||
| `source` | string | One of `inline` (data pasted into the record) or `url` (data fetched once from a remote address and snapshotted into the record). |
|
||||
| `url` | string (url only) | The remote address a `url` dataset was fetched from, retained so it can be re-fetched ("Refresh"). Absent for inline datasets. |
|
||||
| `fetchedAt` | ISO-timestamp or null | For `url` datasets: when the snapshot was last fetched, or `null` if never fetched. Absent for inline datasets. |
|
||||
| `comment` | string | Free-form user note about the dataset. |
|
||||
| `rowCount` | number or null | Number of data rows, or null when unknown/not applicable. |
|
||||
| `columnCount` | number or null | Number of columns, or null when unknown/not applicable. |
|
||||
| `columns` | string[] | Column names, in order. |
|
||||
| `columnTypes` | array of `{ name, type }` | Per-column inferred type. `name` is the column; `type` is one of `number`, `string`, `date`, `boolean`. |
|
||||
| `size` | number | Approximate payload size in bytes. |
|
||||
| `created` | ISO-timestamp string | When the dataset was first added. |
|
||||
| `modified` | ISO-timestamp string | When the dataset was last changed. |
|
||||
|
||||
The `rowCount`, `columnCount`, `columns`, `columnTypes`, and `size` fields are derived summaries computed when data is added or updated; they support previews and type display without re-parsing the full payload.
|
||||
The `rowCount`, `columnCount`, `columns`, `columnTypes`, and `size` fields are derived summaries computed when data is added or updated — including when a `url` dataset is fetched or refreshed; a fetched URL snapshot profiles exactly like inline data. They support previews and type display without re-parsing the full payload.
|
||||
|
||||
### Schema versioning
|
||||
|
||||
Both **Snippet** and **Dataset** records carry a numeric `version` recording the shape of that individual record. When a record is read from storage it is migrated up to the current shape before the app uses it; new writes always store the current version. A record written before versioning existed (no `version` field) is treated as version `1`. This is distinct from the storage container's own layout version, and from the _Import & Export_ envelope `version` (which describes the file format, not a record). Records exported via _Import & Export_ include their `version`.
|
||||
|
||||
The current **Dataset** version is `2`. The v1→v2 migration reflects the URL-snapshot model: a v1 `url` dataset stored its address in `data`, so migration moves that address into the new `url` field and clears `data` to `null` — the record becomes an _unfetched reference_ that renders against its live URL until the user refreshes it, at which point the fetched snapshot is stored and profiled.
|
||||
|
||||
## C. UserSettings
|
||||
|
||||
**UserSettings** holds persisted user preferences as a single structured record. The semantics and UX of each option are covered in _Settings_; the shape below is the storage contract.
|
||||
|
||||
@@ -235,6 +235,32 @@
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
/* URL dataset provenance: the source address + last-fetched time (spec §05). */
|
||||
.sourceMeta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: baseline;
|
||||
gap: var(--space-2) var(--space-4);
|
||||
margin: 0;
|
||||
font-size: 12px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.sourceLink {
|
||||
color: var(--accent);
|
||||
text-decoration: none;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.sourceLink:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.sourceLink:focus-visible {
|
||||
outline: 2px solid var(--focus);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.section {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -335,6 +361,46 @@
|
||||
white-space: pre;
|
||||
}
|
||||
|
||||
/* Tabular preview — a scrollable grid with a sticky header (spec §05 → Preview). */
|
||||
.previewTableWrap {
|
||||
max-height: 260px;
|
||||
overflow: auto;
|
||||
border: var(--border-width) solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
|
||||
.previewTable {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
font-family: var(--font-mono);
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.previewTable th,
|
||||
.previewTable td {
|
||||
text-align: left;
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border-bottom: var(--border-width) solid var(--border);
|
||||
white-space: nowrap;
|
||||
max-width: 240px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.previewTable thead th {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1;
|
||||
background: var(--layer-01);
|
||||
font-weight: 600;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.previewTable tbody tr:last-child td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.muted {
|
||||
margin: 0;
|
||||
font-size: 13px;
|
||||
@@ -439,3 +505,10 @@
|
||||
font-size: 13px;
|
||||
color: var(--support-error);
|
||||
}
|
||||
|
||||
/* The inline-fallback recovery under a failed URL fetch — left-aligned, not
|
||||
stretched, so it reads as a secondary recovery beneath the error message. */
|
||||
.fallback {
|
||||
align-self: flex-start;
|
||||
margin-top: calc(-1 * var(--space-2));
|
||||
}
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
* Snippets stay reactive without a stored back-pointer.
|
||||
*/
|
||||
|
||||
import { useState } from 'react';
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useShallow } from 'zustand/react/shallow';
|
||||
import { datasetReference, type DataSource, type Dataset } from '@core/dataset';
|
||||
import { datasetReference, tabularRows, type DataSource, type Dataset } from '@core/dataset';
|
||||
import { detectFormat, detectFormatFromUrl, type DataFormat } from '@core/format-detection';
|
||||
import { datasetUsageCounts, snippetsReferencingDataset } from '@core/relationships';
|
||||
import { closeModal, openModal, resnapshot } from '../modals/ModalCoordinator';
|
||||
import { fetchRemoteData } from '../infrastructure/remote-data';
|
||||
import { remoteFetchErrorMessage } from '../services/remote-data-errors';
|
||||
import { confirm } from '../stores/ConfirmStore';
|
||||
import { notify } from '../stores/NotificationStore';
|
||||
import {
|
||||
@@ -50,6 +52,17 @@ const SOURCE_OPTIONS: ReadonlyArray<SegmentedOption<DataSource>> = [
|
||||
{ value: 'url', label: 'URL' },
|
||||
];
|
||||
|
||||
/** Rows shown in the tabular preview before truncating (spec §05 → Detail Panel). */
|
||||
const PREVIEW_ROW_LIMIT = 50;
|
||||
|
||||
/** One table cell's text: blank for empty, the string as-is, else JSON (numbers,
|
||||
* booleans, nested values). Avoids `String()` on objects ("[object Object]"). */
|
||||
function cellText(value: unknown): string {
|
||||
if (value == null) return '';
|
||||
if (typeof value === 'string') return value;
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function DatasetsModal() {
|
||||
const datasets = useDatasetStore(useShallow((s) => s.datasets));
|
||||
const view = useDatasetStore((s) => s.view);
|
||||
@@ -116,11 +129,15 @@ function DatasetListItem({
|
||||
onSelect: () => void;
|
||||
}) {
|
||||
// Meta line: source ("URL" prefix), row count when known, format label, size.
|
||||
// A fetched URL snapshot reads like an inline dataset (rows + size); an unfetched
|
||||
// URL reference shows "not fetched" in place of figures it doesn't have yet.
|
||||
const unfetched = dataset.source === 'url' && dataset.data == null;
|
||||
const parts: string[] = [];
|
||||
if (dataset.source === 'url') parts.push('URL');
|
||||
if (dataset.rowCount !== null) parts.push(`${dataset.rowCount} rows`);
|
||||
if (unfetched) parts.push('not fetched');
|
||||
else if (dataset.rowCount !== null) parts.push(`${dataset.rowCount} rows`);
|
||||
parts.push(formatLabel(dataset.format));
|
||||
if (dataset.source !== 'url') parts.push(humanBytes(dataset.size));
|
||||
if (!unfetched) parts.push(humanBytes(dataset.size));
|
||||
|
||||
return (
|
||||
<li className={`${styles.item} ${active ? styles.itemActive : ''}`}>
|
||||
@@ -151,16 +168,46 @@ function DatasetDetail({
|
||||
}) {
|
||||
const startEdit = useDatasetStore((s) => s.startEdit);
|
||||
const remove = useDatasetStore((s) => s.remove);
|
||||
const refreshDataset = useDatasetStore((s) => s.refreshDataset);
|
||||
const selectSnippet = useSnippetStore((s) => s.selectSnippet);
|
||||
const [copied, setCopied] = useState(false);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
|
||||
const linked = snippetsReferencingDataset(snippets, dataset.name);
|
||||
|
||||
// Tabular data (CSV/TSV/JSON-array, inline or fetched) previews as a table of the
|
||||
// first rows under the profiled columns; non-tabular payloads fall back to text.
|
||||
// Memoized on the record so a large CSV isn't re-parsed on unrelated re-renders.
|
||||
const previewRows = useMemo(
|
||||
() => tabularRows(dataset.data, dataset.format, PREVIEW_ROW_LIMIT),
|
||||
[dataset],
|
||||
);
|
||||
|
||||
const handleEdit = () => {
|
||||
startEdit();
|
||||
resnapshot();
|
||||
};
|
||||
|
||||
// Re-fetch a URL dataset's source and re-snapshot it. The visible result (updated
|
||||
// rows + "Fetched" time) is the confirmation, so success raises no toast; only a
|
||||
// failure surfaces one (spec §05 → Actions; docs/architecture/10 → Toast copy).
|
||||
const handleRefresh = async () => {
|
||||
if (!dataset.url) return;
|
||||
setRefreshing(true);
|
||||
try {
|
||||
const { text } = await fetchRemoteData(dataset.url);
|
||||
refreshDataset(dataset.id, { text });
|
||||
} catch (err) {
|
||||
notify({
|
||||
kind: 'error',
|
||||
title: "Couldn't refresh dataset",
|
||||
message: remoteFetchErrorMessage(err, 'retry'),
|
||||
});
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleCopy = async () => {
|
||||
const text = JSON.stringify(datasetReference(dataset.name), null, 2);
|
||||
try {
|
||||
@@ -212,6 +259,16 @@ function DatasetDetail({
|
||||
<span role="status" className="visually-hidden">
|
||||
{copied ? 'Reference copied to clipboard' : ''}
|
||||
</span>
|
||||
{dataset.source === 'url' && (
|
||||
<button
|
||||
type="button"
|
||||
className={styles.action}
|
||||
onClick={() => void handleRefresh()}
|
||||
disabled={refreshing}
|
||||
>
|
||||
{refreshing ? 'Refreshing…' : 'Refresh'}
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className={styles.action} onClick={handleEdit}>
|
||||
Edit
|
||||
</button>
|
||||
@@ -237,6 +294,19 @@ function DatasetDetail({
|
||||
|
||||
{dataset.comment && <p className={styles.comment}>{dataset.comment}</p>}
|
||||
|
||||
{dataset.source === 'url' && (
|
||||
<p className={styles.sourceMeta}>
|
||||
<a className={styles.sourceLink} href={dataset.url} target="_blank" rel="noreferrer">
|
||||
{dataset.url}
|
||||
</a>
|
||||
<span>
|
||||
{dataset.fetchedAt
|
||||
? `Fetched ${new Date(dataset.fetchedAt).toLocaleString()}`
|
||||
: 'Not fetched yet'}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<section className={styles.section}>
|
||||
<h4 className={styles.sectionTitle}>Overview</h4>
|
||||
<dl className={styles.stats}>
|
||||
@@ -254,7 +324,7 @@ function DatasetDetail({
|
||||
</div>
|
||||
<div>
|
||||
<dt>Size</dt>
|
||||
<dd>{dataset.source === 'url' ? 'N/A' : humanBytes(dataset.size)}</dd>
|
||||
<dd>{dataset.data == null ? 'N/A' : humanBytes(dataset.size)}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
|
||||
@@ -277,7 +347,43 @@ function DatasetDetail({
|
||||
|
||||
<section className={styles.section}>
|
||||
<h4 className={styles.sectionTitle}>Preview</h4>
|
||||
<pre className={styles.preview}>{previewText(dataset)}</pre>
|
||||
{previewRows ? (
|
||||
<>
|
||||
{/* TODO: a11y — this scroll container isn't keyboard-reachable; when the
|
||||
preview overflows, keyboard-only users can't scroll it (WCAG 2.1.1).
|
||||
A council pass should decide the pattern (tabindex=0 + aria-label region,
|
||||
or a different overflow treatment) before we lean on it more. */}
|
||||
<div className={styles.previewTableWrap}>
|
||||
<table className={styles.previewTable}>
|
||||
<thead>
|
||||
<tr>
|
||||
{dataset.columns.map((col, ci) => (
|
||||
<th key={ci} scope="col">
|
||||
{col}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{previewRows.map((row, ri) => (
|
||||
<tr key={ri}>
|
||||
{dataset.columns.map((col, ci) => (
|
||||
<td key={ci}>{cellText(row[col])}</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{dataset.rowCount != null && dataset.rowCount > previewRows.length && (
|
||||
<p className={styles.muted}>
|
||||
Showing the first {previewRows.length} of {dataset.rowCount} rows.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<pre className={styles.preview}>{previewText(dataset)}</pre>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className={styles.section}>
|
||||
@@ -307,19 +413,22 @@ function DatasetDetail({
|
||||
);
|
||||
}
|
||||
|
||||
/** A truncated rendering of the data: raw for csv/tsv/url, pretty JSON otherwise. */
|
||||
/** A truncated rendering of the data: raw for csv/tsv, pretty JSON otherwise. */
|
||||
function previewText(dataset: Dataset): string {
|
||||
const MAX = 2000;
|
||||
// No snapshot yet (an unfetched URL reference) — there is nothing to preview.
|
||||
if (dataset.data == null) {
|
||||
return dataset.source === 'url' ? 'Not fetched yet — use Refresh to load the data.' : '';
|
||||
}
|
||||
let text: string;
|
||||
if (dataset.source === 'url') {
|
||||
text = String(dataset.data);
|
||||
} else if (dataset.format === 'csv' || dataset.format === 'tsv') {
|
||||
text = String(dataset.data);
|
||||
if (dataset.format === 'csv' || dataset.format === 'tsv') {
|
||||
// CSV/TSV payloads are raw text; fall back to JSON for any non-string value.
|
||||
text = typeof dataset.data === 'string' ? dataset.data : JSON.stringify(dataset.data);
|
||||
} else {
|
||||
try {
|
||||
text = JSON.stringify(dataset.data, null, 2);
|
||||
} catch {
|
||||
text = String(dataset.data);
|
||||
text = typeof dataset.data === 'string' ? dataset.data : '[unserializable data]';
|
||||
}
|
||||
}
|
||||
return text.length > MAX ? `${text.slice(0, MAX)}\n…` : text;
|
||||
@@ -328,20 +437,59 @@ function previewText(dataset: Dataset): string {
|
||||
function DatasetFormView({ editing }: { editing: boolean }) {
|
||||
const form = useDatasetStore((s) => s.form);
|
||||
const formError = useDatasetStore((s) => s.formError);
|
||||
const selected = useDatasetStore(selectSelectedDataset);
|
||||
const updateForm = useDatasetStore((s) => s.updateForm);
|
||||
const cancelForm = useDatasetStore((s) => s.cancelForm);
|
||||
const save = useDatasetStore((s) => s.save);
|
||||
const commitUrlSnapshot = useDatasetStore((s) => s.commitUrlSnapshot);
|
||||
// Save stays disabled until a name and valid data/URL are present (spec §05).
|
||||
const canSave = useDatasetStore(selectCanSave);
|
||||
|
||||
// URL datasets fetch on save (snapshot model): `fetching` drives the button's
|
||||
// busy state; `fetchError` holds a failed fetch's message + the inline fallback.
|
||||
const [fetching, setFetching] = useState(false);
|
||||
const [fetchError, setFetchError] = useState<string | null>(null);
|
||||
|
||||
// Live format/source hint from the current input (spec §05 → Auto-detection).
|
||||
const detected =
|
||||
form.source === 'url'
|
||||
? { format: detectFormatFromUrl(form.input.trim()), confidence: 'url' as const }
|
||||
: detectFormat(form.input);
|
||||
|
||||
const handleSave = () => {
|
||||
if (save()) resnapshot(); // committed — re-baseline so a later close won't prompt
|
||||
const handleSave = async () => {
|
||||
// Inline saves are synchronous; only URL datasets touch the network.
|
||||
if (form.source !== 'url') {
|
||||
if (save()) resnapshot(); // committed — re-baseline so a later close won't prompt
|
||||
return;
|
||||
}
|
||||
const url = form.input.trim();
|
||||
// A metadata-only edit (same URL, snapshot already present) needs no re-fetch.
|
||||
const metaOnly =
|
||||
editing &&
|
||||
selected?.source === 'url' &&
|
||||
selected.data != null &&
|
||||
url === (selected.url ?? '');
|
||||
if (metaOnly) {
|
||||
if (save()) resnapshot();
|
||||
return;
|
||||
}
|
||||
// Create, changed URL, or inline→URL: fetch once, then snapshot + commit.
|
||||
setFetchError(null);
|
||||
setFetching(true);
|
||||
try {
|
||||
const { text } = await fetchRemoteData(url);
|
||||
if (commitUrlSnapshot({ text })) resnapshot();
|
||||
} catch (err) {
|
||||
setFetchError(remoteFetchErrorMessage(err));
|
||||
} finally {
|
||||
setFetching(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Recovery from a failed fetch: switch to an inline paste, keeping name + comment.
|
||||
const handlePasteInline = () => {
|
||||
setFetchError(null);
|
||||
updateForm({ source: 'inline', input: '' });
|
||||
};
|
||||
|
||||
const handleCancel = () => {
|
||||
@@ -370,7 +518,10 @@ function DatasetFormView({ editing }: { editing: boolean }) {
|
||||
label="Dataset source"
|
||||
options={SOURCE_OPTIONS}
|
||||
value={form.source}
|
||||
onChange={(source) => updateForm({ source })}
|
||||
onChange={(source) => {
|
||||
setFetchError(null);
|
||||
updateForm({ source });
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -381,7 +532,10 @@ function DatasetFormView({ editing }: { editing: boolean }) {
|
||||
type="url"
|
||||
className={styles.input}
|
||||
value={form.input}
|
||||
onChange={(e) => updateForm({ input: e.target.value })}
|
||||
onChange={(e) => {
|
||||
setFetchError(null);
|
||||
updateForm({ input: e.target.value });
|
||||
}}
|
||||
placeholder="https://example.com/data.csv"
|
||||
/>
|
||||
) : (
|
||||
@@ -418,23 +572,34 @@ function DatasetFormView({ editing }: { editing: boolean }) {
|
||||
/>
|
||||
</label>
|
||||
|
||||
{formError && (
|
||||
{(formError || fetchError) && (
|
||||
<p className={styles.formError} role="alert">
|
||||
{formError}
|
||||
{formError ?? fetchError}
|
||||
</p>
|
||||
)}
|
||||
{/* A failed fetch is recoverable by pasting the data inline (the user's choice
|
||||
when CORS or offline blocks the URL) — offered as a direct one-click path. */}
|
||||
{fetchError && (
|
||||
<button
|
||||
type="button"
|
||||
className={`${styles.action} ${styles.fallback}`}
|
||||
onClick={handlePasteInline}
|
||||
>
|
||||
Paste data inline instead
|
||||
</button>
|
||||
)}
|
||||
|
||||
<div className={styles.formActions}>
|
||||
<button type="button" className={styles.action} onClick={handleCancel}>
|
||||
<button type="button" className={styles.action} onClick={handleCancel} disabled={fetching}>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`${styles.action} ${styles.primary}`}
|
||||
disabled={!canSave}
|
||||
onClick={handleSave}
|
||||
disabled={!canSave || fetching}
|
||||
onClick={() => void handleSave()}
|
||||
>
|
||||
{editing ? 'Save changes' : 'Create dataset'}
|
||||
{fetching ? 'Fetching…' : editing ? 'Save changes' : 'Create dataset'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { CURRENT_DATASET_VERSION } from '@core/dataset';
|
||||
import { migrateDataset } from './dataset-migrations';
|
||||
|
||||
describe('migrateDataset', () => {
|
||||
test('stamps the current version and fills missing fields with defaults', () => {
|
||||
const d = migrateDataset({ id: 7, data: 'a,b\n1,2', format: 'csv', source: 'inline' });
|
||||
|
||||
expect(d.id).toBe(7);
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
expect(d.name).toBe('Untitled');
|
||||
expect(d.comment).toBe('');
|
||||
expect(d.columns).toEqual([]);
|
||||
expect(d.columnStats).toEqual([]);
|
||||
});
|
||||
|
||||
test('an inline record carries no url/fetchedAt keys', () => {
|
||||
const d = migrateDataset({ id: 1, data: [{ a: 1 }], format: 'json', source: 'inline' });
|
||||
expect(d.source).toBe('inline');
|
||||
expect('url' in d).toBe(false);
|
||||
expect('fetchedAt' in d).toBe(false);
|
||||
});
|
||||
|
||||
test('v1→v2: a legacy URL record moves its address out of data into url', () => {
|
||||
// Pre-v2 shape: the URL lived in `data`, there was no `url`/`fetchedAt`, and the
|
||||
// profile was N/A. It becomes an unfetched reference: url set, data cleared.
|
||||
const d = migrateDataset({
|
||||
id: 2,
|
||||
data: 'https://example.com/data.csv',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
rowCount: null,
|
||||
columns: [],
|
||||
});
|
||||
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
expect(d.source).toBe('url');
|
||||
expect(d.url).toBe('https://example.com/data.csv');
|
||||
expect(d.data).toBeNull();
|
||||
expect(d.fetchedAt).toBeNull();
|
||||
expect(d.rowCount).toBeNull();
|
||||
});
|
||||
|
||||
test('a v2 URL snapshot is preserved (data, url, and fetchedAt all kept)', () => {
|
||||
const d = migrateDataset({
|
||||
id: 3,
|
||||
version: 2,
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://example.com/data.csv',
|
||||
fetchedAt: '2026-06-10T00:00:00.000Z',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
rowCount: 1,
|
||||
columns: ['a', 'b'],
|
||||
});
|
||||
|
||||
expect(d.data).toBe('a,b\n1,2');
|
||||
expect(d.url).toBe('https://example.com/data.csv');
|
||||
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
|
||||
expect(d.rowCount).toBe(1);
|
||||
});
|
||||
|
||||
test('preserves unknown fields so a newer build round-trips without loss', () => {
|
||||
const d = migrateDataset({
|
||||
id: 1,
|
||||
data: [],
|
||||
format: 'json',
|
||||
source: 'inline',
|
||||
futureField: 42,
|
||||
});
|
||||
expect((d as unknown as Record<string, unknown>).futureField).toBe(42);
|
||||
});
|
||||
});
|
||||
@@ -30,14 +30,33 @@ function asCount(v: unknown): number | null {
|
||||
/** Upgrade a raw stored record to the current Dataset shape. */
|
||||
export function migrateDataset(raw: unknown): Dataset {
|
||||
const r = { ...(raw as Record<string, unknown>) };
|
||||
const source = asSource(r.source);
|
||||
|
||||
// v1→v2: a URL dataset used to store its address in `data`; it now stores the
|
||||
// fetched snapshot in `data` and keeps the address in `url` (+ a `fetchedAt`).
|
||||
// A legacy record (url source, no `url` field) becomes an *unfetched* reference:
|
||||
// move the address to `url` and clear `data`. The N/A profile carries over as-is;
|
||||
// rendering falls back to the live URL until a Refresh fetches + profiles it.
|
||||
const legacyUrl = source === 'url' && typeof r.url !== 'string';
|
||||
const url = legacyUrl
|
||||
? typeof r.data === 'string'
|
||||
? r.data
|
||||
: undefined
|
||||
: typeof r.url === 'string'
|
||||
? r.url
|
||||
: undefined;
|
||||
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
|
||||
|
||||
return {
|
||||
...r,
|
||||
id: typeof r.id === 'number' ? r.id : Number(r.id),
|
||||
version: CURRENT_DATASET_VERSION,
|
||||
name: typeof r.name === 'string' ? r.name : 'Untitled',
|
||||
data: r.data,
|
||||
data: legacyUrl ? null : r.data,
|
||||
format: asFormat(r.format),
|
||||
source: asSource(r.source),
|
||||
source,
|
||||
// Only URL datasets carry url/fetchedAt; inline records stay clean.
|
||||
...(source === 'url' ? { url, fetchedAt } : {}),
|
||||
comment: typeof r.comment === 'string' ? r.comment : '',
|
||||
rowCount: asCount(r.rowCount),
|
||||
columnCount: asCount(r.columnCount),
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { afterEach, describe, expect, test, vi } from 'vitest';
|
||||
import { fetchRemoteData, RemoteFetchError } from './remote-data';
|
||||
|
||||
/** A minimal Response stand-in for the bits the adapter reads. */
|
||||
function fakeResponse(opts: {
|
||||
ok: boolean;
|
||||
status?: number;
|
||||
statusText?: string;
|
||||
body?: string;
|
||||
contentType?: string | null;
|
||||
}): Response {
|
||||
return {
|
||||
ok: opts.ok,
|
||||
status: opts.status ?? (opts.ok ? 200 : 500),
|
||||
statusText: opts.statusText ?? '',
|
||||
text: () => Promise.resolve(opts.body ?? ''),
|
||||
headers: {
|
||||
get: (k: string) => (k.toLowerCase() === 'content-type' ? (opts.contentType ?? null) : null),
|
||||
},
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe('fetchRemoteData', () => {
|
||||
test('returns the body and content-type on success', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() =>
|
||||
Promise.resolve(fakeResponse({ ok: true, body: 'a,b\n1,2', contentType: 'text/csv' })),
|
||||
),
|
||||
);
|
||||
const result = await fetchRemoteData('https://example.com/data.csv');
|
||||
expect(result.text).toBe('a,b\n1,2');
|
||||
expect(result.contentType).toBe('text/csv');
|
||||
});
|
||||
|
||||
test('classifies a non-2xx response as http with its status', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() =>
|
||||
Promise.resolve(fakeResponse({ ok: false, status: 404, statusText: 'Not Found' })),
|
||||
),
|
||||
);
|
||||
await expect(fetchRemoteData('https://example.com/missing.csv')).rejects.toMatchObject({
|
||||
reason: 'http',
|
||||
status: 404,
|
||||
});
|
||||
});
|
||||
|
||||
test('classifies a rejected fetch (CORS / offline) as network', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
|
||||
);
|
||||
await expect(fetchRemoteData('https://blocked.example/data.csv')).rejects.toMatchObject({
|
||||
reason: 'network',
|
||||
});
|
||||
});
|
||||
|
||||
test('classifies an empty body as empty', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.resolve(fakeResponse({ ok: true, body: ' \n' }))),
|
||||
);
|
||||
await expect(fetchRemoteData('https://example.com/blank.csv')).rejects.toMatchObject({
|
||||
reason: 'empty',
|
||||
});
|
||||
});
|
||||
|
||||
test('classifies an aborted fetch as timeout', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => {
|
||||
const e = new Error('aborted');
|
||||
e.name = 'AbortError';
|
||||
return Promise.reject(e);
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
fetchRemoteData('https://slow.example/data.csv', { timeoutMs: 1 }),
|
||||
).rejects.toMatchObject({
|
||||
reason: 'timeout',
|
||||
});
|
||||
});
|
||||
|
||||
test('the thrown error is a RemoteFetchError', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
|
||||
);
|
||||
await expect(fetchRemoteData('https://x.example')).rejects.toBeInstanceOf(RemoteFetchError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Remote-data fetch adapter (snapshot model — spec §05 → URL datasets).
|
||||
*
|
||||
* The ONLY place Astrolabe touches the network. Adding a URL dataset fetches the
|
||||
* resource once here; core then profiles the returned text exactly like inline
|
||||
* data (see dataset.ts). Keeping the fetch in infrastructure preserves the rule
|
||||
* that `src/core` stays pure and that the rest of the app never calls `fetch`
|
||||
* directly (AGENTS → Architecture).
|
||||
*
|
||||
* Failures are classified into a small set of machine-readable `reason`s and
|
||||
* thrown as `RemoteFetchError`; the user-facing wording (and the "paste inline
|
||||
* instead" fallback) lives in the UI layer, where it is council-reviewed — the
|
||||
* same split as `storage-errors`. A browser cross-origin block and an
|
||||
* offline/DNS failure are indistinguishable here — both reject with a `TypeError`
|
||||
* and no status — so they share the `network` reason and the UI hedges the cause.
|
||||
*/
|
||||
|
||||
/** Why a remote fetch failed, in terms the UI maps to copy + a recovery path. */
|
||||
export type RemoteFetchReason = 'network' | 'http' | 'empty' | 'timeout';
|
||||
|
||||
/** A classified remote-fetch failure. `status` is set only for `http`. */
|
||||
export class RemoteFetchError extends Error {
|
||||
readonly reason: RemoteFetchReason;
|
||||
readonly status?: number;
|
||||
|
||||
constructor(reason: RemoteFetchReason, message: string, status?: number) {
|
||||
super(message);
|
||||
this.name = 'RemoteFetchError';
|
||||
this.reason = reason;
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
/** A successful fetch: the raw body plus a weak format hint from the server. */
|
||||
export interface RemoteData {
|
||||
/** The response body, ready for format detection + profiling. */
|
||||
text: string;
|
||||
/** The server's `Content-Type`, when provided — a tertiary format hint. */
|
||||
contentType: string | null;
|
||||
}
|
||||
|
||||
/** Default ceiling on a single fetch before it is aborted as a timeout. */
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
export interface FetchRemoteDataOptions {
|
||||
/** Abort the fetch after this many ms (default 30 s). */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
function describeNetworkError(err: unknown): string {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
return `Couldn't reach the URL (${reason}).`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a remote dataset resource once and return its raw body. Throws
|
||||
* `RemoteFetchError` on any failure — the caller distinguishes recovery by
|
||||
* `reason`. The body is returned verbatim; format detection and profiling happen
|
||||
* downstream so this adapter stays free of core logic.
|
||||
*/
|
||||
export async function fetchRemoteData(
|
||||
url: string,
|
||||
options: FetchRemoteDataOptions = {},
|
||||
): Promise<RemoteData> {
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(url, { redirect: 'follow', signal: controller.signal });
|
||||
} catch (err) {
|
||||
// A timeout aborts via the controller (AbortError); everything else here is a
|
||||
// CORS block, offline, DNS failure, or malformed URL — all opaque TypeErrors.
|
||||
if ((err as { name?: string }).name === 'AbortError') {
|
||||
throw new RemoteFetchError('timeout', `The URL took longer than ${timeoutMs} ms to respond.`);
|
||||
}
|
||||
throw new RemoteFetchError('network', describeNetworkError(err));
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new RemoteFetchError(
|
||||
'http',
|
||||
`The server responded ${response.status} ${response.statusText}.`.trim(),
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
if (text.trim() === '') {
|
||||
throw new RemoteFetchError('empty', 'The URL returned no data.');
|
||||
}
|
||||
|
||||
// TODO: guard very large responses (cap or warn) before snapshotting into
|
||||
// IndexedDB — a multi-hundred-MB file would load fully into memory + storage.
|
||||
return { text, contentType: response.headers.get('content-type') };
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { RemoteFetchError } from '../infrastructure/remote-data';
|
||||
import { remoteFetchErrorMessage } from './remote-data-errors';
|
||||
|
||||
describe('remoteFetchErrorMessage', () => {
|
||||
test('a network failure hedges offline vs cross-origin and offers the inline fallback', () => {
|
||||
const msg = remoteFetchErrorMessage(new RemoteFetchError('network', 'x'));
|
||||
expect(msg).toMatch(/offline/i);
|
||||
expect(msg).toMatch(/cross-origin/i);
|
||||
expect(msg).toMatch(/paste the data inline/i);
|
||||
});
|
||||
|
||||
test('an HTTP 404 explains "not found" without a raw error code', () => {
|
||||
const msg = remoteFetchErrorMessage(new RemoteFetchError('http', 'x', 404));
|
||||
expect(msg).toMatch(/couldn't find/i);
|
||||
expect(msg).not.toMatch(/404/);
|
||||
});
|
||||
|
||||
test('an HTTP 403 explains refused access', () => {
|
||||
expect(remoteFetchErrorMessage(new RemoteFetchError('http', 'x', 403))).toMatch(
|
||||
/refused access/i,
|
||||
);
|
||||
});
|
||||
|
||||
test('an empty body says it returned no data', () => {
|
||||
expect(remoteFetchErrorMessage(new RemoteFetchError('empty', 'x'))).toMatch(/no data/i);
|
||||
});
|
||||
|
||||
test('the retry recovery points at refreshing, not pasting inline', () => {
|
||||
const msg = remoteFetchErrorMessage(new RemoteFetchError('timeout', 'x'), 'retry');
|
||||
expect(msg).toMatch(/refreshing again/i);
|
||||
expect(msg).not.toMatch(/inline/i);
|
||||
});
|
||||
|
||||
test('an unknown error still produces a sensible fallback message', () => {
|
||||
expect(remoteFetchErrorMessage(new Error('boom'))).toMatch(/couldn't fetch this url/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Translate a remote-data fetch failure (snapshot model — spec §05 → URL datasets)
|
||||
* into the message shown when adding or refreshing a URL dataset.
|
||||
*
|
||||
* Pure and framework-free so the wording is unit-tested directly — the same split
|
||||
* as `storage-errors`. Follows the recorded error-copy resolution (docs/architecture
|
||||
* /10 → error copy): plain language, name what stopped, and — because every one of
|
||||
* these is user-fixable — point at the next step. No error codes in the user-facing
|
||||
* line (GOV.UK / NN/g #9). The recovery clause adapts to where the error shows: the
|
||||
* create/edit form can fall back to pasting inline; Refresh can only retry.
|
||||
*/
|
||||
|
||||
import { RemoteFetchError } from '../infrastructure/remote-data';
|
||||
|
||||
/** Where the error surfaces, which decides the recovery the copy points at. */
|
||||
export type FetchRecovery = 'inline' | 'retry';
|
||||
|
||||
function nextStep(recovery: FetchRecovery): string {
|
||||
return recovery === 'inline'
|
||||
? 'Check the address, or paste the data inline instead.'
|
||||
: 'Check the address and try refreshing again.';
|
||||
}
|
||||
|
||||
function httpCause(status?: number): string {
|
||||
if (status === 404) return "The server couldn't find anything at this URL.";
|
||||
if (status === 401 || status === 403) return 'The server refused access to this URL.';
|
||||
if (status && status >= 500) return 'The server hit an error serving this URL.';
|
||||
return "The server couldn't return this URL.";
|
||||
}
|
||||
|
||||
/**
|
||||
* The user-facing message for a fetch failure. `recovery` defaults to `inline`
|
||||
* (the add/edit form); pass `retry` for the Refresh action, which has no inline
|
||||
* fallback.
|
||||
*/
|
||||
export function remoteFetchErrorMessage(err: unknown, recovery: FetchRecovery = 'inline'): string {
|
||||
const next = nextStep(recovery);
|
||||
if (err instanceof RemoteFetchError) {
|
||||
switch (err.reason) {
|
||||
case 'network':
|
||||
return `Couldn't fetch this URL. Your device may be offline, or the host may block cross-origin requests. ${next}`;
|
||||
case 'http':
|
||||
return `${httpCause(err.status)} ${next}`;
|
||||
case 'empty':
|
||||
return `This URL returned no data. Make sure it points directly at a data file. ${next}`;
|
||||
case 'timeout':
|
||||
return `This URL took too long to respond. ${next}`;
|
||||
}
|
||||
}
|
||||
return `Couldn't fetch this URL. ${next}`;
|
||||
}
|
||||
@@ -42,15 +42,82 @@ describe('save — create', () => {
|
||||
expect(ds?.rowCount).toBe(2);
|
||||
});
|
||||
|
||||
test('a valid URL infers format from the extension', () => {
|
||||
test('save() does not commit a URL create — it must go through commitUrlSnapshot', () => {
|
||||
store().startCreate();
|
||||
store().updateForm({ name: 'Remote', source: 'url', input: 'https://example.com/data.csv' });
|
||||
// The component fetches first; save() never touches the network, so a URL create
|
||||
// through save() is a no-op rather than an unfetched record.
|
||||
expect(store().save(T)).toBe(false);
|
||||
expect(store().datasets).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('commitUrlSnapshot — create from a fetched body', () => {
|
||||
test('snapshots the body, infers format from content, and profiles it', () => {
|
||||
store().startCreate();
|
||||
store().updateForm({ name: 'Remote', source: 'url', input: 'https://example.com/data.csv' });
|
||||
|
||||
expect(store().save(T)).toBe(true);
|
||||
expect(store().commitUrlSnapshot({ text: 'a,b\n1,2\n3,4' }, T)).toBe(true);
|
||||
const ds = selectSelectedDataset(store());
|
||||
expect(ds?.source).toBe('url');
|
||||
expect(ds?.url).toBe('https://example.com/data.csv');
|
||||
expect(ds?.format).toBe('csv');
|
||||
expect(ds?.rowCount).toBeNull(); // URL datasets aren't profiled
|
||||
expect(ds?.data).toBe('a,b\n1,2\n3,4');
|
||||
expect(ds?.rowCount).toBe(2);
|
||||
expect(ds?.columns).toEqual(['a', 'b']);
|
||||
expect(ds?.fetchedAt).toBe(T.toISOString());
|
||||
expect(store().view).toBe('detail');
|
||||
});
|
||||
|
||||
test('a duplicate name is rejected even with a successful fetch', () => {
|
||||
store().add(
|
||||
createDataset({ name: 'Dupe', data: [{ a: 1 }], format: 'json', source: 'inline', now: T }),
|
||||
);
|
||||
store().startCreate();
|
||||
store().updateForm({ name: 'dupe', source: 'url', input: 'https://x/y.csv' });
|
||||
expect(store().commitUrlSnapshot({ text: 'a\n1' }, T)).toBe(false);
|
||||
expect(store().formError).toMatch(/already exists/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('URL dataset edits & refresh', () => {
|
||||
const seedUrl = () => {
|
||||
store().startCreate();
|
||||
store().updateForm({ name: 'Remote', source: 'url', input: 'https://x/y.csv' });
|
||||
store().commitUrlSnapshot({ text: 'a,b\n1,2' }, T);
|
||||
return store().selectedId!;
|
||||
};
|
||||
|
||||
test('a metadata-only edit (same URL) updates name without re-fetching', () => {
|
||||
const id = seedUrl();
|
||||
store().startEdit();
|
||||
store().updateForm({ name: 'Renamed' }); // URL field stays https://x/y.csv
|
||||
const later = new Date('2026-07-01T00:00:00Z');
|
||||
expect(store().save(later)).toBe(true);
|
||||
const ds = store().datasets.find((d) => d.id === id)!;
|
||||
expect(ds.name).toBe('Renamed');
|
||||
expect(ds.data).toBe('a,b\n1,2'); // snapshot untouched
|
||||
expect(ds.fetchedAt).toBe(T.toISOString()); // not re-fetched
|
||||
expect(ds.modified).toBe(later.toISOString());
|
||||
});
|
||||
|
||||
test('refreshDataset re-snapshots, re-profiles, and advances fetchedAt', () => {
|
||||
const id = seedUrl();
|
||||
const later = new Date('2026-07-01T00:00:00Z');
|
||||
expect(store().refreshDataset(id, { text: 'a,b\n1,2\n3,4\n5,6' }, later)).toBe(true);
|
||||
const ds = store().datasets.find((d) => d.id === id)!;
|
||||
expect(ds.rowCount).toBe(3);
|
||||
expect(ds.fetchedAt).toBe(later.toISOString());
|
||||
expect(ds.modified).toBe(later.toISOString());
|
||||
expect(ds.name).toBe('Remote'); // preserved
|
||||
});
|
||||
|
||||
test('refreshDataset is a no-op for an inline dataset', () => {
|
||||
store().add(
|
||||
createDataset({ name: 'Inline', data: [{ a: 1 }], format: 'json', source: 'inline', now: T }),
|
||||
);
|
||||
const id = store().selectedId!;
|
||||
expect(store().refreshDataset(id, { text: 'a\n1' }, T)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+141
-19
@@ -19,8 +19,14 @@
|
||||
*/
|
||||
|
||||
import { create } from 'zustand';
|
||||
import { detectFormat, detectFormatFromUrl, type DataFormat } from '@core/format-detection';
|
||||
import { createDataset, computeDatasetProfile, type DataSource, type Dataset } from '@core/dataset';
|
||||
import { detectFormat, type DataFormat } from '@core/format-detection';
|
||||
import {
|
||||
createDataset,
|
||||
computeDatasetProfile,
|
||||
snapshotFromText,
|
||||
type DataSource,
|
||||
type Dataset,
|
||||
} from '@core/dataset';
|
||||
import { isNameTaken } from '@core/naming';
|
||||
import { useSnippetStore } from './SnippetStore';
|
||||
|
||||
@@ -67,6 +73,21 @@ export interface DatasetState {
|
||||
*/
|
||||
save: (now?: Date) => boolean;
|
||||
|
||||
/**
|
||||
* Commit a URL dataset from an already-fetched body. The component performs the
|
||||
* network fetch via the remote-data adapter and passes the result here, keeping
|
||||
* this store browser-free. Validates the form, snapshots + profiles the body, and
|
||||
* creates (view `new`) or updates (view `edit`) the dataset — including renaming
|
||||
* referencing snippets on an edit. Returns whether it committed.
|
||||
*/
|
||||
commitUrlSnapshot: (fetched: { text: string }, now?: Date) => boolean;
|
||||
/**
|
||||
* Re-snapshot an existing URL dataset from a freshly-fetched body ("Refresh"):
|
||||
* re-detect the format, re-profile, and advance `fetchedAt`/`modified`. Name,
|
||||
* comment, and URL are preserved. Returns false for a non-URL dataset.
|
||||
*/
|
||||
refreshDataset: (id: number, fetched: { text: string }, now?: Date) => boolean;
|
||||
|
||||
/**
|
||||
* Low-level: add a fully-formed dataset and select it. The id is (re)assigned
|
||||
* via `nextDatasetId`, so a `createDataset` default id (`Date.now()`) can never
|
||||
@@ -144,7 +165,12 @@ function validateForm(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Validate a form into a saveable shape, or return an error message. */
|
||||
/**
|
||||
* Validate an **inline** form into a saveable shape, or return an error message.
|
||||
* URL datasets never reach here — they are fetched first and committed through
|
||||
* `commitUrlSnapshot` (which snapshots the fetched body), so this only shapes the
|
||||
* pasted-inline payload.
|
||||
*/
|
||||
function resolveForm(
|
||||
form: DatasetForm,
|
||||
datasets: Dataset[],
|
||||
@@ -153,20 +179,12 @@ function resolveForm(
|
||||
const error = validateForm(form, datasets, excludeId);
|
||||
if (error) return { error };
|
||||
|
||||
const name = form.name.trim();
|
||||
const input = form.input.trim();
|
||||
|
||||
if (form.source === 'url') {
|
||||
// Format is inferred from the extension; default to JSON when unknown (§05).
|
||||
return { name, data: input, format: detectFormatFromUrl(input) ?? 'json', source: 'url' };
|
||||
}
|
||||
|
||||
// validateForm guaranteed a detectable format above.
|
||||
const format = detectFormat(form.input).format as DataFormat;
|
||||
// JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text (data model §09B).
|
||||
const data =
|
||||
format === 'json' || format === 'topojson' ? (JSON.parse(form.input) as unknown) : form.input;
|
||||
return { name, data, format, source: 'inline' };
|
||||
return { name: form.name.trim(), data, format, source: 'inline' };
|
||||
}
|
||||
|
||||
export const useDatasetStore = create<DatasetState>((set, get) => ({
|
||||
@@ -201,12 +219,14 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
|
||||
name: ds.name,
|
||||
source: ds.source,
|
||||
comment: ds.comment,
|
||||
// Re-render the stored payload as editable text: raw for csv/tsv/url,
|
||||
// pretty-printed JSON for json/topojson.
|
||||
// Re-render the editable text: the URL for url datasets, raw text for
|
||||
// csv/tsv, pretty-printed JSON for json/topojson.
|
||||
input:
|
||||
ds.source === 'url' || ds.format === 'csv' || ds.format === 'tsv'
|
||||
? String(ds.data)
|
||||
: JSON.stringify(ds.data, null, 2),
|
||||
ds.source === 'url'
|
||||
? (ds.url ?? '')
|
||||
: ds.format === 'csv' || ds.format === 'tsv'
|
||||
? String(ds.data)
|
||||
: JSON.stringify(ds.data, null, 2),
|
||||
},
|
||||
});
|
||||
},
|
||||
@@ -225,6 +245,30 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
|
||||
const editing = view === 'edit';
|
||||
const excludeId = editing ? (selectedId ?? undefined) : undefined;
|
||||
|
||||
// URL datasets that need the network — a create, a URL change, or an inline→URL
|
||||
// conversion — are routed through `commitUrlSnapshot` after the component
|
||||
// fetches; save() never fetches. The one URL case it commits is a metadata-only
|
||||
// edit of an existing snapshot (same URL): just update name/comment, no re-fetch.
|
||||
if (form.source === 'url') {
|
||||
const error = validateForm(form, datasets, excludeId);
|
||||
if (error) {
|
||||
set({ formError: error });
|
||||
return false;
|
||||
}
|
||||
if (!editing || selectedId === null) return false;
|
||||
const existing = datasets.find((d) => d.id === selectedId);
|
||||
if (!existing || existing.source !== 'url' || form.input.trim() !== (existing.url ?? '')) {
|
||||
return false;
|
||||
}
|
||||
const name = form.name.trim();
|
||||
get().update(selectedId, { name, comment: form.comment }, now);
|
||||
if (name !== existing.name) {
|
||||
useSnippetStore.getState().renameDatasetRefs(existing.name, name, now);
|
||||
}
|
||||
set({ view: 'detail', form: EMPTY_FORM, formError: null });
|
||||
return true;
|
||||
}
|
||||
|
||||
const resolved = resolveForm(form, datasets, excludeId);
|
||||
if ('error' in resolved) {
|
||||
set({ formError: resolved.error });
|
||||
@@ -234,11 +278,12 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
|
||||
if (editing && selectedId !== null) {
|
||||
const existing = datasets.find((d) => d.id === selectedId);
|
||||
if (!existing) return false;
|
||||
const profile = computeDatasetProfile(resolved.data, resolved.format, resolved.source);
|
||||
const profile = computeDatasetProfile(resolved.data, resolved.format);
|
||||
// Re-profile and update the record (including any new name), then propagate
|
||||
// the rename across referencing snippets so each spec and its datasetRefs
|
||||
// stay consistent (docs/architecture/07 §6). SnippetStore never imports this
|
||||
// store, so the direct call is cycle-free.
|
||||
// store, so the direct call is cycle-free. Clear any url/fetchedAt left over
|
||||
// from a URL→inline conversion so the record carries no stale remote origin.
|
||||
get().update(
|
||||
selectedId,
|
||||
{
|
||||
@@ -246,6 +291,13 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
|
||||
data: resolved.data,
|
||||
format: resolved.format,
|
||||
source: resolved.source,
|
||||
// TODO: `update` shallow-merges, so these set the keys to `undefined`
|
||||
// rather than removing them — a URL→inline record keeps `url`/`fetchedAt`
|
||||
// present-but-undefined. Invisible today (rendering/profiling key off
|
||||
// `source`/`data == null`, and JSON export drops undefined), but it breaks
|
||||
// the "inline records carry no url/fetchedAt keys" invariant on this path.
|
||||
url: undefined,
|
||||
fetchedAt: undefined,
|
||||
comment: form.comment,
|
||||
...profile,
|
||||
},
|
||||
@@ -275,6 +327,76 @@ export const useDatasetStore = create<DatasetState>((set, get) => ({
|
||||
return true;
|
||||
},
|
||||
|
||||
commitUrlSnapshot: (fetched, now) => {
|
||||
const { view, form, datasets, selectedId } = get();
|
||||
const editing = view === 'edit';
|
||||
const excludeId = editing ? (selectedId ?? undefined) : undefined;
|
||||
|
||||
const error = validateForm(form, datasets, excludeId);
|
||||
if (error) {
|
||||
set({ formError: error });
|
||||
return false;
|
||||
}
|
||||
|
||||
const name = form.name.trim();
|
||||
const url = form.input.trim();
|
||||
const iso = (now ?? new Date()).toISOString();
|
||||
// The fetched body snapshots + profiles exactly like inline data (snapshot
|
||||
// model): detect format from content, shape, and profile through core.
|
||||
const { data, format } = snapshotFromText(fetched.text, url);
|
||||
|
||||
if (editing && selectedId !== null) {
|
||||
const existing = datasets.find((d) => d.id === selectedId);
|
||||
if (!existing) return false;
|
||||
const profile = computeDatasetProfile(data, format);
|
||||
get().update(
|
||||
selectedId,
|
||||
{
|
||||
name,
|
||||
data,
|
||||
format,
|
||||
source: 'url',
|
||||
url,
|
||||
fetchedAt: iso,
|
||||
comment: form.comment,
|
||||
...profile,
|
||||
},
|
||||
now,
|
||||
);
|
||||
if (name !== existing.name) {
|
||||
useSnippetStore.getState().renameDatasetRefs(existing.name, name, now);
|
||||
}
|
||||
set({ view: 'detail', form: EMPTY_FORM, formError: null });
|
||||
return true;
|
||||
}
|
||||
|
||||
const dataset = createDataset({
|
||||
name,
|
||||
data,
|
||||
format,
|
||||
source: 'url',
|
||||
url,
|
||||
fetchedAt: iso,
|
||||
comment: form.comment,
|
||||
now,
|
||||
});
|
||||
get().add(dataset);
|
||||
set({ view: 'detail', form: EMPTY_FORM, formError: null });
|
||||
return true;
|
||||
},
|
||||
|
||||
refreshDataset: (id, fetched, now) => {
|
||||
const dataset = get().datasets.find((d) => d.id === id);
|
||||
if (!dataset || dataset.source !== 'url' || !dataset.url) return false;
|
||||
const iso = (now ?? new Date()).toISOString();
|
||||
const { data, format } = snapshotFromText(fetched.text, dataset.url);
|
||||
const profile = computeDatasetProfile(data, format);
|
||||
// Name, comment, and url are preserved; only the snapshot + its profile and the
|
||||
// fetch time change. `update` advances `modified`.
|
||||
get().update(id, { data, format, fetchedAt: iso, ...profile }, now);
|
||||
return true;
|
||||
},
|
||||
|
||||
add: (dataset) =>
|
||||
set((s) => {
|
||||
const withId = { ...dataset, id: nextDatasetId(s.datasets) };
|
||||
|
||||
+122
-21
@@ -5,6 +5,8 @@ import {
|
||||
createDataset,
|
||||
datasetReference,
|
||||
parseDelimited,
|
||||
snapshotFromText,
|
||||
tabularRows,
|
||||
} from './dataset';
|
||||
|
||||
describe('datasetReference', () => {
|
||||
@@ -65,51 +67,125 @@ describe('parseDelimited', () => {
|
||||
});
|
||||
|
||||
describe('computeDatasetProfile', () => {
|
||||
test('inline JSON array-of-objects is profiled', () => {
|
||||
test('JSON array-of-objects is profiled', () => {
|
||||
const data = [
|
||||
{ a: 1, b: 'x' },
|
||||
{ a: 2, b: 'y' },
|
||||
];
|
||||
const profile = computeDatasetProfile(data, 'json', 'inline');
|
||||
const profile = computeDatasetProfile(data, 'json');
|
||||
expect(profile.rowCount).toBe(2);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(data)).length);
|
||||
});
|
||||
|
||||
test('inline JSON that is not an array is N/A but sized', () => {
|
||||
const profile = computeDatasetProfile({ a: 1 }, 'json', 'inline');
|
||||
test('JSON that is not an array is N/A but sized', () => {
|
||||
const profile = computeDatasetProfile({ a: 1 }, 'json');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('inline CSV is parsed and profiled; size is the raw text byte length', () => {
|
||||
test('CSV is parsed and profiled; size is the raw text byte length', () => {
|
||||
const text = 'a,b\n1,2';
|
||||
const profile = computeDatasetProfile(text, 'csv', 'inline');
|
||||
const profile = computeDatasetProfile(text, 'csv');
|
||||
expect(profile.rowCount).toBe(1);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
expect(profile.size).toBe(new TextEncoder().encode(text).length);
|
||||
});
|
||||
|
||||
test('inline TSV is parsed and profiled', () => {
|
||||
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv', 'inline');
|
||||
test('TSV is parsed and profiled', () => {
|
||||
const profile = computeDatasetProfile('a\tb\n1\t2', 'tsv');
|
||||
expect(profile.rowCount).toBe(1);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('inline TopoJSON is N/A (non-tabular) but sized', () => {
|
||||
test('TopoJSON is N/A (non-tabular) but sized', () => {
|
||||
const topo = { type: 'Topology', objects: {} };
|
||||
const profile = computeDatasetProfile(topo, 'topojson', 'inline');
|
||||
const profile = computeDatasetProfile(topo, 'topojson');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.size).toBe(new TextEncoder().encode(JSON.stringify(topo)).length);
|
||||
});
|
||||
|
||||
test('URL is N/A; size is the byte length of the URL string', () => {
|
||||
const url = 'https://example.com/data.csv';
|
||||
const profile = computeDatasetProfile(url, 'csv', 'url');
|
||||
test('a fetched URL snapshot is profiled exactly like inline data', () => {
|
||||
// The snapshot model: once fetched, a URL dataset carries its payload in `data`
|
||||
// and profiles through the same path as inline (no source distinction here).
|
||||
const profile = computeDatasetProfile('a,b\n1,2\n3,4', 'csv');
|
||||
expect(profile.rowCount).toBe(2);
|
||||
expect(profile.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('an unfetched URL reference (null data) is N/A with size 0', () => {
|
||||
const profile = computeDatasetProfile(null, 'csv');
|
||||
expect(profile.rowCount).toBeNull();
|
||||
expect(profile.columnCount).toBeNull();
|
||||
expect(profile.size).toBe(new TextEncoder().encode(url).length);
|
||||
expect(profile.columns).toEqual([]);
|
||||
expect(profile.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('tabularRows', () => {
|
||||
test('CSV parses to rows (same parsing as profiling)', () => {
|
||||
expect(tabularRows('a,b\n1,2\n3,4', 'csv')).toEqual([
|
||||
{ a: '1', b: '2' },
|
||||
{ a: '3', b: '4' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('TSV parses to rows', () => {
|
||||
expect(tabularRows('x\ty\n1\t2', 'tsv')).toEqual([{ x: '1', y: '2' }]);
|
||||
});
|
||||
|
||||
test('a JSON array of objects is returned as-is', () => {
|
||||
const data = [{ a: 1 }, { a: 2 }];
|
||||
expect(tabularRows(data, 'json')).toEqual(data);
|
||||
});
|
||||
|
||||
test('limit returns only the head', () => {
|
||||
const rows = tabularRows('n\n1\n2\n3\n4', 'csv', 2);
|
||||
expect(rows).toEqual([{ n: '1' }, { n: '2' }]);
|
||||
});
|
||||
|
||||
test('non-tabular payloads return null (single object, topojson, unfetched)', () => {
|
||||
expect(tabularRows({ a: 1 }, 'json')).toBeNull();
|
||||
expect(tabularRows({ type: 'Topology', objects: {} }, 'topojson')).toBeNull();
|
||||
expect(tabularRows(null, 'csv')).toBeNull();
|
||||
expect(tabularRows('a,b', 'csv')).toBeNull(); // header only — no rows
|
||||
});
|
||||
});
|
||||
|
||||
describe('snapshotFromText', () => {
|
||||
test('JSON content is parsed and typed json (content beats extension)', () => {
|
||||
const { data, format } = snapshotFromText('[{"a":1}]', 'https://x/data.txt');
|
||||
expect(format).toBe('json');
|
||||
expect(data).toEqual([{ a: 1 }]);
|
||||
});
|
||||
|
||||
test('CSV content keeps its raw text and is typed csv', () => {
|
||||
const { data, format } = snapshotFromText('a,b\n1,2', 'https://x/data.csv');
|
||||
expect(format).toBe('csv');
|
||||
expect(data).toBe('a,b\n1,2');
|
||||
});
|
||||
|
||||
test('TSV content is typed tsv', () => {
|
||||
expect(snapshotFromText('a\tb\n1\t2', 'https://x/d').format).toBe('tsv');
|
||||
});
|
||||
|
||||
test('a Topology body is typed topojson and parsed', () => {
|
||||
const { data, format } = snapshotFromText('{"type":"Topology","objects":{}}', 'https://x/d');
|
||||
expect(format).toBe('topojson');
|
||||
expect(data).toEqual({ type: 'Topology', objects: {} });
|
||||
});
|
||||
|
||||
test('unrecognized content falls back to the URL extension', () => {
|
||||
// A single token: not JSON, not delimited. The .json extension decides the
|
||||
// format; the unparseable body is kept verbatim rather than throwing.
|
||||
const { data, format } = snapshotFromText('not-data', 'https://x/data.json');
|
||||
expect(format).toBe('json');
|
||||
expect(data).toBe('not-data');
|
||||
});
|
||||
|
||||
test('unrecognized content with no useful extension defaults to json', () => {
|
||||
expect(snapshotFromText('not-data', 'https://x/feed').format).toBe('json');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -157,20 +233,45 @@ describe('createDataset', () => {
|
||||
expect(d.columns).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('URL dataset gets an N/A profile with a size', () => {
|
||||
test('URL dataset carries url + fetchedAt and profiles the fetched snapshot', () => {
|
||||
const d = createDataset({
|
||||
id: 2,
|
||||
name: 'Remote',
|
||||
data: 'https://example.com/x.json',
|
||||
format: 'json',
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://example.com/x.csv',
|
||||
fetchedAt: '2026-06-10T00:00:00.000Z',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
comment: 'remote source',
|
||||
});
|
||||
expect(d.rowCount).toBeNull();
|
||||
expect(d.columnCount).toBeNull();
|
||||
expect(d.columns).toEqual([]);
|
||||
expect(d.source).toBe('url');
|
||||
expect(d.url).toBe('https://example.com/x.csv');
|
||||
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
|
||||
expect(d.rowCount).toBe(1);
|
||||
expect(d.columns).toEqual(['a', 'b']);
|
||||
expect(d.comment).toBe('remote source');
|
||||
expect(d.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('an unfetched URL dataset (null data) gets an N/A profile and null fetchedAt', () => {
|
||||
const d = createDataset({
|
||||
id: 3,
|
||||
name: 'Unfetched',
|
||||
data: null,
|
||||
url: 'https://example.com/x.csv',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
});
|
||||
expect(d.url).toBe('https://example.com/x.csv');
|
||||
expect(d.fetchedAt).toBeNull();
|
||||
expect(d.rowCount).toBeNull();
|
||||
expect(d.columns).toEqual([]);
|
||||
expect(d.size).toBe(0);
|
||||
});
|
||||
|
||||
test('inline dataset carries no url/fetchedAt keys', () => {
|
||||
const d = createDataset({ id: 4, name: 'I', data: [], format: 'json', source: 'inline' });
|
||||
expect('url' in d).toBe(false);
|
||||
expect('fetchedAt' in d).toBe(false);
|
||||
});
|
||||
|
||||
test('defaults the id when not injected', () => {
|
||||
|
||||
+102
-29
@@ -7,22 +7,29 @@
|
||||
* a simple delimited-text parser, the profiling orchestration, and a factory that
|
||||
* stamps timestamps/version and fills the derived summary fields.
|
||||
*
|
||||
* A dataset has one of two **sources** — `inline` (data stored in the record) or
|
||||
* `url` (only the link is stored, fetched on demand at render time) — and one of
|
||||
* four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
|
||||
* The `data` field's shape follows source/format: a URL string for `url`; raw
|
||||
* text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
|
||||
* A dataset has one of two **sources** — `inline` (data pasted into the record) or
|
||||
* `url` (fetched once from a remote address and **snapshotted** into the record) —
|
||||
* and one of four **formats** (reused from format-detection: `json`/`csv`/`tsv`/
|
||||
* `topojson`). Either way `data` holds the actual payload, shaped by format: raw
|
||||
* text for CSV/TSV, a parsed value for JSON/TopoJSON. A `url` dataset additionally
|
||||
* keeps its source `url` (so it can be re-fetched) and a `fetchedAt` timestamp;
|
||||
* until its first successful fetch `data` is `null` (an unfetched reference).
|
||||
*
|
||||
* Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
|
||||
* non-tabular data get an N/A profile but are still sized (see profile.ts).
|
||||
* Any tabular payload (JSON array-of-objects, CSV, TSV) is profiled — including a
|
||||
* fetched URL snapshot; non-tabular or not-yet-fetched data gets an N/A profile but
|
||||
* is still sized (see profile.ts).
|
||||
*/
|
||||
|
||||
import type { DataFormat } from './format-detection';
|
||||
import { detectFormat, detectFormatFromUrl, type DataFormat } from './format-detection';
|
||||
import { profileData, type ColumnStats, type DatasetProfile } from './profile';
|
||||
import type { ColumnType } from './type-inference';
|
||||
|
||||
/** Current schema version for a Dataset record (read-time migration target). */
|
||||
export const CURRENT_DATASET_VERSION = 1;
|
||||
/**
|
||||
* Current schema version for a Dataset record (read-time migration target).
|
||||
* v2 moved a URL dataset's address out of `data` into its own `url` field and made
|
||||
* `data` hold the fetched snapshot (see `migrateDataset`).
|
||||
*/
|
||||
export const CURRENT_DATASET_VERSION = 2;
|
||||
|
||||
/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
|
||||
export type DataSource = 'inline' | 'url';
|
||||
@@ -35,14 +42,25 @@ export interface Dataset {
|
||||
/** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
|
||||
name: string;
|
||||
/**
|
||||
* The payload. For `source = url`: the URL string. For `source = inline`: the
|
||||
* raw CSV/TSV text, or the parsed JSON/TopoJSON value.
|
||||
* The payload, shaped by format: raw CSV/TSV text, or the parsed JSON/TopoJSON
|
||||
* value. For `source = url` this is the fetched snapshot, or `null` before the
|
||||
* first successful fetch.
|
||||
*/
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/**
|
||||
* For `source = url`: the remote address the snapshot was fetched from, retained
|
||||
* so the dataset can be re-fetched ("Refresh"). Absent for inline datasets.
|
||||
*/
|
||||
url?: string;
|
||||
/**
|
||||
* For `source = url`: ISO timestamp of the last successful fetch, or `null` when
|
||||
* it has never been fetched. Absent for inline datasets.
|
||||
*/
|
||||
fetchedAt?: string | null;
|
||||
/** Free-form user note about the dataset. */
|
||||
comment: string;
|
||||
/** Data rows, or `null` when N/A (URL / non-tabular). */
|
||||
@@ -187,26 +205,52 @@ function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
|
||||
* tabular vs N/A by source/format, and delegate to `profileData`.
|
||||
*
|
||||
* - `url` (any format) → N/A profile; size = byte length of the URL string.
|
||||
* - inline `json` → rows when a non-empty array of objects, else N/A.
|
||||
* - inline `topojson` → N/A (non-tabular).
|
||||
* - inline `csv` / `tsv` → `parseDelimited` rows.
|
||||
*
|
||||
* `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
|
||||
* `JSON.stringify(data)` for json/topojson.
|
||||
* The tabular rows of a dataset payload for a table preview, or `null` when the
|
||||
* payload isn't tabular (a single JSON object, TopoJSON, or an unfetched URL). Uses
|
||||
* the **same** parsing as profiling — `parseDelimited` for CSV/TSV, `asObjectRows`
|
||||
* for JSON — so the previewed rows agree exactly with the profiled `columns`. A
|
||||
* positive `limit` returns only the head (a preview needs a sample, not the whole
|
||||
* payload). Returns non-null on precisely the inputs `computeDatasetProfile` counts
|
||||
* as rows, so a caller can gate "table vs. raw text" on this alone.
|
||||
*/
|
||||
export function computeDatasetProfile(
|
||||
export function tabularRows(
|
||||
data: unknown,
|
||||
format: DataFormat,
|
||||
source: DataSource,
|
||||
): DatasetProfile {
|
||||
if (source === 'url') {
|
||||
const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
|
||||
return profileData(null, byteLength(url));
|
||||
limit?: number,
|
||||
): Array<Record<string, unknown>> | null {
|
||||
if (data == null) return null;
|
||||
let rows: Array<Record<string, unknown>> | null;
|
||||
switch (format) {
|
||||
case 'csv':
|
||||
case 'tsv':
|
||||
rows = parseDelimited(typeof data === 'string' ? data : '', format);
|
||||
break;
|
||||
case 'json':
|
||||
rows = asObjectRows(data);
|
||||
break;
|
||||
default:
|
||||
rows = null;
|
||||
}
|
||||
if (!rows || rows.length === 0) return null;
|
||||
return limit != null && limit >= 0 && rows.length > limit ? rows.slice(0, limit) : rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
|
||||
* tabular vs N/A by format, and delegate to `profileData`. Identical for inline
|
||||
* data and for a fetched URL snapshot — both carry the payload in `data`.
|
||||
*
|
||||
* - `null` data (unfetched URL) → N/A profile, size 0.
|
||||
* - `json` → rows when a non-empty array of objects, else N/A.
|
||||
* - `topojson` → N/A (non-tabular).
|
||||
* - `csv` / `tsv` → `parseDelimited` rows.
|
||||
*
|
||||
* `size` is the UTF-8 byte length of the raw string for csv/tsv, or of
|
||||
* `JSON.stringify(data)` for json/topojson.
|
||||
*/
|
||||
export function computeDatasetProfile(data: unknown, format: DataFormat): DatasetProfile {
|
||||
// An unfetched URL reference (or a genuinely absent payload): nothing to profile.
|
||||
if (data == null) return profileData(null, 0);
|
||||
|
||||
switch (format) {
|
||||
case 'csv':
|
||||
@@ -226,6 +270,29 @@ export function computeDatasetProfile(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shape a freshly-fetched URL body into the `{ data, format }` a snapshot stores
|
||||
* (spec §05 → URL datasets, snapshot model). Format is sniffed from the **content**
|
||||
* first — authoritative, since `detectFormat` only reports `json` when the body
|
||||
* actually parses — falling back to the URL's file extension, then JSON.
|
||||
* JSON/TopoJSON are stored parsed; CSV/TSV keep their raw text — the same
|
||||
* per-format shaping inline data uses, so a fetched dataset profiles and renders
|
||||
* identically to an inline one (see `computeDatasetProfile`, rendering.ts).
|
||||
*/
|
||||
export function snapshotFromText(text: string, url: string): { data: unknown; format: DataFormat } {
|
||||
const format = detectFormat(text).format ?? detectFormatFromUrl(url) ?? 'json';
|
||||
if (format === 'json' || format === 'topojson') {
|
||||
try {
|
||||
return { data: JSON.parse(text) as unknown, format };
|
||||
} catch {
|
||||
// The extension promised JSON but the body isn't — keep the raw text so the
|
||||
// render surfaces a readable error instead of us throwing mid-commit.
|
||||
return { data: text, format };
|
||||
}
|
||||
}
|
||||
return { data: text, format };
|
||||
}
|
||||
|
||||
export interface CreateDatasetOptions {
|
||||
/** The dataset name (uniqueness is enforced upstream — see naming.ts). */
|
||||
name: string;
|
||||
@@ -235,6 +302,10 @@ export interface CreateDatasetOptions {
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** For `source = url`: the remote address (retained for Refresh). */
|
||||
url?: string;
|
||||
/** For `source = url`: ISO timestamp of the fetch that produced `data`. */
|
||||
fetchedAt?: string | null;
|
||||
/** Optional free-form note. */
|
||||
comment?: string;
|
||||
/** Clock injection for deterministic tests; defaults to the current time. */
|
||||
@@ -253,7 +324,7 @@ export interface CreateDatasetOptions {
|
||||
export function createDataset(options: CreateDatasetOptions): Dataset {
|
||||
const now = options.now ?? new Date();
|
||||
const iso = now.toISOString();
|
||||
const profile = computeDatasetProfile(options.data, options.format, options.source);
|
||||
const profile = computeDatasetProfile(options.data, options.format);
|
||||
|
||||
return {
|
||||
id: options.id ?? Date.now(),
|
||||
@@ -262,6 +333,8 @@ export function createDataset(options: CreateDatasetOptions): Dataset {
|
||||
data: options.data,
|
||||
format: options.format,
|
||||
source: options.source,
|
||||
// URL datasets carry their address + fetch time; inline records stay clean.
|
||||
...(options.source === 'url' ? { url: options.url, fetchedAt: options.fetchedAt ?? null } : {}),
|
||||
comment: options.comment ?? '',
|
||||
rowCount: profile.rowCount,
|
||||
columnCount: profile.columnCount,
|
||||
|
||||
@@ -233,6 +233,42 @@ describe('normalizeImport — dataset normalization', () => {
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
});
|
||||
|
||||
it('v1→v2: a legacy URL dataset (address in data) imports as an unfetched reference', () => {
|
||||
const parsed = {
|
||||
version: '1.0',
|
||||
snippets: [],
|
||||
datasets: [{ id: 1, name: 'Remote', source: 'url', format: 'csv', data: 'https://x/y.csv' }],
|
||||
};
|
||||
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
|
||||
expect(d.source).toBe('url');
|
||||
expect(d.url).toBe('https://x/y.csv');
|
||||
expect(d.data).toBeNull();
|
||||
expect(d.fetchedAt).toBeNull();
|
||||
expect(d.version).toBe(CURRENT_DATASET_VERSION);
|
||||
});
|
||||
|
||||
it('a v2 URL snapshot imports with its data, url, and fetchedAt intact', () => {
|
||||
const parsed = {
|
||||
version: '1.0',
|
||||
snippets: [],
|
||||
datasets: [
|
||||
{
|
||||
id: 1,
|
||||
name: 'Remote',
|
||||
source: 'url',
|
||||
format: 'csv',
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://x/y.csv',
|
||||
fetchedAt: '2026-06-10T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
};
|
||||
const d = normalizeImport(parsed, { now: FIXED_NOW }).datasets[0];
|
||||
expect(d.data).toBe('a,b\n1,2');
|
||||
expect(d.url).toBe('https://x/y.csv');
|
||||
expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('fills gaps with defaults and coerces id to a number', () => {
|
||||
const parsed = { version: '1.0', snippets: [], datasets: [{ id: '42', name: 'D' }] };
|
||||
const result = normalizeImport(parsed, { now: FIXED_NOW });
|
||||
|
||||
@@ -151,6 +151,11 @@ function normalizeSnippet(raw: unknown, nowIso: string, makeId: () => string): S
|
||||
* datasets already carry their derived summary fields (rowCount, columns, …); we
|
||||
* preserve those and only fill gaps. We do NOT re-profile here — that needs the
|
||||
* profiling pipeline and would be wasteful for already-summarized records.
|
||||
*
|
||||
* Applies the v1→v2 URL-snapshot shaping (mirrors `migrateDataset`): a pre-v2 export
|
||||
* stored a URL dataset's address in `data`, so we move it into `url` and clear `data`
|
||||
* to `null` — importing such a record yields an unfetched reference, not a record
|
||||
* with a URL string masquerading as its snapshot.
|
||||
*/
|
||||
function normalizeDataset(raw: unknown, nowIso: string): Dataset {
|
||||
const r = isPlainObject(raw) ? raw : {};
|
||||
@@ -158,13 +163,25 @@ function normalizeDataset(raw: unknown, nowIso: string): Dataset {
|
||||
const created = asNonEmptyString(r.created) ?? nowIso;
|
||||
const modified = asNonEmptyString(r.modified) ?? created;
|
||||
|
||||
const source = (typeof r.source === 'string' ? r.source : 'inline') as DataSource;
|
||||
const legacyUrl = source === 'url' && typeof r.url !== 'string';
|
||||
const url = legacyUrl
|
||||
? typeof r.data === 'string'
|
||||
? r.data
|
||||
: undefined
|
||||
: typeof r.url === 'string'
|
||||
? r.url
|
||||
: undefined;
|
||||
const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
|
||||
|
||||
return {
|
||||
id: typeof r.id === 'number' ? r.id : Number(r.id) || 0,
|
||||
version: CURRENT_DATASET_VERSION,
|
||||
name: typeof r.name === 'string' ? r.name : 'Untitled',
|
||||
data: r.data,
|
||||
data: legacyUrl ? null : r.data,
|
||||
format: (typeof r.format === 'string' ? r.format : 'json') as DataFormat,
|
||||
source: (typeof r.source === 'string' ? r.source : 'inline') as DataSource,
|
||||
source,
|
||||
...(source === 'url' ? { url, fetchedAt } : {}),
|
||||
comment: typeof r.comment === 'string' ? r.comment : '',
|
||||
rowCount: typeof r.rowCount === 'number' ? r.rowCount : null,
|
||||
columnCount: typeof r.columnCount === 'number' ? r.columnCount : null,
|
||||
|
||||
@@ -111,7 +111,16 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
|
||||
format: 'topojson',
|
||||
source: 'inline',
|
||||
},
|
||||
{ name: 'UrlDs', data: 'https://x/y.csv', format: 'csv', source: 'url' },
|
||||
// A fetched URL snapshot carries its payload in `data` (like inline); an
|
||||
// unfetched reference has `data: null` and only its `url`.
|
||||
{
|
||||
name: 'UrlFetchedDs',
|
||||
data: 'a,b\n1,2',
|
||||
url: 'https://x/y.csv',
|
||||
format: 'csv',
|
||||
source: 'url',
|
||||
},
|
||||
{ name: 'UrlUnfetchedDs', data: null, url: 'https://x/y.csv', format: 'csv', source: 'url' },
|
||||
];
|
||||
|
||||
test('inline JSON → values inlined', () => {
|
||||
@@ -140,8 +149,13 @@ describe('prepareSpecForRender — dataset resolution (spec §04 Rendering Contr
|
||||
});
|
||||
});
|
||||
|
||||
test('URL → url reference tagged with the dataset format', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlDs' } }, { datasets });
|
||||
test('fetched URL snapshot → its payload inlined, tagged with the format (like inline)', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlFetchedDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ values: 'a,b\n1,2', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
test('unfetched URL reference → live url fallback tagged with the format', () => {
|
||||
const out = prepareSpecForRender({ data: { name: 'UrlUnfetchedDs' } }, { datasets });
|
||||
expect(out.data).toEqual({ url: 'https://x/y.csv', format: { type: 'csv' } });
|
||||
});
|
||||
|
||||
|
||||
+11
-2
@@ -42,6 +42,8 @@ export interface ResolvableDataset {
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** For `source = url`: the remote address, used only as the unfetched fallback. */
|
||||
url?: string;
|
||||
}
|
||||
|
||||
/** Thrown when a spec references a library dataset name that does not exist. */
|
||||
@@ -129,13 +131,20 @@ function selfDefinedDatasetNames(spec: unknown): Set<string> {
|
||||
*/
|
||||
function resolvedData(dataset: ResolvableDataset, rest: SpecNode): SpecNode {
|
||||
const restFormat = isSpecNode(rest.format) ? rest.format : {};
|
||||
if (dataset.source === 'url') {
|
||||
|
||||
// An unfetched URL reference (a legacy record, or one whose snapshot fetch never
|
||||
// succeeded) has no local data — fall back to a live Vega-Lite URL fetch so it
|
||||
// still renders until a Refresh snapshots it (snapshot model; spec §04 step 1).
|
||||
if (dataset.source === 'url' && dataset.data == null) {
|
||||
return {
|
||||
...rest,
|
||||
url: typeof dataset.data === 'string' ? dataset.data : (JSON.stringify(dataset.data) ?? ''),
|
||||
url: typeof dataset.url === 'string' ? dataset.url : '',
|
||||
format: { ...restFormat, type: dataset.format },
|
||||
};
|
||||
}
|
||||
|
||||
// Inline data and fetched URL snapshots resolve identically: the payload is in
|
||||
// `data`, shaped by format.
|
||||
switch (dataset.format) {
|
||||
case 'json':
|
||||
return { ...rest, values: dataset.data };
|
||||
|
||||
Reference in New Issue
Block a user