Snapshot URL datasets locally on add; preview tabular data as a table

src/app/infrastructure/dataset-migrations.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, test } from 'vitest';
+import { CURRENT_DATASET_VERSION } from '@core/dataset';
+import { migrateDataset } from './dataset-migrations';
+
+describe('migrateDataset', () => {
+  test('stamps the current version and fills missing fields with defaults', () => {
+    const d = migrateDataset({ id: 7, data: 'a,b\n1,2', format: 'csv', source: 'inline' });
+
+    expect(d.id).toBe(7);
+    expect(d.version).toBe(CURRENT_DATASET_VERSION);
+    expect(d.name).toBe('Untitled');
+    expect(d.comment).toBe('');
+    expect(d.columns).toEqual([]);
+    expect(d.columnStats).toEqual([]);
+  });
+
+  test('an inline record carries no url/fetchedAt keys', () => {
+    const d = migrateDataset({ id: 1, data: [{ a: 1 }], format: 'json', source: 'inline' });
+    expect(d.source).toBe('inline');
+    expect('url' in d).toBe(false);
+    expect('fetchedAt' in d).toBe(false);
+  });
+
+  test('v1→v2: a legacy URL record moves its address out of data into url', () => {
+    // Pre-v2 shape: the URL lived in `data`, there was no `url`/`fetchedAt`, and the
+    // profile was N/A. It becomes an unfetched reference: url set, data cleared.
+    const d = migrateDataset({
+      id: 2,
+      data: 'https://example.com/data.csv',
+      format: 'csv',
+      source: 'url',
+      rowCount: null,
+      columns: [],
+    });
+
+    expect(d.version).toBe(CURRENT_DATASET_VERSION);
+    expect(d.source).toBe('url');
+    expect(d.url).toBe('https://example.com/data.csv');
+    expect(d.data).toBeNull();
+    expect(d.fetchedAt).toBeNull();
+    expect(d.rowCount).toBeNull();
+  });
+
+  test('a v2 URL snapshot is preserved (data, url, and fetchedAt all kept)', () => {
+    const d = migrateDataset({
+      id: 3,
+      version: 2,
+      data: 'a,b\n1,2',
+      url: 'https://example.com/data.csv',
+      fetchedAt: '2026-06-10T00:00:00.000Z',
+      format: 'csv',
+      source: 'url',
+      rowCount: 1,
+      columns: ['a', 'b'],
+    });
+
+    expect(d.data).toBe('a,b\n1,2');
+    expect(d.url).toBe('https://example.com/data.csv');
+    expect(d.fetchedAt).toBe('2026-06-10T00:00:00.000Z');
+    expect(d.rowCount).toBe(1);
+  });
+
+  test('preserves unknown fields so a newer build round-trips without loss', () => {
+    const d = migrateDataset({
+      id: 1,
+      data: [],
+      format: 'json',
+      source: 'inline',
+      futureField: 42,
+    });
+    expect((d as unknown as Record<string, unknown>).futureField).toBe(42);
+  });
+});

src/app/infrastructure/dataset-migrations.ts

@@ -30,14 +30,33 @@ function asCount(v: unknown): number | null {
 /** Upgrade a raw stored record to the current Dataset shape. */
 export function migrateDataset(raw: unknown): Dataset {
   const r = { ...(raw as Record<string, unknown>) };
+  const source = asSource(r.source);
+
+  // v1→v2: a URL dataset used to store its address in `data`; it now stores the
+  // fetched snapshot in `data` and keeps the address in `url` (+ a `fetchedAt`).
+  // A legacy record (url source, no `url` field) becomes an *unfetched* reference:
+  // move the address to `url` and clear `data`. The N/A profile carries over as-is;
+  // rendering falls back to the live URL until a Refresh fetches + profiles it.
+  const legacyUrl = source === 'url' && typeof r.url !== 'string';
+  const url = legacyUrl
+    ? typeof r.data === 'string'
+      ? r.data
+      : undefined
+    : typeof r.url === 'string'
+      ? r.url
+      : undefined;
+  const fetchedAt = legacyUrl ? null : typeof r.fetchedAt === 'string' ? r.fetchedAt : null;
+
   return {
     ...r,
     id: typeof r.id === 'number' ? r.id : Number(r.id),
     version: CURRENT_DATASET_VERSION,
     name: typeof r.name === 'string' ? r.name : 'Untitled',
-    data: r.data,
+    data: legacyUrl ? null : r.data,
     format: asFormat(r.format),
-    source: asSource(r.source),
+    source,
+    // Only URL datasets carry url/fetchedAt; inline records stay clean.
+    ...(source === 'url' ? { url, fetchedAt } : {}),
     comment: typeof r.comment === 'string' ? r.comment : '',
     rowCount: asCount(r.rowCount),
     columnCount: asCount(r.columnCount),

src/app/infrastructure/remote-data.test.ts

@@ -0,0 +1,94 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { fetchRemoteData, RemoteFetchError } from './remote-data';
+
+/** A minimal Response stand-in for the bits the adapter reads. */
+function fakeResponse(opts: {
+  ok: boolean;
+  status?: number;
+  statusText?: string;
+  body?: string;
+  contentType?: string | null;
+}): Response {
+  return {
+    ok: opts.ok,
+    status: opts.status ?? (opts.ok ? 200 : 500),
+    statusText: opts.statusText ?? '',
+    text: () => Promise.resolve(opts.body ?? ''),
+    headers: {
+      get: (k: string) => (k.toLowerCase() === 'content-type' ? (opts.contentType ?? null) : null),
+    },
+  } as unknown as Response;
+}
+
+afterEach(() => vi.unstubAllGlobals());
+
+describe('fetchRemoteData', () => {
+  test('returns the body and content-type on success', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() =>
+        Promise.resolve(fakeResponse({ ok: true, body: 'a,b\n1,2', contentType: 'text/csv' })),
+      ),
+    );
+    const result = await fetchRemoteData('https://example.com/data.csv');
+    expect(result.text).toBe('a,b\n1,2');
+    expect(result.contentType).toBe('text/csv');
+  });
+
+  test('classifies a non-2xx response as http with its status', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() =>
+        Promise.resolve(fakeResponse({ ok: false, status: 404, statusText: 'Not Found' })),
+      ),
+    );
+    await expect(fetchRemoteData('https://example.com/missing.csv')).rejects.toMatchObject({
+      reason: 'http',
+      status: 404,
+    });
+  });
+
+  test('classifies a rejected fetch (CORS / offline) as network', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
+    );
+    await expect(fetchRemoteData('https://blocked.example/data.csv')).rejects.toMatchObject({
+      reason: 'network',
+    });
+  });
+
+  test('classifies an empty body as empty', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() => Promise.resolve(fakeResponse({ ok: true, body: '   \n' }))),
+    );
+    await expect(fetchRemoteData('https://example.com/blank.csv')).rejects.toMatchObject({
+      reason: 'empty',
+    });
+  });
+
+  test('classifies an aborted fetch as timeout', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() => {
+        const e = new Error('aborted');
+        e.name = 'AbortError';
+        return Promise.reject(e);
+      }),
+    );
+    await expect(
+      fetchRemoteData('https://slow.example/data.csv', { timeoutMs: 1 }),
+    ).rejects.toMatchObject({
+      reason: 'timeout',
+    });
+  });
+
+  test('the thrown error is a RemoteFetchError', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() => Promise.reject(new TypeError('Failed to fetch'))),
+    );
+    await expect(fetchRemoteData('https://x.example')).rejects.toBeInstanceOf(RemoteFetchError);
+  });
+});

src/app/infrastructure/remote-data.ts

@@ -0,0 +1,99 @@
+/**
+ * Remote-data fetch adapter (snapshot model — spec §05 → URL datasets).
+ *
+ * The ONLY place Astrolabe touches the network. Adding a URL dataset fetches the
+ * resource once here; core then profiles the returned text exactly like inline
+ * data (see dataset.ts). Keeping the fetch in infrastructure preserves the rule
+ * that `src/core` stays pure and that the rest of the app never calls `fetch`
+ * directly (AGENTS → Architecture).
+ *
+ * Failures are classified into a small set of machine-readable `reason`s and
+ * thrown as `RemoteFetchError`; the user-facing wording (and the "paste inline
+ * instead" fallback) lives in the UI layer, where it is council-reviewed — the
+ * same split as `storage-errors`. A browser cross-origin block and an
+ * offline/DNS failure are indistinguishable here — both reject with a `TypeError`
+ * and no status — so they share the `network` reason and the UI hedges the cause.
+ */
+
+/** Why a remote fetch failed, in terms the UI maps to copy + a recovery path. */
+export type RemoteFetchReason = 'network' | 'http' | 'empty' | 'timeout';
+
+/** A classified remote-fetch failure. `status` is set only for `http`. */
+export class RemoteFetchError extends Error {
+  readonly reason: RemoteFetchReason;
+  readonly status?: number;
+
+  constructor(reason: RemoteFetchReason, message: string, status?: number) {
+    super(message);
+    this.name = 'RemoteFetchError';
+    this.reason = reason;
+    this.status = status;
+  }
+}
+
+/** A successful fetch: the raw body plus a weak format hint from the server. */
+export interface RemoteData {
+  /** The response body, ready for format detection + profiling. */
+  text: string;
+  /** The server's `Content-Type`, when provided — a tertiary format hint. */
+  contentType: string | null;
+}
+
+/** Default ceiling on a single fetch before it is aborted as a timeout. */
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+export interface FetchRemoteDataOptions {
+  /** Abort the fetch after this many ms (default 30 s). */
+  timeoutMs?: number;
+}
+
+function describeNetworkError(err: unknown): string {
+  const reason = err instanceof Error ? err.message : String(err);
+  return `Couldn't reach the URL (${reason}).`;
+}
+
+/**
+ * Fetch a remote dataset resource once and return its raw body. Throws
+ * `RemoteFetchError` on any failure — the caller distinguishes recovery by
+ * `reason`. The body is returned verbatim; format detection and profiling happen
+ * downstream so this adapter stays free of core logic.
+ */
+export async function fetchRemoteData(
+  url: string,
+  options: FetchRemoteDataOptions = {},
+): Promise<RemoteData> {
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  let response: Response;
+  try {
+    response = await fetch(url, { redirect: 'follow', signal: controller.signal });
+  } catch (err) {
+    // A timeout aborts via the controller (AbortError); everything else here is a
+    // CORS block, offline, DNS failure, or malformed URL — all opaque TypeErrors.
+    if ((err as { name?: string }).name === 'AbortError') {
+      throw new RemoteFetchError('timeout', `The URL took longer than ${timeoutMs} ms to respond.`);
+    }
+    throw new RemoteFetchError('network', describeNetworkError(err));
+  } finally {
+    clearTimeout(timer);
+  }
+
+  if (!response.ok) {
+    throw new RemoteFetchError(
+      'http',
+      `The server responded ${response.status} ${response.statusText}.`.trim(),
+      response.status,
+    );
+  }
+
+  const text = await response.text();
+  if (text.trim() === '') {
+    throw new RemoteFetchError('empty', 'The URL returned no data.');
+  }
+
+  // TODO: guard very large responses (cap or warn) before snapshotting into
+  // IndexedDB — a multi-hundred-MB file would load fully into memory + storage.
+  return { text, contentType: response.headers.get('content-type') };
+}