mirror of
https://github.com/olehomelchenko/astrolabe.git
synced 2026-08-08 02:02:33 +00:00
Add dataset library, extract-to-dataset, and render-time reference resolution
This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Dataset — a named, reusable data source snippets reference by name
|
||||
* (spec §09B → Dataset; spec §05 → Datasets).
|
||||
*
|
||||
* Portable core: no browser APIs, no React. Defines the record shape, the current
|
||||
* record schema version, the by-name reference object (spec §05 → Copy Reference),
|
||||
* a simple delimited-text parser, the profiling orchestration, and a factory that
|
||||
* stamps timestamps/version and fills the derived summary fields.
|
||||
*
|
||||
* A dataset has one of two **sources** — `inline` (data stored in the record) or
|
||||
* `url` (only the link is stored, fetched on demand at render time) — and one of
|
||||
* four **formats** (reused from format-detection: `json`/`csv`/`tsv`/`topojson`).
|
||||
* The `data` field's shape follows source/format: a URL string for `url`; raw
|
||||
* text for inline CSV/TSV; a parsed value for inline JSON/TopoJSON.
|
||||
*
|
||||
* Only tabular inline data (JSON array-of-objects, CSV, TSV) is profiled; URL and
|
||||
* non-tabular data get an N/A profile but are still sized (see profile.ts).
|
||||
*/
|
||||
|
||||
import type { DataFormat } from './format-detection';
|
||||
import { profileData, type DatasetProfile } from './profile';
|
||||
import type { ColumnType } from './type-inference';
|
||||
|
||||
/** Current schema version for a Dataset record (read-time migration target). */
|
||||
export const CURRENT_DATASET_VERSION = 1;
|
||||
|
||||
/** Where a dataset's data lives: embedded in the record, or fetched from a URL. */
|
||||
export type DataSource = 'inline' | 'url';
|
||||
|
||||
export interface Dataset {
|
||||
/** Unique numeric identifier. */
|
||||
id: number;
|
||||
/** Record schema version, for read-time migration. */
|
||||
version: number;
|
||||
/** Unique, human-readable name; the key snippets reference via `datasetRefs`. */
|
||||
name: string;
|
||||
/**
|
||||
* The payload. For `source = url`: the URL string. For `source = inline`: the
|
||||
* raw CSV/TSV text, or the parsed JSON/TopoJSON value.
|
||||
*/
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** Free-form user note about the dataset. */
|
||||
comment: string;
|
||||
/** Data rows, or `null` when N/A (URL / non-tabular). */
|
||||
rowCount: number | null;
|
||||
/** Columns, or `null` when N/A. */
|
||||
columnCount: number | null;
|
||||
/** Column names, in order. */
|
||||
columns: string[];
|
||||
/** Per-column inferred type. */
|
||||
columnTypes: Array<{ name: string; type: ColumnType }>;
|
||||
/** Approximate payload size in bytes. */
|
||||
size: number;
|
||||
/** ISO timestamp — when first added. */
|
||||
created: string;
|
||||
/** ISO timestamp — when last changed. */
|
||||
modified: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The by-name reference object copied into a spec via "Copy Reference"
|
||||
* (spec §05 → Actions): `{ "data": { "name": "MyDataset" } }`.
|
||||
*/
|
||||
export function datasetReference(name: string): { data: { name: string } } {
|
||||
return { data: { name } };
|
||||
}
|
||||
|
||||
/** UTF-8 byte length of a string (`TextEncoder` is a platform global, not DOM). */
|
||||
function byteLength(str: string): number {
|
||||
return new TextEncoder().encode(str).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize delimited text into rows of string cells, honoring RFC-4180 quoting:
|
||||
* a field wrapped in `"` may contain the delimiter, newlines, and escaped quotes
|
||||
* (`""` → `"`); a quote is only special at the start of a field. This mirrors how
|
||||
* d3-dsv (the parser Vega-Lite uses at render time) reads the same text, so the
|
||||
* profile Astrolabe shows agrees with the field names the chart actually sees.
|
||||
* Whitespace is NOT trimmed — like d3, the cell is taken verbatim (type inference
|
||||
* trims internally when classifying, so numeric columns still detect).
|
||||
*/
|
||||
function tokenizeDelimited(text: string, delimiter: string): string[][] {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
let inQuotes = false;
|
||||
let started = false; // any character seen for the current record?
|
||||
let i = text.charCodeAt(0) === 0xfeff ? 1 : 0; // skip a leading BOM
|
||||
const n = text.length;
|
||||
|
||||
const endField = () => {
|
||||
row.push(field);
|
||||
field = '';
|
||||
};
|
||||
const endRow = () => {
|
||||
endField();
|
||||
rows.push(row);
|
||||
row = [];
|
||||
started = false;
|
||||
};
|
||||
|
||||
while (i < n) {
|
||||
const c = text[i];
|
||||
if (inQuotes) {
|
||||
if (c === '"') {
|
||||
if (text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
inQuotes = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
field += c;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '"' && field === '') {
|
||||
inQuotes = true;
|
||||
started = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === delimiter) {
|
||||
endField();
|
||||
started = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '\n' || c === '\r') {
|
||||
if (c === '\r' && text[i + 1] === '\n') i++;
|
||||
endRow();
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
field += c;
|
||||
started = true;
|
||||
i++;
|
||||
}
|
||||
// Flush a final field/row only if the last record had content (no phantom row
|
||||
// from a trailing newline).
|
||||
if (started || field !== '' || row.length > 0) endRow();
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse delimited text into rows-of-objects (header row → keys), RFC-4180 quoting
|
||||
* aware (see `tokenizeDelimited`). The first record is the header; each later
|
||||
* record is zipped header→cell. A record with fewer cells than the header leaves
|
||||
* the missing columns `undefined`; extra cells beyond the header are ignored.
|
||||
* Fully-empty records (e.g. a blank line) are skipped.
|
||||
*/
|
||||
export function parseDelimited(
|
||||
text: string,
|
||||
format: 'csv' | 'tsv',
|
||||
): Array<Record<string, unknown>> {
|
||||
const delimiter = format === 'tsv' ? '\t' : ',';
|
||||
const records = tokenizeDelimited(text, delimiter);
|
||||
if (records.length < 2) return [];
|
||||
|
||||
const header = records[0];
|
||||
const rows: Array<Record<string, unknown>> = [];
|
||||
for (let i = 1; i < records.length; i++) {
|
||||
const cells = records[i];
|
||||
if (cells.length === 1 && cells[0] === '') continue; // blank record
|
||||
const row: Record<string, unknown> = {};
|
||||
for (let c = 0; c < header.length; c++) {
|
||||
row[header[c]] = c < cells.length ? cells[c] : undefined;
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** A JSON value is tabular when it is a non-empty array of plain objects. */
|
||||
function asObjectRows(value: unknown): Array<Record<string, unknown>> | null {
|
||||
if (!Array.isArray(value) || value.length === 0) return null;
|
||||
const allObjects = value.every((v) => v !== null && typeof v === 'object' && !Array.isArray(v));
|
||||
return allObjects ? (value as Array<Record<string, unknown>>) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate profiling for a dataset payload: compute `size` (always), decide
|
||||
* tabular vs N/A by source/format, and delegate to `profileData`.
|
||||
*
|
||||
* - `url` (any format) → N/A profile; size = byte length of the URL string.
|
||||
* - inline `json` → rows when a non-empty array of objects, else N/A.
|
||||
* - inline `topojson` → N/A (non-tabular).
|
||||
* - inline `csv` / `tsv` → `parseDelimited` rows.
|
||||
*
|
||||
* `size` is the UTF-8 byte length of the raw string for csv/tsv/url, or of
|
||||
* `JSON.stringify(data)` for json/topojson.
|
||||
*/
|
||||
export function computeDatasetProfile(
|
||||
data: unknown,
|
||||
format: DataFormat,
|
||||
source: DataSource,
|
||||
): DatasetProfile {
|
||||
if (source === 'url') {
|
||||
const url = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
|
||||
return profileData(null, byteLength(url));
|
||||
}
|
||||
|
||||
switch (format) {
|
||||
case 'csv':
|
||||
case 'tsv': {
|
||||
const text = typeof data === 'string' ? data : (JSON.stringify(data) ?? '');
|
||||
return profileData(parseDelimited(text, format), byteLength(text));
|
||||
}
|
||||
case 'json': {
|
||||
const size = byteLength(JSON.stringify(data) ?? '');
|
||||
return profileData(asObjectRows(data), size);
|
||||
}
|
||||
case 'topojson':
|
||||
default: {
|
||||
const size = byteLength(JSON.stringify(data) ?? '');
|
||||
return profileData(null, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateDatasetOptions {
|
||||
/** The dataset name (uniqueness is enforced upstream — see naming.ts). */
|
||||
name: string;
|
||||
/** The payload, shaped per source/format (see `Dataset.data`). */
|
||||
data: unknown;
|
||||
/** One of `json`, `csv`, `tsv`, `topojson`. */
|
||||
format: DataFormat;
|
||||
/** One of `inline` or `url`. */
|
||||
source: DataSource;
|
||||
/** Optional free-form note. */
|
||||
comment?: string;
|
||||
/** Clock injection for deterministic tests; defaults to the current time. */
|
||||
now?: Date;
|
||||
/** Id injection for deterministic tests; defaults to `Date.now()`. */
|
||||
id?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Dataset: stamps version/timestamps and runs `computeDatasetProfile` to
|
||||
* fill the derived summary fields. `id` defaults to `Date.now()` (numeric;
|
||||
* collision-prone for batch creation — see the DatasetStore.add TODO) and is
|
||||
* injectable for tests.
|
||||
*/
|
||||
export function createDataset(options: CreateDatasetOptions): Dataset {
|
||||
const now = options.now ?? new Date();
|
||||
const iso = now.toISOString();
|
||||
const profile = computeDatasetProfile(options.data, options.format, options.source);
|
||||
|
||||
return {
|
||||
id: options.id ?? Date.now(),
|
||||
version: CURRENT_DATASET_VERSION,
|
||||
name: options.name,
|
||||
data: options.data,
|
||||
format: options.format,
|
||||
source: options.source,
|
||||
comment: options.comment ?? '',
|
||||
rowCount: profile.rowCount,
|
||||
columnCount: profile.columnCount,
|
||||
columns: profile.columns,
|
||||
columnTypes: profile.columnTypes,
|
||||
size: profile.size,
|
||||
created: iso,
|
||||
modified: iso,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user