diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 0e4767f1..00000000 --- a/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -.git -.user -cli/ -docs/ -dpkit/ -site/ -**/node_modules -**/build -**/compile -**/coverage diff --git a/.github/workflows/general.yaml b/.github/workflows/general.yaml index 8e99d93c..3035cec9 100644 --- a/.github/workflows/general.yaml +++ b/.github/workflows/general.yaml @@ -26,10 +26,8 @@ jobs: node-version-file: package.json - name: Install Dependencies run: pnpm install:ci - - name: Build Packages - run: pnpm build - - name: Compile Binaries - run: pnpm compile + - name: Build Terminal + run: pnpm -F terminal build - name: Upload Binaries uses: actions/upload-artifact@v4 with: @@ -54,9 +52,7 @@ jobs: node-version-file: package.json - name: Install Dependencies run: pnpm install:ci - - name: Build Packages - run: pnpm build - - name: Test Packages + - name: Run Tests run: pnpm test env: DPKIT_MYSQL_URL: mysql://test:test@localhost:3306/test @@ -111,9 +107,7 @@ jobs: node-version-file: package.json - name: Install Dependencies run: pnpm install:ci - - name: Build Packages - run: pnpm build - - name: Test Packages + - name: Run Tests run: pnpm test - name: Download Binaries uses: actions/download-artifact@v5 @@ -142,9 +136,7 @@ jobs: node-version-file: package.json - name: Install Dependencies run: pnpm install:ci - - name: Build Packages - run: pnpm build - - name: Test Packages + - name: Run Tests run: pnpm test:win - name: Download Binaries uses: actions/download-artifact@v5 @@ -216,16 +208,8 @@ jobs: run: pnpm install:ci - name: Set Version run: pnpm version:ci ${{ needs.release-draft.outputs.version }} - - name: Build Packages - run: pnpm build - - name: Compile Packages - run: pnpm compile - - name: Login Publisher - run: pnpm login:ci - - name: Publish Packages - run: pnpm publish:ci - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Build Terminal + run: pnpm -F terminal build - name: Publish Release uses: softprops/action-gh-release@v2 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81fee01b..b6fc1317 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,22 +1,18 @@ ---- -title: Contributing -sidebar: - order: 2 ---- +# Contributing + Thank you for your interest in contributing to dpkit! This document provides guidelines and instructions for contributing to this project. ## Project Overview Project is a monorepo with the following packages: -- `@dpkit/metadata`: Core metadata functionality -- `@dpkit/dataset`: File-related functionality -- `@dpkit/table`: Table-related functionality -- `@dpkit/`: Domain-specific functionality -- `@dpkit/library`: All-in-one package that re-exports all functionality -- `@dpkit/terminal`: Terminal interface for running tasks -- `@dpkit/website`: Website-related functionality -- `dpkit`: Meta-package that re-exports the underlying functionality +- `@dpkit/desktop`: Desktop application +- `@dpkit/engine`: OpenAPI-based service +- `@dpkit/library`: Node library +- `@dpkit/mobile`: Mobile application +- `@dpkit/terminal`: Terminal application +- `@dpkit/toolbox`: Visual toolbox +- `@dpkit/website`: Web application ## Development Environment @@ -82,7 +78,7 @@ Tests are located in `__spec__` directories and use Vitest: - **Run a Specific Test**: ```bash - pnpm exec vitest run core/actions/__spec__/findTask.ts + pnpm exec vitest run library/aspect/__spec__/findTask.ts ``` ### Dependencies diff --git a/audio/README.md b/audio/README.md deleted file mode 100644 index aac0d22c..00000000 --- a/audio/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/audio - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/audio/package.json b/audio/package.json deleted file mode 100644 index 51d48a2a..00000000 --- a/audio/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "@dpkit/audio", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "audio" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "@dpkit/dataset": "workspace:*" - } -} diff --git a/audio/tsconfig.json b/audio/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/audio/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/audio/typedoc.json b/audio/typedoc.json deleted file mode 100644 index f8e49f3a..00000000 --- a/audio/typedoc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "entryPoints": ["index.ts"], - "skipErrorChecking": true -} diff --git a/database/README.md b/database/README.md deleted file mode 100644 index bb3f51e2..00000000 --- a/database/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/database - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/database/adapters/base.ts b/database/adapters/base.ts deleted file mode 100644 index ea4ddde4..00000000 --- a/database/adapters/base.ts +++ /dev/null @@ -1,97 +0,0 @@ -import type { Field, FieldType, Schema } from "@dpkit/metadata" -import type { Dialect } from "kysely" -import { Kysely } from "kysely" -import { LRUCache } from "lru-cache" -import type { DatabaseField, DatabaseType } from "../field/index.ts" -import type { DatabaseSchema } from "../schema/index.ts" - -// We cache database connections (only works in serverfull environments) -const databases = new LRUCache>({ - dispose: database => database.destroy(), - max: 10, -}) - -export abstract class BaseAdapter { - abstract get nativeTypes(): FieldType[] - - async connectDatabase(path: string, options?: { create?: boolean }) { - const cachedDatabase = databases.get(path) - if (cachedDatabase) { - return cachedDatabase - } - - const dialect = await this.createDialect(path, options) - const database = new Kysely({ dialect }) - databases.set(path, new Kysely({ dialect })) - - return database - } - - abstract createDialect( - path: string, - options?: { create?: boolean }, - ): Promise - - normalizeSchema(databaseSchema: DatabaseSchema) { - const schema: Schema = { fields: [] } - - for (const databaseField of databaseSchema.columns) { - schema.fields.push(this.normalizeField(databaseField)) - } - - return schema - } - - normalizeField(databaseField: DatabaseField) { - const field: Field = { - name: databaseField.name, - type: this.normalizeType(databaseField.dataType), - } - - if (!databaseField.isNullable) { - field.constraints ??= {} - field.constraints.required = true - } - - if (databaseField.comment) { - field.description = databaseField.comment - } - - return field - } - - abstract normalizeType(databaseType: DatabaseType): Field["type"] - - denormalizeSchema(schema: Schema, tableName: string): DatabaseSchema { - const databaseSchema: DatabaseSchema = { - name: tableName, - columns: [], - isView: false, - } - - for (const field of schema.fields) { - databaseSchema.columns.push(this.denormalizeField(field)) - } - - if (schema.primaryKey) { - databaseSchema.primaryKey = schema.primaryKey - } - - return databaseSchema - } - - denormalizeField(field: Field): DatabaseField { - const databaseField: DatabaseField = { - name: field.name, - dataType: this.denormalizeType(field.type), - isNullable: !field.constraints?.required, - comment: field.description, - isAutoIncrementing: false, - hasDefaultValue: false, - } - - return databaseField - } - - abstract denormalizeType(fieldType: Field["type"]): DatabaseType -} diff --git a/database/adapters/create.ts b/database/adapters/create.ts deleted file mode 100644 index 68a2f9a2..00000000 --- a/database/adapters/create.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { DatabaseFormat } from "../resource/index.ts" -import { MysqlAdapter } from "./mysql.ts" -import { PostgresqlAdapter } from "./postgresql.ts" -import { SqliteAdapter } from "./sqlite.ts" - -// TODO: Enable SQLite support - -export function createAdapter(format: DatabaseFormat) { - switch (format) { - case "postgresql": - return new PostgresqlAdapter() - case "mysql": - return new MysqlAdapter() - case "sqlite": - return new SqliteAdapter() - default: - throw new Error(`Unsupported database format: "${format}"`) - } -} diff --git a/database/adapters/mysql.spec.ts b/database/adapters/mysql.spec.ts deleted file mode 100644 index b0800279..00000000 --- a/database/adapters/mysql.spec.ts +++ /dev/null @@ -1,192 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromDatabase } from "../package/index.ts" -import { inferDatabaseSchema } from "../schema/index.ts" -import { loadDatabaseTable, saveDatabaseTable } from "../table/index.ts" -import { createAdapter } from "./create.ts" - -useRecording() - -const path = process.env.DPKIT_MYSQL_URL - -// Vitest runs in-file tests sequentially so we can use the same table -const dialect = { table: "dpkit" } -const record1 = { id: 1, name: "english" } -const record2 = { id: 2, name: "中文" } - -describe.skipIf(!path)("MysqlAdapter", () => { - if (!path) return - - it("should infer schema", async () => { - const source = pl - .DataFrame([ - pl.Series("string", ["string"], pl.Utf8), - pl.Series("integer", [1], pl.Int32), - pl.Series("number", [1.1], pl.Float64), - ]) - .lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "mysql", - overwrite: true, - }) - - const schema = await inferDatabaseSchema({ - path, - dialect, - format: "mysql", - }) - - expect(schema).toEqual({ - fields: [ - { name: "string", type: "string" }, - { name: "integer", type: "integer" }, - { name: "number", type: "number" }, - ], - }) - }) - - it("should save/load table", async () => { - const source = pl.DataFrame([record1, record2]).lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "mysql", - overwrite: true, - }) - - const target = await loadDatabaseTable({ - path, - dialect, - format: "mysql", - }) - - expect((await target.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should save/load table with various data types", async () => { - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "mysql", - overwrite: true, - fieldTypes: { - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadDatabaseTable( - { path, dialect, format: "mysql" }, - { denormalized: true }, - ) - - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: 1, - date: "2025-01-01", - datetime: new Date(Date.UTC(2025, 0, 1)), - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: "1.0,2.0,3.0", - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) - - it("should load package from database", async () => { - const adapter = createAdapter("mysql") - const database = await adapter.connectDatabase(path) - - await database.schema.dropTable("table1").ifExists().execute() - await database.schema - .createTable("table1") - .ifNotExists() - .addColumn("id", "integer", column => column.notNull()) - .addColumn("name", "text") - .execute() - - await database.schema.dropTable("table2").ifExists().execute() - await database.schema - .createTable("table2") - .ifNotExists() - .addColumn("id", "integer", column => column.notNull()) - .addColumn("number", "numeric") - .addColumn("datetime", "datetime") - .execute() - - const datapackage = await loadPackageFromDatabase(path, { - format: "mysql", - includeTables: ["table1", "table2"], - }) - - expect(datapackage).toEqual({ - resources: [ - { - path, - name: "table1", - format: "mysql", - dialect: { table: "table1" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "name", type: "string" }, - ], - }, - }, - { - path, - name: "table2", - format: "mysql", - dialect: { table: "table2" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "number", type: "number" }, - { name: "datetime", type: "datetime" }, - ], - }, - }, - ], - }) - }) -}) diff --git a/database/adapters/mysql.ts b/database/adapters/mysql.ts deleted file mode 100644 index 04088192..00000000 --- a/database/adapters/mysql.ts +++ /dev/null @@ -1,96 +0,0 @@ -import type { FieldType } from "@dpkit/metadata" -import { MysqlDialect } from "kysely" -import { createPool } from "mysql2" -import type { DatabaseType } from "../field/index.ts" -import { BaseAdapter } from "./base.ts" - -// TODO: Support more native types - -export class MysqlAdapter extends BaseAdapter { - nativeTypes = [ - "boolean", - "datetime", - "integer", - "number", - "string", - "year", - ] satisfies FieldType[] - - async createDialect(path: string) { - return new MysqlDialect({ - pool: createPool({ uri: path }), - }) - } - - normalizeType(databaseType: DatabaseType): FieldType { - switch (databaseType.toLowerCase()) { - case "tinyint": - case "smallint": - case "mediumint": - case "int": - case "integer": - case "bigint": - return "integer" - case "decimal": - case "numeric": - case "float": - case "double": - case "real": - return "number" - case "bit": - case "bool": - case "boolean": - return "boolean" - case "char": - case "varchar": - case "tinytext": - case "text": - case "mediumtext": - case "longtext": - case "enum": - case "set": - return "string" - case "date": - return "date" - case "time": - return "time" - case "datetime": - case "timestamp": - return "datetime" - case "year": - return "year" - case "json": - return "object" - case "geometry": - case "point": - case "linestring": - case "polygon": - case "multipoint": - case "multilinestring": - case "multipolygon": - case "geometrycollection": - return "geojson" - default: - return "string" - } - } - - denormalizeType(fieldType: FieldType): DatabaseType { - switch (fieldType) { - case "boolean": - return "boolean" - case "datetime": - return "datetime" - case "integer": - return "integer" - case "number": - return "double precision" - case "string": - return "text" - case "year": - return "integer" - default: - return "text" - } - } -} diff --git a/database/adapters/postgresql.spec.ts b/database/adapters/postgresql.spec.ts deleted file mode 100644 index c2f3b357..00000000 --- a/database/adapters/postgresql.spec.ts +++ /dev/null @@ -1,192 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromDatabase } from "../package/index.ts" -import { inferDatabaseSchema } from "../schema/index.ts" -import { loadDatabaseTable, saveDatabaseTable } from "../table/index.ts" -import { createAdapter } from "./create.ts" - -useRecording() - -const path = process.env.DPKIT_POSTGRESQL_URL - -// Vitest runs in-file tests sequentially so we can use the same table -const dialect = { table: "dpkit" } -const record1 = { id: 1, name: "english" } -const record2 = { id: 2, name: "中文" } - -describe.skipIf(!path)("PostgresqlAdapter", () => { - if (!path) return - - it("should infer schema", async () => { - const source = pl - .DataFrame([ - pl.Series("string", ["string"], pl.Utf8), - pl.Series("integer", [1], pl.Int32), - pl.Series("number", [1.1], pl.Float64), - ]) - .lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "postgresql", - overwrite: true, - }) - - const schema = await inferDatabaseSchema({ - path, - dialect, - format: "postgresql", - }) - - expect(schema).toEqual({ - fields: [ - { name: "string", type: "string" }, - { name: "integer", type: "integer" }, - { name: "number", type: "number" }, - ], - }) - }) - - it("should save/load table", async () => { - const source = pl.DataFrame([record1, record2]).lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "postgresql", - overwrite: true, - }) - - const target = await loadDatabaseTable({ - path, - dialect, - format: "postgresql", - }) - - expect((await target.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should save/load table with various data types", async () => { - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "postgresql", - overwrite: true, - fieldTypes: { - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadDatabaseTable( - { path, dialect, format: "postgresql" }, - { denormalized: true }, - ) - - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: true, - date: "2025-01-01", - datetime: new Date(Date.UTC(2025, 0, 1)), - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: "1.0,2.0,3.0", - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) - - it("should load package from database", async () => { - const adapter = createAdapter("postgresql") - const database = await adapter.connectDatabase(path) - - await database.schema.dropTable("table1").ifExists().execute() - await database.schema - .createTable("table1") - .ifNotExists() - .addColumn("id", "integer", column => column.notNull()) - .addColumn("name", "text") - .execute() - - await database.schema.dropTable("table2").ifExists().execute() - await database.schema - .createTable("table2") - .ifNotExists() - .addColumn("id", "integer", column => column.notNull()) - .addColumn("number", "numeric") - .addColumn("datetime", "timestamp") - .execute() - - const datapackage = await loadPackageFromDatabase(path, { - format: "postgresql", - includeTables: ["table1", "table2"], - }) - - expect(datapackage).toEqual({ - resources: [ - { - path, - name: "table1", - format: "postgresql", - dialect: { table: "table1" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "name", type: "string" }, - ], - }, - }, - { - path, - name: "table2", - format: "postgresql", - dialect: { table: "table2" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "number", type: "number" }, - { name: "datetime", type: "datetime" }, - ], - }, - }, - ], - }) - }) -}) diff --git a/database/adapters/postgresql.ts b/database/adapters/postgresql.ts deleted file mode 100644 index ff0b5ae9..00000000 --- a/database/adapters/postgresql.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { FieldType } from "@dpkit/metadata" -import { PostgresDialect } from "kysely" -import { Pool } from "pg" -import type { DatabaseType } from "../field/index.ts" -import { BaseAdapter } from "./base.ts" - -// TODO: Support more native types - -export class PostgresqlAdapter extends BaseAdapter { - nativeTypes = [ - "boolean", - "datetime", - "integer", - "number", - "string", - "year", - ] satisfies FieldType[] - - async createDialect(path: string) { - return new PostgresDialect({ - pool: new Pool({ connectionString: path }), - }) - } - - normalizeType(databaseType: DatabaseType): FieldType { - switch (databaseType.toLowerCase()) { - case "smallint": - case "integer": - case "int": - case "int2": - case "int4": - case "int8": - case "bigint": - case "smallserial": - case "serial": - case "bigserial": - return "integer" - case "decimal": - case "numeric": - case "real": - case "float4": - case "double precision": - case "float8": - return "number" - case "boolean": - case "bool": - return "boolean" - case "char": - case "character": - case "varchar": - case "character varying": - case "text": - case "citext": - case "uuid": - return "string" - case "date": - return "date" - case "time": - case "time without time zone": - case "time with time zone": - case "timetz": - return "time" - case "timestamp": - case "timestamp without time zone": - case "timestamp with time zone": - case "timestamptz": - return "datetime" - case "interval": - return "duration" - case "json": - case "jsonb": - return "object" - case "point": - case "line": - case "lseg": - case "box": - case "path": - case "polygon": - case "circle": - case "geometry": - case "geography": - return "geojson" - default: - return "string" - } - } - - denormalizeType(fieldType: FieldType): DatabaseType { - switch (fieldType) { - case "boolean": - return "boolean" - case "datetime": - return "timestamp" - case "integer": - return "integer" - case "number": - return "double precision" - case "string": - return "text" - case "year": - return "integer" - default: - return "text" - } - } -} diff --git a/database/adapters/sqlite.bun.ts b/database/adapters/sqlite.bun.ts deleted file mode 100644 index dbb28c47..00000000 --- a/database/adapters/sqlite.bun.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { BunSqliteDialect } from "kysely-bun-sqlite" - -export async function createBunSqliteDialect(path: string) { - // @ts-ignore - const { Database } = await import("bun:sqlite") - return new BunSqliteDialect({ database: new Database(path) }) -} diff --git a/database/adapters/sqlite.node.ts b/database/adapters/sqlite.node.ts deleted file mode 100644 index 45e29713..00000000 --- a/database/adapters/sqlite.node.ts +++ /dev/null @@ -1,53 +0,0 @@ -// @ts-nocheck -import { buildQueryFn, parseBigInt } from "kysely-generic-sqlite" -import { GenericSqliteDialect } from "kysely-generic-sqlite" -import type { IGenericSqlite } from "kysely-generic-sqlite" - -export async function createNodeSqliteDialect(path: string) { - const { DatabaseSync } = await import("node:sqlite") - - return new GenericSqliteDialect(() => - createSqliteExecutor(new DatabaseSync(path)), - ) -} - -function createSqliteExecutor(db: DatabaseSync): IGenericSqlite { - const getStmt = (sql: string) => { - const stmt = db.prepare(sql) - // We change it from original to use plain numbers - //stmt.setReadBigInts(true) - return stmt - } - - return { - db, - query: buildQueryFn({ - all: (sql, parameters = []) => - getStmt(sql) - .all(...parameters) - // We change it from original to make it work - // (by default it returns object with null prototype which breaks polars) - .map(row => ({ ...row })), - - run: (sql, parameters = []) => { - const { changes, lastInsertRowid } = getStmt(sql).run(...parameters) - return { - insertId: parseBigInt(lastInsertRowid), - numAffectedRows: parseBigInt(changes), - } - }, - }), - close: () => db.close(), - iterator: (isSelect, sql, parameters = []) => { - if (!isSelect) { - throw new Error("Only support select in stream()") - } - return ( - getStmt(sql) - .iterate(...parameters) // We change it from original to make it work - // (by default it returns object with null prototype which breaks polars) - .map(row => ({ ...row })) as any - ) - }, - } -} diff --git a/database/adapters/sqlite.spec.ts b/database/adapters/sqlite.spec.ts deleted file mode 100644 index 448b9b09..00000000 --- a/database/adapters/sqlite.spec.ts +++ /dev/null @@ -1,291 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import type { Package } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromDatabase } from "../package/index.ts" -import { savePackageToDatabase } from "../package/index.ts" -import { inferDatabaseSchema } from "../schema/index.ts" -import { loadDatabaseTable, saveDatabaseTable } from "../table/index.ts" -import { createAdapter } from "./create.ts" - -useRecording() - -const dialect = { table: "dpkit" } -const record1 = { id: 1, name: "english" } -const record2 = { id: 2, name: "中文" } - -// TODO: Enable when libsql@0.6 is fixed -describe("SqliteAdapter", () => { - it("should infer schema", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("string", ["string"], pl.Utf8), - pl.Series("integer", [1], pl.Int32), - pl.Series("number", [1.1], pl.Float64), - ]) - .lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "sqlite", - overwrite: true, - }) - - const schema = await inferDatabaseSchema({ - path, - dialect, - format: "sqlite", - }) - - expect(schema).toEqual({ - fields: [ - { name: "string", type: "string" }, - { name: "integer", type: "integer" }, - { name: "number", type: "number" }, - ], - }) - }) - - it("should save/load table", async () => { - const path = getTempFilePath() - - const source = pl.DataFrame([record1, record2]).lazy() - await saveDatabaseTable(source, { - path, - dialect, - format: "sqlite", - overwrite: true, - }) - - const target = await loadDatabaseTable({ path, dialect, format: "sqlite" }) - expect((await target.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should save/load table with protocol", async () => { - const path = `sqlite://${getTempFilePath()}` - - const source = pl.DataFrame([record1, record2]).lazy() - await saveDatabaseTable(source, { - path, - dialect, - format: "sqlite", - overwrite: true, - }) - - const target = await loadDatabaseTable({ path, dialect, format: "sqlite" }) - expect((await target.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should save/load table with various data types", async () => { - const path = `sqlite://${getTempFilePath()}` - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveDatabaseTable(source, { - path, - dialect, - format: "sqlite", - overwrite: true, - fieldTypes: { - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadDatabaseTable( - { path, dialect, format: "sqlite" }, - { denormalized: true }, - ) - - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: "true", - date: "2025-01-01", - datetime: "2025-01-01T00:00:00", - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: "1.0,2.0,3.0", - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) - - it("should load package from database", async () => { - const path = getTempFilePath() - const adapter = createAdapter("sqlite") - const database = await adapter.connectDatabase(path, { create: true }) - - await database.schema - .createTable("table1") - .addColumn("id", "integer", column => column.notNull()) - .addColumn("name", "text") - .execute() - - await database.schema - .createTable("table2") - .addColumn("id", "integer", column => column.notNull()) - .addColumn("number", "numeric") - .addColumn("boolean", "boolean") - .execute() - - const datapackage = await loadPackageFromDatabase(path, { - format: "sqlite", - }) - - expect(datapackage).toEqual({ - resources: [ - { - path, - name: "table1", - format: "sqlite", - dialect: { table: "table1" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "name", type: "string" }, - ], - }, - }, - { - path, - name: "table2", - format: "sqlite", - dialect: { table: "table2" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "number", type: "number" }, - { name: "boolean", type: "string" }, - ], - }, - }, - ], - }) - }) - - it("should save package to database", async () => { - const path = getTempFilePath() - - const dataPackage: Package = { - resources: [ - { - path: "table1.csv", - name: "table1", - format: "sqlite", - dialect: { table: "table1" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "name", type: "string" }, - ], - }, - }, - { - path: "table2.csv", - name: "table2", - format: "sqlite", - dialect: { table: "table2" }, - schema: { - fields: [ - { name: "id", type: "integer", constraints: { required: true } }, - { name: "number", type: "number" }, - { name: "boolean", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToDatabase(dataPackage, { - target: path, - format: "sqlite", - plugins: [ - { - loadTable: async resource => { - if (resource.name === "table1") { - return pl - .DataFrame([ - pl.Series("id", [1, 2]), - pl.Series("name", ["english", "中文"]), - ]) - .lazy() - } - - if (resource.name === "table2") { - return pl - .DataFrame([ - pl.Series("id", [1, 2]), - pl.Series("number", [1.1, 2.2]), - pl.Series("boolean", ["true", "false"]), - ]) - .lazy() - } - - return undefined - }, - }, - ], - }) - - const adapter = createAdapter("sqlite") - const database = await adapter.connectDatabase(path) - - const records1 = await database.selectFrom("table1").selectAll().execute() - const records2 = await database.selectFrom("table2").selectAll().execute() - - expect(records1).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - - expect(records2).toEqual([ - { id: 1, number: 1.1, boolean: "true" }, - { id: 2, number: 2.2, boolean: "false" }, - ]) - }) - - it("should throw error when loading from non-existent database", async () => { - const path = "non-existent-database.db" - - await expect( - loadDatabaseTable({ path, format: "sqlite", dialect }), - ).rejects.toThrow('Database file "non-existent-database.db" does not exist') - }) -}) diff --git a/database/adapters/sqlite.ts b/database/adapters/sqlite.ts deleted file mode 100644 index c93eb291..00000000 --- a/database/adapters/sqlite.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { isLocalPathExist } from "@dpkit/dataset" -import type { FieldType } from "@dpkit/metadata" -import type { DatabaseType } from "../field/index.ts" -import { BaseAdapter } from "./base.ts" - -// TODO: Currently, the solution is not optimal / hacky -// We need to rebase on proper sqlite dialect when it will be available -// - https://github.com/kysely-org/kysely/issues/1292 -// - https://github.com/oven-sh/bun/issues/20412 - -export class SqliteAdapter extends BaseAdapter { - nativeTypes = ["integer", "number", "string", "year"] satisfies FieldType[] - - async createDialect(path: string, options?: { create?: boolean }) { - path = path.replace(/^sqlite:\/\//, "") - - if (!options?.create) { - const isExist = await isLocalPathExist(path) - if (!isExist) { - throw new Error(`Database file "${path}" does not exist`) - } - } - - // @ts-ignore - if (typeof Bun !== "undefined") { - const { createBunSqliteDialect } = await import("./sqlite.bun.ts") - return await createBunSqliteDialect(path) - } - - const { createNodeSqliteDialect } = await import("./sqlite.node.ts") - return await createNodeSqliteDialect(path) - } - - normalizeType(databaseType: DatabaseType): FieldType { - switch (databaseType.toLowerCase()) { - case "blob": - return "string" - case "text": - return "string" - case "integer": - return "integer" - case "numeric": - case "real": - return "number" - default: - return "string" - } - } - - denormalizeType(fieldType: FieldType): DatabaseType { - switch (fieldType) { - case "boolean": - return "integer" - case "integer": - return "integer" - case "number": - return "real" - case "string": - return "text" - case "year": - return "integer" - default: - return "text" - } - } -} diff --git a/database/field/Field.ts b/database/field/Field.ts deleted file mode 100644 index a6d0b4b9..00000000 --- a/database/field/Field.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { ColumnMetadata } from "kysely" - -export interface DatabaseField extends ColumnMetadata {} diff --git a/database/field/Type.ts b/database/field/Type.ts deleted file mode 100644 index a1b74258..00000000 --- a/database/field/Type.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { DatabaseField } from "./Field.ts" - -export type DatabaseType = DatabaseField["dataType"] diff --git a/database/field/index.ts b/database/field/index.ts deleted file mode 100644 index b6ff85ad..00000000 --- a/database/field/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { DatabaseField } from "./Field.ts" -export type { DatabaseType } from "./Type.ts" diff --git a/database/index.ts b/database/index.ts deleted file mode 100644 index 6d4ebc6b..00000000 --- a/database/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { DatabasePlugin } from "./plugin.ts" - -export { inferDatabaseSchema } from "./schema/index.ts" -export { loadDatabaseTable } from "./table/index.ts" -export { loadPackageFromDatabase } from "./package/index.ts" -export { saveDatabaseTable } from "./table/index.ts" -export { savePackageToDatabase } from "./package/index.ts" diff --git a/database/package.json b/database/package.json deleted file mode 100644 index 249227a3..00000000 --- a/database/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@dpkit/database", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "database" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "@dpkit/table": "workspace:*", - "kysely": "^0.28.5", - "kysely-bun-sqlite": "^0.4.0", - "kysely-generic-sqlite": "^1.2.1", - "lru-cache": "^11.2.1", - "mysql2": "^3.14.4", - "nodejs-polars": "^0.22.2", - "pg": "^8.16.3" - }, - "devDependencies": { - "@dpkit/dataset": "workspace:*", - "@types/pg": "^8.15.5" - } -} diff --git a/database/package/index.ts b/database/package/index.ts deleted file mode 100644 index bf14c237..00000000 --- a/database/package/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadPackageFromDatabase } from "./load.ts" -export { savePackageToDatabase } from "./save.ts" diff --git a/database/package/load.ts b/database/package/load.ts deleted file mode 100644 index c0e83a8b..00000000 --- a/database/package/load.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { createAdapter } from "../adapters/create.ts" -import type { DatabaseFormat } from "../resource/index.ts" - -export async function loadPackageFromDatabase( - connectionString: string, - options: { - format: DatabaseFormat - includeTables?: string[] - excludeTables?: string[] - }, -) { - const { includeTables, excludeTables } = options - - const adapter = createAdapter(options.format) - const database = await adapter.connectDatabase(connectionString) - const databaseSchemas = await database.introspection.getTables() - - const dataPackage: Package = { - resources: [], - } - - for (const databaseSchema of databaseSchemas) { - const name = databaseSchema.name - - if (includeTables && !includeTables.includes(name)) { - continue - } - - if (excludeTables?.includes(name)) { - continue - } - - const schema = adapter.normalizeSchema(databaseSchema) - const dialect = { table: name } - - dataPackage.resources.push({ - name, - path: connectionString, - format: options.format, - dialect, - schema, - }) - } - - return dataPackage -} diff --git a/database/package/save.ts b/database/package/save.ts deleted file mode 100644 index 0c8c5a0d..00000000 --- a/database/package/save.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { SavePackageOptions } from "@dpkit/dataset" -import type { Package } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { isRemoteResource } from "@dpkit/metadata" -import type { TablePlugin } from "@dpkit/table" -import type { DatabaseFormat } from "../resource/index.ts" -import { saveDatabaseTable } from "../table/index.ts" - -export async function savePackageToDatabase( - dataPackage: Package, - options: SavePackageOptions & { - format: DatabaseFormat - plugins?: TablePlugin[] - }, -) { - for (const resource of dataPackage.resources) { - for (const plugin of options.plugins ?? []) { - const isRemote = isRemoteResource(resource) - if (isRemote && !options.withRemote) { - continue - } - - const table = await plugin.loadTable?.(resource) - - if (table) { - const dialect = { table: resource.name } - const schema = await resolveSchema(resource.schema) - - // TODO: support parallel saving? - await saveDatabaseTable(table, { - path: options.target, - format: options.format, - dialect, - schema, - }) - - break - } - } - } - - return { path: options.target } -} diff --git a/database/plugin.spec.ts b/database/plugin.spec.ts deleted file mode 100644 index a3cafcd9..00000000 --- a/database/plugin.spec.ts +++ /dev/null @@ -1,417 +0,0 @@ -import type { Package, Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/index.ts" -import { DatabasePlugin } from "./plugin.ts" -import * as schemaModule from "./schema/index.ts" -import * as tableModule from "./table/index.ts" - -vi.mock("./package/index.ts", () => ({ - loadPackageFromDatabase: vi.fn(), - savePackageToDatabase: vi.fn(), -})) - -vi.mock("./table/index.ts", () => ({ - loadDatabaseTable: vi.fn(), - saveDatabaseTable: vi.fn(), -})) - -vi.mock("./schema/index.ts", () => ({ - inferDatabaseSchema: vi.fn(), -})) - -describe("DatabasePlugin", () => { - let plugin: DatabasePlugin - let mockLoadPackageFromDatabase: ReturnType - let mockSavePackageToDatabase: ReturnType - let mockLoadDatabaseTable: ReturnType - let mockSaveDatabaseTable: ReturnType - let mockInferDatabaseSchema: ReturnType - - beforeEach(() => { - plugin = new DatabasePlugin() - mockLoadPackageFromDatabase = vi.mocked( - packageModule.loadPackageFromDatabase, - ) - mockSavePackageToDatabase = vi.mocked(packageModule.savePackageToDatabase) - mockLoadDatabaseTable = vi.mocked(tableModule.loadDatabaseTable) - mockSaveDatabaseTable = vi.mocked(tableModule.saveDatabaseTable) - mockInferDatabaseSchema = vi.mocked(schemaModule.inferDatabaseSchema) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from postgresql database", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromDatabase.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("postgresql://localhost/testdb") - - expect(mockLoadPackageFromDatabase).toHaveBeenCalledWith( - "postgresql://localhost/testdb", - { format: "postgresql" }, - ) - expect(result).toEqual(mockPackage) - }) - - it("should load package from mysql database", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromDatabase.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("mysql://localhost/testdb") - - expect(mockLoadPackageFromDatabase).toHaveBeenCalledWith( - "mysql://localhost/testdb", - { format: "mysql" }, - ) - expect(result).toEqual(mockPackage) - }) - - it("should load package from sqlite database", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromDatabase.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("sqlite://test.db") - - expect(mockLoadPackageFromDatabase).toHaveBeenCalledWith( - "sqlite://test.db", - { - format: "sqlite", - }, - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for non-database sources", async () => { - const result = await plugin.loadPackage("test.csv") - - expect(mockLoadPackageFromDatabase).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for http urls", async () => { - const result = await plugin.loadPackage("https://example.com/data") - - expect(mockLoadPackageFromDatabase).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("savePackage", () => { - it("should save package to postgresql database", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockSavePackageToDatabase.mockResolvedValue(undefined) - - await plugin.savePackage(mockPackage, { - target: "postgresql://localhost/testdb", - }) - - expect(mockSavePackageToDatabase).toHaveBeenCalledWith(mockPackage, { - target: "postgresql://localhost/testdb", - format: "postgresql", - }) - }) - - it("should save package to mysql database", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockSavePackageToDatabase.mockResolvedValue(undefined) - - await plugin.savePackage(mockPackage, { - target: "mysql://localhost/testdb", - }) - - expect(mockSavePackageToDatabase).toHaveBeenCalledWith(mockPackage, { - target: "mysql://localhost/testdb", - format: "mysql", - }) - }) - - it("should save package to sqlite database", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockSavePackageToDatabase.mockResolvedValue(undefined) - - await plugin.savePackage(mockPackage, { target: "sqlite://test.db" }) - - expect(mockSavePackageToDatabase).toHaveBeenCalledWith(mockPackage, { - target: "sqlite://test.db", - format: "sqlite", - }) - }) - - it("should return undefined for non-database targets", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - - const result = await plugin.savePackage(mockPackage, { - target: "test.csv", - }) - - expect(mockSavePackageToDatabase).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should pass through plugins option", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - const mockPlugins: any[] = [] - mockSavePackageToDatabase.mockResolvedValue(undefined) - - await plugin.savePackage(mockPackage, { - target: "sqlite://test.db", - plugins: mockPlugins, - }) - - expect(mockSavePackageToDatabase).toHaveBeenCalledWith(mockPackage, { - target: "sqlite://test.db", - format: "sqlite", - plugins: mockPlugins, - }) - }) - }) - - describe("loadTable", () => { - it("should load table from postgresql resource", async () => { - const resource: Partial = { - path: "postgresql://localhost/testdb", - } - const mockTable = pl.DataFrame().lazy() - mockLoadDatabaseTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadDatabaseTable).toHaveBeenCalledWith({ - ...resource, - format: "postgresql", - }) - expect(result).toEqual(mockTable) - }) - - it("should load table from mysql resource", async () => { - const resource: Partial = { - path: "mysql://localhost/testdb", - } - const mockTable = pl.DataFrame().lazy() - mockLoadDatabaseTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadDatabaseTable).toHaveBeenCalledWith({ - ...resource, - format: "mysql", - }) - expect(result).toEqual(mockTable) - }) - - it("should load table from sqlite resource", async () => { - const resource: Partial = { - path: "sqlite://test.db", - } - const mockTable = pl.DataFrame().lazy() - mockLoadDatabaseTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadDatabaseTable).toHaveBeenCalledWith({ - ...resource, - format: "sqlite", - }) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-database resources", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadDatabaseTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "sqlite", - } - const mockTable = pl.DataFrame().lazy() - mockLoadDatabaseTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadDatabaseTable).toHaveBeenCalledWith({ - ...resource, - format: "sqlite", - }) - expect(result).toEqual(mockTable) - }) - }) - - describe("saveTable", () => { - it("should save table to postgresql database", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "postgresql://localhost/testdb" } - mockSaveDatabaseTable.mockResolvedValue("postgresql://localhost/testdb") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveDatabaseTable).toHaveBeenCalledWith(table, { - ...options, - format: "postgresql", - }) - expect(result).toBe("postgresql://localhost/testdb") - }) - - it("should save table to mysql database", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "mysql://localhost/testdb" } - mockSaveDatabaseTable.mockResolvedValue("mysql://localhost/testdb") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveDatabaseTable).toHaveBeenCalledWith(table, { - ...options, - format: "mysql", - }) - expect(result).toBe("mysql://localhost/testdb") - }) - - it("should save table to sqlite database", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "sqlite://test.db" } - mockSaveDatabaseTable.mockResolvedValue("sqlite://test.db") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveDatabaseTable).toHaveBeenCalledWith(table, { - ...options, - format: "sqlite", - }) - expect(result).toBe("sqlite://test.db") - }) - - it("should return undefined for non-database paths", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveDatabaseTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "test.txt", format: "sqlite" as const } - mockSaveDatabaseTable.mockResolvedValue("test.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveDatabaseTable).toHaveBeenCalledWith(table, { - ...options, - format: "sqlite", - }) - expect(result).toBe("test.txt") - }) - }) - - describe("inferSchema", () => { - it("should infer schema for postgresql resource", async () => { - const resource: Partial = { - path: "postgresql://localhost/testdb", - } - const mockSchema = { fields: [] } - mockInferDatabaseSchema.mockResolvedValue(mockSchema) - - const result = await plugin.inferSchema(resource) - - expect(mockInferDatabaseSchema).toHaveBeenCalledWith({ - ...resource, - format: "postgresql", - }) - expect(result).toEqual(mockSchema) - }) - - it("should infer schema for mysql resource", async () => { - const resource: Partial = { - path: "mysql://localhost/testdb", - } - const mockSchema = { fields: [] } - mockInferDatabaseSchema.mockResolvedValue(mockSchema) - - const result = await plugin.inferSchema(resource) - - expect(mockInferDatabaseSchema).toHaveBeenCalledWith({ - ...resource, - format: "mysql", - }) - expect(result).toEqual(mockSchema) - }) - - it("should infer schema for sqlite resource", async () => { - const resource: Partial = { - path: "sqlite://test.db", - } - const mockSchema = { fields: [] } - mockInferDatabaseSchema.mockResolvedValue(mockSchema) - - const result = await plugin.inferSchema(resource) - - expect(mockInferDatabaseSchema).toHaveBeenCalledWith({ - ...resource, - format: "sqlite", - }) - expect(result).toEqual(mockSchema) - }) - - it("should return undefined for non-database resources", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.inferSchema(resource) - - expect(mockInferDatabaseSchema).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "sqlite", - } - const mockSchema = { fields: [] } - mockInferDatabaseSchema.mockResolvedValue(mockSchema) - - const result = await plugin.inferSchema(resource) - - expect(mockInferDatabaseSchema).toHaveBeenCalledWith({ - ...resource, - format: "sqlite", - }) - expect(result).toEqual(mockSchema) - }) - }) -}) diff --git a/database/plugin.ts b/database/plugin.ts deleted file mode 100644 index 74858a2d..00000000 --- a/database/plugin.ts +++ /dev/null @@ -1,67 +0,0 @@ -import type { SavePackageOptions } from "@dpkit/dataset" -import type { Package, Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { TablePlugin } from "@dpkit/table" -import type { SaveTableOptions, Table } from "@dpkit/table" -import { loadPackageFromDatabase } from "./package/index.ts" -import { savePackageToDatabase } from "./package/index.ts" -import { inferDatabaseSchema } from "./schema/index.ts" -import { loadDatabaseTable } from "./table/index.ts" -import { saveDatabaseTable } from "./table/index.ts" - -export class DatabasePlugin implements TablePlugin { - async savePackage( - dataPackage: Package, - options: SavePackageOptions & { plugins?: TablePlugin[] }, - ) { - const databaseFormat = getDatabaseFormat({ path: options.target }) - if (!databaseFormat) return undefined - - return await savePackageToDatabase(dataPackage, { - format: databaseFormat, - ...options, - }) - } - - async loadPackage(source: string) { - const databaseFormat = getDatabaseFormat({ path: source }) - if (!databaseFormat) return undefined - - return await loadPackageFromDatabase(source, { - format: databaseFormat, - }) - } - - async inferSchema(resource: Partial) { - const databaseFormat = getDatabaseFormat(resource) - if (!databaseFormat) return undefined - - return await inferDatabaseSchema({ ...resource, format: databaseFormat }) - } - - async loadTable(resource: Partial) { - const databaseFormat = getDatabaseFormat(resource) - if (!databaseFormat) return undefined - - return await loadDatabaseTable({ ...resource, format: databaseFormat }) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const databaseFormat = getDatabaseFormat({ path, format }) - if (!databaseFormat) return undefined - - return await saveDatabaseTable(table, { - ...options, - format: databaseFormat, - }) - } -} - -function getDatabaseFormat(resource: Partial) { - const format = inferFormat(resource) - return format === "postgresql" || format === "mysql" || format === "sqlite" - ? format - : undefined -} diff --git a/database/resource/Format.ts b/database/resource/Format.ts deleted file mode 100644 index da7e240d..00000000 --- a/database/resource/Format.ts +++ /dev/null @@ -1 +0,0 @@ -export type DatabaseFormat = "sqlite" | "postgresql" | "mysql" diff --git a/database/resource/index.ts b/database/resource/index.ts deleted file mode 100644 index dea20589..00000000 --- a/database/resource/index.ts +++ /dev/null @@ -1 +0,0 @@ -export type { DatabaseFormat } from "./Format.ts" diff --git a/database/schema/Schema.ts b/database/schema/Schema.ts deleted file mode 100644 index 7aa0fdc1..00000000 --- a/database/schema/Schema.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { TableMetadata } from "kysely" - -export interface DatabaseSchema extends TableMetadata { - primaryKey?: string[] -} diff --git a/database/schema/index.ts b/database/schema/index.ts deleted file mode 100644 index 694d37bb..00000000 --- a/database/schema/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { inferDatabaseSchema } from "./infer.ts" -export type { DatabaseSchema } from "./Schema.ts" diff --git a/database/schema/infer.spec.ts b/database/schema/infer.spec.ts deleted file mode 100644 index 5f98a9d1..00000000 --- a/database/schema/infer.spec.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, it } from "vitest" -import { inferDatabaseSchema } from "./infer.ts" - -// TODO: Enable when SQLite support is fixed -describe.skip("inferDatabaseSchema", () => { - it("throws error when resource path is not defined", async () => { - await expect( - inferDatabaseSchema({ - format: "sqlite", - dialect: { table: "dpkit" }, - }), - ).rejects.toThrow("Resource path is not defined") - }) - - it("throws error when table name is not defined in dialect", async () => { - await expect( - inferDatabaseSchema({ - path: "path", - format: "sqlite", - }), - ).rejects.toThrow("Table name is not defined in dialect") - }) - - it("throws error when format is not supported", async () => { - await expect( - inferDatabaseSchema({ - path: "path", - format: "unsupported" as any, - dialect: { table: "dpkit" }, - }), - ).rejects.toThrow('Unsupported database format: "unsupported"') - }) -}) diff --git a/database/schema/infer.ts b/database/schema/infer.ts deleted file mode 100644 index f82159e3..00000000 --- a/database/schema/infer.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { resolveDialect } from "@dpkit/metadata" -import { createAdapter } from "../adapters/create.ts" - -export async function inferDatabaseSchema( - resource: Partial & { format: "postgresql" | "mysql" | "sqlite" }, -) { - const adapter = createAdapter(resource.format) - if (!adapter) { - throw new Error("Supported database format is not defined") - } - - const dialect = await resolveDialect(resource.dialect) - if (!dialect?.table) { - throw new Error("Table name is not defined in dialect") - } - - const path = typeof resource.path === "string" ? resource.path : undefined - if (!path) { - throw new Error("Resource path is not defined") - } - - const database = await adapter.connectDatabase(path) - const databaseSchemas = await database.introspection.getTables() - - const databaseSchema = databaseSchemas.find(s => s.name === dialect.table) - if (!databaseSchema) { - throw new Error(`Table is not found in database: ${dialect.table}`) - } - - return adapter.normalizeSchema(databaseSchema) -} diff --git a/database/table/index.ts b/database/table/index.ts deleted file mode 100644 index 8c8238f0..00000000 --- a/database/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadDatabaseTable } from "./load.ts" -export { saveDatabaseTable } from "./save.ts" diff --git a/database/table/load.spec.ts b/database/table/load.spec.ts deleted file mode 100644 index 36801850..00000000 --- a/database/table/load.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, expect, it } from "vitest" -import { loadDatabaseTable } from "./load.ts" - -describe("loadDatabaseTable", () => { - it("throws error when resource path is not defined", async () => { - await expect( - loadDatabaseTable({ - format: "sqlite", - dialect: { table: "dpkit" }, - }), - ).rejects.toThrow("Resource path is not defined") - }) - - it("throws error when table name is not defined in dialect", async () => { - await expect( - loadDatabaseTable({ - path: "path", - format: "sqlite", - }), - ).rejects.toThrow("Table name is not defined in dialect") - }) - - it("throws error when format is not supported", async () => { - await expect( - loadDatabaseTable({ - path: "path", - format: "unsupported" as any, - dialect: { table: "dpkit" }, - }), - ).rejects.toThrow('Unsupported database format: "unsupported"') - }) -}) diff --git a/database/table/load.ts b/database/table/load.ts deleted file mode 100644 index c6999474..00000000 --- a/database/table/load.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { resolveDialect, resolveSchema } from "@dpkit/metadata" -import type { Resource } from "@dpkit/metadata" -import { normalizeTable } from "@dpkit/table" -import type { LoadTableOptions } from "@dpkit/table" -import * as pl from "nodejs-polars" -import { createAdapter } from "../adapters/create.ts" -import { inferDatabaseSchema } from "../schema/index.ts" - -// Currently, we use slow non-rust implementation as in the future -// polars-rust might be able to provide a faster native implementation - -export async function loadDatabaseTable( - resource: Partial & { format: "postgresql" | "mysql" | "sqlite" }, - options?: LoadTableOptions, -) { - const dialect = await resolveDialect(resource.dialect) - if (!dialect?.table) { - throw new Error("Table name is not defined in dialect") - } - - const path = typeof resource.path === "string" ? resource.path : undefined - if (!path) { - throw new Error("Resource path is not defined") - } - - const adapter = createAdapter(resource.format) - const database = await adapter.connectDatabase(path) - const records = await database.selectFrom(dialect.table).selectAll().execute() - - let table = pl.DataFrame(records).lazy() - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferDatabaseSchema(resource) - table = await normalizeTable(table, schema) - } - - return table -} diff --git a/database/table/save.spec.ts b/database/table/save.spec.ts deleted file mode 100644 index 34a3f78e..00000000 --- a/database/table/save.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { saveDatabaseTable } from "./save.js" - -describe("saveDatabaseTable", () => { - const mockTable = pl.DataFrame({ col1: [1, 2, 3] }).lazy() - - it("throws error when table name is not defined in dialect", async () => { - await expect( - saveDatabaseTable(mockTable, { - path: "test.db", - format: "sqlite", - }), - ).rejects.toThrow("Table name is not defined in dialect") - }) - - it("throws error when format is not supported", async () => { - await expect( - saveDatabaseTable(mockTable, { - path: "test.db", - format: "unsupported" as any, - dialect: { table: "test_table" }, - }), - ).rejects.toThrow('Unsupported database format: "unsupported"') - }) -}) diff --git a/database/table/save.ts b/database/table/save.ts deleted file mode 100644 index 6df0c47e..00000000 --- a/database/table/save.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { SaveTableOptions, Table } from "@dpkit/table" -import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" -import type { Kysely } from "kysely" -import { createAdapter } from "../adapters/create.ts" -import type { DatabaseSchema } from "../schema/index.ts" - -// Currently, we use slow non-rust implementation as in the future -// polars-rust might be able to provide a faster native implementation -// (if not supported we can use COPY in PostgreSQL/MySQL) - -export async function saveDatabaseTable( - table: Table, - options: SaveTableOptions & { format: "postgresql" | "mysql" | "sqlite" }, -) { - const { path, format, dialect, overwrite } = options - - const tableName = dialect?.table - if (!tableName) { - throw new Error("Table name is not defined in dialect") - } - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - const adapter = createAdapter(format) - table = await denormalizeTable(table, schema, { - nativeTypes: adapter.nativeTypes, - }) - - const database = await adapter.connectDatabase(path, { create: true }) - const databaseSchema = adapter.denormalizeSchema(schema, tableName) - - await defineTable(database, databaseSchema, { overwrite }) - await populateTable(database, tableName, table) - - return path -} - -async function defineTable( - database: Kysely, - databaseSchema: DatabaseSchema, - options: { - overwrite?: boolean - }, -) { - if (options.overwrite) { - await database.schema.dropTable(databaseSchema.name).ifExists().execute() - } - - let query = database.schema.createTable(databaseSchema.name) - - for (const field of databaseSchema.columns) { - // @ts-ignore - query = query.addColumn(field.name, field.dataType) - } - - if (databaseSchema.primaryKey) { - query = query.addPrimaryKeyConstraint( - `${databaseSchema.name}_pkey`, - // @ts-ignore - databaseSchema.primaryKey, - ) - } - - await query.execute() -} - -async function populateTable( - database: Kysely, - tableName: string, - table: Table, -) { - let offset = 0 - const frame = await table.collect({ streaming: true }) - while (true) { - const buffer = frame.slice(offset, offset + BUFFER_SIZE) - offset += BUFFER_SIZE - - const records = buffer.toRecords() - if (!records.length) { - break - } - - await database.insertInto(tableName).values(records).execute() - } -} - -const BUFFER_SIZE = 10_000 diff --git a/database/tsconfig.json b/database/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/database/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/database/typedoc.json b/database/typedoc.json deleted file mode 100644 index f8e49f3a..00000000 --- a/database/typedoc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "entryPoints": ["index.ts"], - "skipErrorChecking": true -} diff --git a/dataset/README.md b/dataset/README.md deleted file mode 100644 index 0b8014d7..00000000 --- a/dataset/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/dataset - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/dataset/file/copy.spec.ts b/dataset/file/copy.spec.ts deleted file mode 100644 index b1cd24af..00000000 --- a/dataset/file/copy.spec.ts +++ /dev/null @@ -1,134 +0,0 @@ -import * as fs from "node:fs/promises" -import * as path from "node:path" -import { temporaryDirectory } from "tempy" -import { afterEach, beforeEach, describe, expect, it } from "vitest" -import { copyFile } from "./copy.ts" -import { writeTempFile } from "./temp.ts" - -describe("copyFile", () => { - let testDir: string - - beforeEach(() => { - testDir = temporaryDirectory() - }) - - afterEach(async () => { - try { - await fs.rm(testDir, { recursive: true, force: true }) - } catch (error) { - if (error instanceof Error && !error.message.includes("ENOENT")) { - console.error(`Failed to clean up test directory: ${testDir}`, error) - } - } - }) - - it("should copy file from source to target", async () => { - const sourcePath = await writeTempFile("test content") - const targetPath = path.join(testDir, "target.txt") - - await copyFile({ sourcePath, targetPath }) - - const fileExists = await fs - .stat(targetPath) - .then(() => true) - .catch(() => false) - expect(fileExists).toBe(true) - - const content = await fs.readFile(targetPath, "utf-8") - expect(content).toBe("test content") - }) - - it("should copy file with exact content", async () => { - const content = "Hello, World! This is a test file." - const sourcePath = await writeTempFile(content) - const targetPath = path.join(testDir, "copy.txt") - - await copyFile({ sourcePath, targetPath }) - - const copiedContent = await fs.readFile(targetPath, "utf-8") - expect(copiedContent).toBe(content) - }) - - it("should copy binary file", async () => { - const binaryData = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]) - const sourcePath = await writeTempFile(binaryData) - const targetPath = path.join(testDir, "binary.bin") - - await copyFile({ sourcePath, targetPath }) - - const copiedData = await fs.readFile(targetPath) - expect(Buffer.compare(copiedData, binaryData)).toBe(0) - }) - - it("should copy empty file", async () => { - const sourcePath = await writeTempFile("") - const targetPath = path.join(testDir, "empty.txt") - - await copyFile({ sourcePath, targetPath }) - - const content = await fs.readFile(targetPath, "utf-8") - expect(content).toBe("") - }) - - it("should copy large file", async () => { - const largeContent = "x".repeat(100000) - const sourcePath = await writeTempFile(largeContent) - const targetPath = path.join(testDir, "large.txt") - - await copyFile({ sourcePath, targetPath }) - - const copiedContent = await fs.readFile(targetPath, "utf-8") - expect(copiedContent).toBe(largeContent) - expect(copiedContent.length).toBe(100000) - }) - - it("should copy file with special characters", async () => { - const content = "Special characters: é, ñ, ü, ö, à, 中文, 日本語" - const sourcePath = await writeTempFile(content) - const targetPath = path.join(testDir, "special.txt") - - await copyFile({ sourcePath, targetPath }) - - const copiedContent = await fs.readFile(targetPath, "utf-8") - expect(copiedContent).toBe(content) - }) - - it("should copy file to nested directory", async () => { - const sourcePath = await writeTempFile("nested content") - const targetPath = path.join(testDir, "nested", "dir", "file.txt") - - await copyFile({ sourcePath, targetPath }) - - const fileExists = await fs - .stat(targetPath) - .then(() => true) - .catch(() => false) - expect(fileExists).toBe(true) - - const content = await fs.readFile(targetPath, "utf-8") - expect(content).toBe("nested content") - }) - - it("should copy json file", async () => { - const jsonContent = JSON.stringify({ name: "test", value: 123 }) - const sourcePath = await writeTempFile(jsonContent) - const targetPath = path.join(testDir, "data.json") - - await copyFile({ sourcePath, targetPath }) - - const copiedContent = await fs.readFile(targetPath, "utf-8") - expect(copiedContent).toBe(jsonContent) - expect(JSON.parse(copiedContent)).toEqual({ name: "test", value: 123 }) - }) - - it("should copy file with newlines", async () => { - const content = "Line 1\nLine 2\nLine 3\n" - const sourcePath = await writeTempFile(content) - const targetPath = path.join(testDir, "multiline.txt") - - await copyFile({ sourcePath, targetPath }) - - const copiedContent = await fs.readFile(targetPath, "utf-8") - expect(copiedContent).toBe(content) - }) -}) diff --git a/dataset/file/copy.ts b/dataset/file/copy.ts deleted file mode 100644 index ceae3fd7..00000000 --- a/dataset/file/copy.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { loadFileStream } from "../stream/load.ts" -import { saveFileStream } from "../stream/save.ts" - -export async function copyFile(options: { - sourcePath: string - targetPath: string -}) { - const stream = await loadFileStream(options.sourcePath) - await saveFileStream(stream, { path: options.targetPath }) -} diff --git a/dataset/file/describe.ts b/dataset/file/describe.ts deleted file mode 100644 index 47a09a78..00000000 --- a/dataset/file/describe.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { prefetchFile } from "./fetch.ts" -import { inferBytes, inferHash } from "./infer.ts" -import type { HashType } from "./infer.ts" - -export async function describeFile( - path: string, - options?: { hashType?: HashType }, -) { - const localPath = await prefetchFile(path) - - const bytes = await inferBytes({ path: localPath }) - const hash = await inferHash( - { path: localPath }, - { hashType: options?.hashType }, - ) - - return { bytes, hash } -} diff --git a/dataset/file/fetch.ts b/dataset/file/fetch.ts deleted file mode 100644 index f6ea467c..00000000 --- a/dataset/file/fetch.ts +++ /dev/null @@ -1,26 +0,0 @@ -import os from "node:os" -import { isRemotePath } from "@dpkit/metadata" -import pAll from "p-all" -import { copyFile } from "./copy.ts" -import { getTempFilePath } from "./temp.ts" - -export async function prefetchFiles(path?: string | string[]) { - if (!path) return [] - - const paths = Array.isArray(path) ? path : [path] - const concurrency = os.cpus().length - - const newPaths = await pAll( - paths.map(path => () => prefetchFile(path)), - { concurrency }, - ) - - return newPaths -} - -export async function prefetchFile(path: string) { - if (!isRemotePath(path)) return path - const newPath = getTempFilePath() - await copyFile({ sourcePath: path, targetPath: newPath }) - return newPath -} diff --git a/dataset/file/index.ts b/dataset/file/index.ts deleted file mode 100644 index 3f58548b..00000000 --- a/dataset/file/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { loadFile } from "./load.ts" -export { copyFile } from "./copy.ts" -export { saveFile } from "./save.ts" -export { getTempFilePath, writeTempFile } from "./temp.ts" -export { assertLocalPathVacant, isLocalPathExist } from "./path.ts" -export { prefetchFile, prefetchFiles } from "./fetch.ts" -export { inferEncoding, inferBytes, inferHash } from "./infer.ts" -export { describeFile } from "./describe.ts" -export { validateFile } from "./validate.ts" diff --git a/dataset/file/infer.spec.ts b/dataset/file/infer.spec.ts deleted file mode 100644 index d8c81d51..00000000 --- a/dataset/file/infer.spec.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as fetchModule from "./fetch.ts" -import { inferBytes, inferEncoding, inferHash } from "./infer.ts" -import { writeTempFile } from "./temp.ts" - -vi.mock("./fetch.ts", () => ({ - prefetchFiles: vi.fn(), -})) - -describe("inferHash", () => { - let mockPrefetchFiles: ReturnType - let tempFilePath: string - - beforeEach(async () => { - mockPrefetchFiles = vi.mocked(fetchModule.prefetchFiles) - tempFilePath = await writeTempFile("Hello, World!") - vi.clearAllMocks() - }) - - it("should compute sha256 hash by default", async () => { - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferHash({ path: "https://example.com/file.txt" }) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.txt", - ) - expect(result).toMatch(/^sha256:[a-f0-9]{64}$/) - }) - - it("should compute md5 hash when specified", async () => { - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferHash( - { path: "https://example.com/file.txt" }, - { - hashType: "md5", - }, - ) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.txt", - ) - expect(result).toMatch(/^md5:[a-f0-9]{32}$/) - }) - - it("should compute sha1 hash when specified", async () => { - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferHash( - { path: "https://example.com/file.txt" }, - { - hashType: "sha1", - }, - ) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.txt", - ) - expect(result).toMatch(/^sha1:[a-f0-9]{40}$/) - }) - - it("should compute sha512 hash when specified", async () => { - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferHash( - { path: "https://example.com/file.txt" }, - { - hashType: "sha512", - }, - ) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.txt", - ) - expect(result).toMatch(/^sha512:[a-f0-9]{128}$/) - }) - - it("should compute consistent hashes for same content", async () => { - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result1 = await inferHash({ path: "https://example.com/file.txt" }) - const result2 = await inferHash({ path: "https://example.com/file.txt" }) - - expect(result1).toBe(result2) - }) -}) - -describe("inferBytes", () => { - let mockPrefetchFiles: ReturnType - - beforeEach(() => { - mockPrefetchFiles = vi.mocked(fetchModule.prefetchFiles) - vi.clearAllMocks() - }) - - it("should return file size in bytes", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferBytes({ path: "https://example.com/file.txt" }) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.txt", - ) - expect(result).toBe(13) - }) - - it("should handle empty files", async () => { - const tempFilePath = await writeTempFile("") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferBytes({ path: "https://example.com/empty.txt" }) - - expect(result).toBe(0) - }) - - it("should handle larger files", async () => { - const tempFilePath = await writeTempFile("x".repeat(10000)) - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferBytes({ path: "https://example.com/large.txt" }) - - expect(result).toBe(10000) - }) - - it("should handle binary data", async () => { - const tempFilePath = await writeTempFile( - Buffer.from([0xff, 0xd8, 0xff, 0xe0]), - ) - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const result = await inferBytes({ path: "https://example.com/file.bin" }) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.bin", - ) - expect(result).toBe(4) - }) -}) - -describe("inferEncoding", () => { - it("should detect utf-8 encoding", async () => { - const tempFilePath = await writeTempFile( - "Hello, World! This is UTF-8 text.", - ) - - const result = await inferEncoding({ path: tempFilePath }) - - expect(result).toBeDefined() - expect(["utf-8", "utf8", "ascii"]).toContain(result) - }) - - it("should return undefined for binary files", async () => { - const tempFilePath = await writeTempFile( - Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00]), - ) - - const result = await inferEncoding({ path: tempFilePath }) - - expect(result).toBeUndefined() - }) - - it("should use custom sample bytes", async () => { - const tempFilePath = await writeTempFile( - "This is a test file with UTF-8 content.", - ) - - const result = await inferEncoding( - { path: tempFilePath }, - { sampleBytes: 20 }, - ) - - expect(result).toBeDefined() - }) - - it("should use custom confidence threshold", async () => { - const tempFilePath = await writeTempFile("Sample text content") - - const result = await inferEncoding( - { path: tempFilePath }, - { - confidencePercent: 50, - }, - ) - - expect(result).toBeDefined() - }) - - it("should handle large text files", async () => { - const tempFilePath = await writeTempFile("Hello World! ".repeat(1000)) - - const result = await inferEncoding({ path: tempFilePath }) - - expect(result).toBeDefined() - expect(["utf-8", "utf8", "ascii"]).toContain(result) - }) - - it("should return encoding in lowercase", async () => { - const tempFilePath = await writeTempFile( - "Test content for encoding detection", - ) - - const result = await inferEncoding({ path: tempFilePath }) - - if (result) { - expect(result).toBe(result.toLowerCase()) - } - }) - - it("should handle empty files", async () => { - const tempFilePath = await writeTempFile("") - - const result = await inferEncoding({ path: tempFilePath }) - - expect([undefined, "utf-8", "utf8", "ascii"]).toContain(result) - }) - - it("should handle files with special characters", async () => { - const tempFilePath = await writeTempFile("Special: é, ñ, ü, ö, à") - - const result = await inferEncoding({ path: tempFilePath }) - - expect(result).toBeDefined() - }) - - it("should detect encoding with low confidence threshold", async () => { - const tempFilePath = await writeTempFile("Simple text") - - const result = await inferEncoding( - { path: tempFilePath }, - { - confidencePercent: 30, - }, - ) - - expect(result).toBeDefined() - }) -}) diff --git a/dataset/file/infer.ts b/dataset/file/infer.ts deleted file mode 100644 index b7157790..00000000 --- a/dataset/file/infer.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { stat } from "node:fs/promises" -import type { Resource } from "@dpkit/metadata" -import chardet from "chardet" -import * as hasha from "hasha" -import { isBinaryFile } from "isbinaryfile" -import pMap from "p-map" -import { concatFileStreams } from "../stream/concat.ts" -import { loadFileStream } from "../stream/index.ts" -import { prefetchFiles } from "./fetch.ts" -import { loadFile } from "./load.ts" - -export type HashType = "md5" | "sha1" | "sha256" | "sha512" - -export async function inferBytes(resource: Partial) { - const localPaths = await prefetchFiles(resource.path) - - let bytes = 0 - for (const localPath of localPaths) { - const result = await stat(localPath) - bytes += result.size - } - - return bytes -} - -export async function inferHash( - resource: Partial, - options?: { hashType?: HashType }, -) { - const algorithm = options?.hashType ?? "sha256" - const localPaths = await prefetchFiles(resource.path) - - const streams = await pMap(localPaths, async path => loadFileStream(path)) - const stream = concatFileStreams(streams) - - const hash = await hasha.hash(stream, { algorithm }) - return `${algorithm}:${hash}` -} - -export async function inferEncoding( - resource: Partial, - options?: { sampleBytes?: number; confidencePercent?: number }, -) { - const maxBytes = options?.sampleBytes ?? 10_000 - const confidencePercent = options?.confidencePercent ?? 80 - - const firstPath = Array.isArray(resource.path) - ? resource.path[0] - : resource.path - - if (!firstPath) { - return undefined - } - - const buffer = await loadFile(firstPath, { maxBytes }) - const isBinary = await isBinaryFile(buffer) - - if (!isBinary) { - const matches = chardet.analyse(buffer) - for (const match of matches) { - if (match.confidence >= confidencePercent) { - return match.name.toLowerCase() - } - } - } - - return undefined -} diff --git a/dataset/file/load.ts b/dataset/file/load.ts deleted file mode 100644 index 6ff4e6b5..00000000 --- a/dataset/file/load.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { buffer } from "node:stream/consumers" -import { loadFileStream } from "../stream/index.ts" - -export async function loadFile(path: string, options?: { maxBytes?: number }) { - const stream = await loadFileStream(path, options) - return await buffer(stream) -} diff --git a/dataset/file/path.ts b/dataset/file/path.ts deleted file mode 100644 index 84b8ecf8..00000000 --- a/dataset/file/path.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { access } from "node:fs/promises" - -export async function isLocalPathExist(path: string) { - try { - await access(path) - return true - } catch (error) { - return false - } -} - -export async function assertLocalPathVacant(path: string) { - const isExist = await isLocalPathExist(path) - - if (isExist) { - throw new Error(`Path "${path}" already exists`) - } -} diff --git a/dataset/file/save.ts b/dataset/file/save.ts deleted file mode 100644 index eac4e0ea..00000000 --- a/dataset/file/save.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Readable } from "node:stream" -import { saveFileStream } from "../stream/index.ts" - -export async function saveFile( - path: string, - buffer: Buffer, - options?: { overwrite?: boolean }, -) { - const { overwrite } = options ?? {} - - await saveFileStream(Readable.from(buffer), { path, overwrite }) -} diff --git a/dataset/file/temp.ts b/dataset/file/temp.ts deleted file mode 100644 index 2b1866f8..00000000 --- a/dataset/file/temp.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { Buffer } from "node:buffer" -import { unlinkSync } from "node:fs" -import { writeFile } from "node:fs/promises" -import exitHook from "exit-hook" -import { temporaryFile } from "tempy" - -export async function writeTempFile( - content: string | Buffer, - options?: { persist?: boolean; filename?: string; format?: string }, -) { - const path = getTempFilePath(options) - await writeFile(path, content) - return path -} - -export function getTempFilePath(options?: { - persist?: boolean - filename?: string - format?: string -}) { - const { filename, format } = options ?? {} - - const path = temporaryFile( - filename ? { name: filename } : { extension: format }, - ) - - if (!options?.persist) { - exitHook(() => { - try { - unlinkSync(path) - } catch {} - }) - } - - return path -} diff --git a/dataset/file/validate.spec.ts b/dataset/file/validate.spec.ts deleted file mode 100644 index 22c1e4f0..00000000 --- a/dataset/file/validate.spec.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as fetchModule from "./fetch.ts" -import { inferHash } from "./infer.ts" -import { writeTempFile } from "./temp.ts" -import { validateFile } from "./validate.ts" - -vi.mock("./fetch.ts", () => ({ - prefetchFiles: vi.fn(), -})) - -describe("validateFile", () => { - let mockPrefetchFiles: ReturnType - - beforeEach(() => { - mockPrefetchFiles = vi.mocked(fetchModule.prefetchFiles) - vi.clearAllMocks() - }) - - it("should return valid report when no validation options provided", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const report = await validateFile({ path: "https://example.com/file.txt" }) - - expect(mockPrefetchFiles).toHaveBeenCalledWith( - "https://example.com/file.txt", - ) - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should validate bytes successfully when they match", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const report = await validateFile({ - path: "https://example.com/file.txt", - bytes: 13, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should return error when bytes do not match", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const report = await validateFile({ - path: "https://example.com/file.txt", - bytes: 1024, - }) - - expect(report.valid).toBe(false) - expect(report.errors).toHaveLength(1) - expect(report.errors[0]).toEqual({ - type: "file/bytes", - bytes: 1024, - actualBytes: 13, - }) - }) - - it("should validate hash successfully when it matches", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "md5" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - hash: actualHash, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should return error when hash does not match", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "md5" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - hash: "md5:wronghash", - }) - - expect(report.valid).toBe(false) - expect(report.errors).toHaveLength(1) - expect(report.errors[0]).toEqual({ - type: "file/hash", - hash: "md5:wronghash", - actualHash, - }) - }) - - it("should validate sha256 hash", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "sha256" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - hash: actualHash, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should validate sha1 hash", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "sha1" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - hash: actualHash, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should validate sha512 hash", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "sha512" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - hash: actualHash, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should validate both bytes and hash when both match", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "md5" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - bytes: 13, - hash: actualHash, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should return multiple errors when both bytes and hash do not match", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "md5" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - bytes: 1024, - hash: "md5:wronghash", - }) - - expect(report.valid).toBe(false) - expect(report.errors).toHaveLength(2) - expect(report.errors[0]).toEqual({ - type: "file/bytes", - bytes: 1024, - actualBytes: 13, - }) - expect(report.errors[1]).toEqual({ - type: "file/hash", - hash: "md5:wronghash", - actualHash, - }) - }) - - it("should return error when only bytes mismatch", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const actualHash = await inferHash( - { path: tempFilePath }, - { hashType: "md5" }, - ) - - const report = await validateFile({ - path: "https://example.com/file.txt", - bytes: 1024, - hash: actualHash, - }) - - expect(report.valid).toBe(false) - expect(report.errors).toHaveLength(1) - expect(report.errors[0]?.type).toBe("file/bytes") - }) - - it("should return error when only hash mismatch", async () => { - const tempFilePath = await writeTempFile("Hello, World!") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const report = await validateFile({ - path: "https://example.com/file.txt", - bytes: 13, - hash: "md5:wronghash", - }) - - expect(report.valid).toBe(false) - expect(report.errors).toHaveLength(1) - expect(report.errors[0]?.type).toBe("file/hash") - }) - - it("should handle local file paths", async () => { - const tempFilePath = await writeTempFile("x".repeat(2048)) - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const report = await validateFile({ - path: "/local/path/file.txt", - bytes: 2048, - }) - - expect(mockPrefetchFiles).toHaveBeenCalledWith("/local/path/file.txt") - expect(report).toEqual({ valid: true, errors: [] }) - }) - - it("should handle empty file validation", async () => { - const tempFilePath = await writeTempFile("") - mockPrefetchFiles.mockResolvedValue([tempFilePath]) - - const report = await validateFile({ - path: "https://example.com/empty.txt", - bytes: 0, - }) - - expect(report).toEqual({ valid: true, errors: [] }) - }) -}) diff --git a/dataset/file/validate.ts b/dataset/file/validate.ts deleted file mode 100644 index fc6b619b..00000000 --- a/dataset/file/validate.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { FileError } from "@dpkit/metadata" -import { createReport } from "@dpkit/metadata" -import type { Resource } from "@dpkit/metadata" -import { prefetchFiles } from "./fetch.ts" -import { inferBytes, inferEncoding, inferHash } from "./infer.ts" - -export async function validateFile(resource: Partial) { - const errors: FileError[] = [] - const localPaths = await prefetchFiles(resource.path) - - if (resource.bytes) { - const bytes = resource.bytes - const actualBytes = await inferBytes({ path: localPaths }) - - if (bytes !== actualBytes) { - errors.push({ - type: "file/bytes", - bytes, - actualBytes, - }) - } - } - - if (resource.hash) { - const [hashValue, hashType = "md5"] = resource.hash.split(":").toReversed() - - const hash = `${hashType}:${hashValue}` - const actualHash = await inferHash( - { path: localPaths }, - { - hashType: hashType as any, - }, - ) - - if (hash !== actualHash) { - errors.push({ - type: "file/hash", - hash, - actualHash, - }) - } - } - - if (resource.encoding) { - const encoding = resource.encoding - const actualEncoding = await inferEncoding({ path: localPaths }) - - if (actualEncoding) { - if (encoding !== actualEncoding) { - errors.push({ - type: "file/encoding", - encoding, - actualEncoding, - }) - } - } - } - - return createReport(errors) -} diff --git a/dataset/folder/create.ts b/dataset/folder/create.ts deleted file mode 100644 index 1e0fc83d..00000000 --- a/dataset/folder/create.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { mkdir } from "node:fs/promises" - -export async function createFolder(path: string) { - await mkdir(path, { recursive: true }) -} diff --git a/dataset/folder/index.ts b/dataset/folder/index.ts deleted file mode 100644 index ea9a6587..00000000 --- a/dataset/folder/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { createFolder } from "./create.ts" -export { getTempFolderPath } from "./temp.ts" diff --git a/dataset/folder/temp.ts b/dataset/folder/temp.ts deleted file mode 100644 index b31bb997..00000000 --- a/dataset/folder/temp.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { rmSync } from "node:fs" -import exitHook from "exit-hook" -import { temporaryDirectory } from "tempy" - -export function getTempFolderPath(options?: { persist?: boolean }) { - const path = temporaryDirectory() - - if (!options?.persist) { - exitHook(() => { - try { - rmSync(path, { recursive: true, force: true }) - } catch {} - }) - } - - return path -} diff --git a/dataset/index.ts b/dataset/index.ts deleted file mode 100644 index 0cb13d03..00000000 --- a/dataset/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -export type { DatasetPlugin } from "./plugin.ts" -export type { SavePackageOptions } from "./plugin.ts" - -export { CkanPlugin } from "./plugins/ckan/index.ts" -export { DatahubPlugin } from "./plugins/datahub/index.ts" -export { DescriptorPlugin } from "./plugins/descriptor/index.ts" -export { FolderPlugin } from "./plugins/folder/index.ts" -export { GithubPlugin } from "./plugins/github/index.ts" -export { ZenodoPlugin } from "./plugins/zenodo/index.ts" -export { ZipPlugin } from "./plugins/zip/index.ts" - -export { assertLocalPathVacant } from "./file/index.ts" -export { copyFile } from "./file/index.ts" -export { describeFile } from "./file/index.ts" -export { getTempFilePath } from "./file/index.ts" -export { getTempFolderPath } from "./folder/index.ts" -export { inferBytes } from "./file/index.ts" -export { inferEncoding } from "./file/index.ts" -export { inferHash } from "./file/index.ts" -export { isLocalPathExist } from "./file/index.ts" -export { loadFile } from "./file/index.ts" -export { loadFileStream } from "./stream/index.ts" -export { loadPackageFromCkan } from "./plugins/ckan/index.ts" -export { loadPackageFromDatahub } from "./plugins/datahub/index.ts" -export { loadPackageFromFolder } from "./plugins/folder/index.ts" -export { loadPackageFromGithub } from "./plugins/github/index.ts" -export { loadPackageFromZenodo } from "./plugins/zenodo/index.ts" -export { loadPackageFromZip } from "./plugins/zip/index.ts" -export { prefetchFile } from "./file/index.ts" -export { prefetchFiles } from "./file/index.ts" -export { saveFile } from "./file/index.ts" -export { savePackageToCkan } from "./plugins/ckan/index.ts" -export { savePackageToFolder } from "./plugins/folder/index.ts" -export { savePackageToGithub } from "./plugins/github/index.ts" -export { savePackageToZenodo } from "./plugins/zenodo/index.ts" -export { savePackageToZip } from "./plugins/zip/index.ts" -export { validateFile } from "./file/index.ts" -export { writeTempFile } from "./file/index.ts" diff --git a/dataset/package.json b/dataset/package.json deleted file mode 100644 index a7014bf3..00000000 --- a/dataset/package.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "name": "@dpkit/dataset", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "dataset", - "ckan", - "datahub" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "chardet": "^2.1.0", - "exit-hook": "^4.0.0", - "fflate": "^0.8.2", - "hasha": "^6.0.0", - "isbinaryfile": "^5.0.4", - "multistream": "^4.1.0", - "p-all": "^5.0.1", - "p-map": "^7.0.3", - "tempy": "3.1.0", - "tiny-invariant": "^1.3.3" - }, - "devDependencies": { - "@types/multistream": "4.1.3" - } -} diff --git a/dataset/package/index.ts b/dataset/package/index.ts deleted file mode 100644 index 933dd45b..00000000 --- a/dataset/package/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { getPackageBasepath } from "./path.ts" -export { mergePackages } from "./merge.ts" diff --git a/dataset/package/merge.ts b/dataset/package/merge.ts deleted file mode 100644 index cf5ec041..00000000 --- a/dataset/package/merge.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { loadPackageDescriptor } from "@dpkit/metadata" - -/** - * Merges a system data package into a user data package if provided - */ -export async function mergePackages(options: { - systemPackage: Package - userPackagePath?: string -}) { - const systemPackage = options.systemPackage - - const userPackage = options.userPackagePath - ? await loadPackageDescriptor(options.userPackagePath) - : undefined - - return { ...systemPackage, ...userPackage } -} diff --git a/dataset/package/path.spec.ts b/dataset/package/path.spec.ts deleted file mode 100644 index 639bb1db..00000000 --- a/dataset/package/path.spec.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { describe, expect, it } from "vitest" -import { getCommonLocalBasepath } from "./path.ts" - -describe("getCommonLocalBasepath", () => { - it.each([ - { - description: "same directory different files", - paths: ["data/table1.csv", "data/table2.csv"], - basepath: "data", - }, - { - description: "nested directories", - paths: [ - "data/nested/file1.csv", - "data/nested/file2.csv", - "data/file3.csv", - ], - basepath: "data", - }, - { - description: "single path", - paths: ["data/file.csv"], - basepath: "data", - }, - { - description: "root level files", - paths: ["file1.csv", "file2.csv"], - basepath: "", - }, - { - description: "different top-level directories", - paths: ["data1/file1.csv", "data2/file2.csv"], - basepath: "", - }, - { - description: "empty paths array", - paths: [], - basepath: undefined, - }, - { - description: "some path are remote", - paths: ["https://example.com/table.csv", "data/table.csv"], - basepath: "data", - }, - { - description: "all paths are remote", - paths: [ - "https://example.com/table1.csv", - "https://example.com/table2.csv", - ], - basepath: undefined, - }, - ])("$description", ({ paths, basepath }) => { - expect(getCommonLocalBasepath(paths)).toEqual(basepath) - }) -}) diff --git a/dataset/package/path.ts b/dataset/package/path.ts deleted file mode 100644 index c46c78fb..00000000 --- a/dataset/package/path.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { join, relative, resolve, sep } from "node:path" -import { type Package, getBasepath, isRemotePath } from "@dpkit/metadata" - -export function getPackageBasepath(dataPackage: Package) { - const paths: string[] = [] - - for (const resource of dataPackage.resources) { - if (!resource.path) { - continue - } - - const resourcePaths = Array.isArray(resource.path) - ? resource.path - : [resource.path] - - paths.push(...resourcePaths) - } - - return getCommonLocalBasepath(paths) -} - -export function getCommonLocalBasepath(paths: string[]) { - const absoluteBasepaths = paths - .filter(path => !isRemotePath(path)) - .map(path => resolve(getBasepath(path))) - - if (!absoluteBasepaths.length) { - return undefined - } - - // On Unix it split the root fs as an empty segment - const segmentTable = absoluteBasepaths.map(path => - path.split(sep).map(segment => segment || "/"), - ) - - let column = 0 - const segments: string[] = [] - - while (true) { - const segmentColumn = segmentTable.map(segments => segments[column]) - const uniqueSegments = new Set(segmentColumn) - - if (uniqueSegments.size !== 1) break - if (!segmentColumn[0]) break - - column++ - segments.push(segmentColumn[0]) - } - - if (!segments.length) { - throw new Error("Cannot find common basepath") - } - - const basepath = relative(process.cwd(), join(...segments)) - return basepath -} diff --git a/dataset/plugin.ts b/dataset/plugin.ts deleted file mode 100644 index ca49dd50..00000000 --- a/dataset/plugin.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Package } from "@dpkit/metadata" - -export type SavePackageOptions = { - target: string - withRemote?: boolean -} - -export interface DatasetPlugin { - loadPackage?(source: string): Promise - - savePackage?( - dataPackage: Package, - options: SavePackageOptions, - ): Promise<{ path?: string } | undefined> -} diff --git a/dataset/plugins/ckan/ckan/index.ts b/dataset/plugins/ckan/ckan/index.ts deleted file mode 100644 index f149c005..00000000 --- a/dataset/plugins/ckan/ckan/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { makeCkanApiRequest } from "./request.ts" diff --git a/dataset/plugins/ckan/ckan/request.ts b/dataset/plugins/ckan/ckan/request.ts deleted file mode 100644 index 8e8c8cf9..00000000 --- a/dataset/plugins/ckan/ckan/request.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { Descriptor } from "@dpkit/metadata" - -export async function makeCkanApiRequest(options: { - ckanUrl: string - action: string - payload: Descriptor - upload?: { name: string; data: Blob } - apiKey?: string -}) { - let body: string | FormData - const headers: Record = {} - - const url = new URL(options.ckanUrl) - url.pathname = `/api/3/action/${options.action}` - - if (options.apiKey) { - headers.Authorization = options.apiKey - } - - if (options.upload) { - body = new FormData() - body.append("upload", options.upload.data, options.upload.name) - - for (const [key, value] of Object.entries(options.payload)) { - body.append(key, value) - } - } else { - body = JSON.stringify(options.payload) - headers["Content-Type"] = "application/json" - } - - const response = await fetch(url.toString(), { - method: "POST", - headers, - body, - }) - - if (!response.ok) { - const errorText = await response.text() - throw new Error( - `CKAN API error: ${response.status} ${response.statusText}\n${errorText}`, - ) - } - - const data = (await response.json()) as Descriptor - if (!data.success) { - throw new Error(`CKAN API error: ${data.error}`) - } - - return data.result as T -} diff --git a/dataset/plugins/ckan/index.ts b/dataset/plugins/ckan/index.ts deleted file mode 100644 index 9f97e644..00000000 --- a/dataset/plugins/ckan/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./package/index.ts" -export * from "./resource/index.ts" -export * from "./schema/index.ts" -export * from "./plugin.ts" diff --git a/dataset/plugins/ckan/package/Organization.ts b/dataset/plugins/ckan/package/Organization.ts deleted file mode 100644 index 46d9db27..00000000 --- a/dataset/plugins/ckan/package/Organization.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * CKAN Organization interface - */ -export interface CkanOrganization { - /** - * Organization identifier - */ - id: string - - /** - * Organization name - */ - name: string - - /** - * Organization title - */ - title: string - - /** - * Organization description - */ - description: string -} diff --git a/dataset/plugins/ckan/package/Package.ts b/dataset/plugins/ckan/package/Package.ts deleted file mode 100644 index c0fa7413..00000000 --- a/dataset/plugins/ckan/package/Package.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { CkanResource } from "../resource/index.ts" -import type { CkanOrganization } from "./Organization.ts" -import type { CkanTag } from "./Tag.ts" - -/** - * CKAN Package interface - */ -export interface CkanPackage { - /** - * List of resources - */ - resources: CkanResource[] - - /** - * Organization information - */ - organization?: CkanOrganization - - /** - * List of tags - */ - tags: CkanTag[] - - /** - * Package identifier - */ - id: string - - /** - * Package name - */ - name: string - - /** - * Package title - */ - title?: string - - /** - * Package notes/description - */ - notes?: string - - /** - * Package version - */ - version?: string - - /** - * License identifier - */ - license_id?: string - - /** - * License title - */ - license_title?: string - - /** - * License URL - */ - license_url?: string - - /** - * Package author - */ - author?: string - - /** - * Package author email - */ - author_email?: string - - /** - * Package maintainer - */ - maintainer?: string - - /** - * Package maintainer email - */ - maintainer_email?: string - - /** - * Metadata creation timestamp - */ - metadata_created?: string - - /** - * Metadata modification timestamp - */ - metadata_modified?: string -} diff --git a/dataset/plugins/ckan/package/Tag.ts b/dataset/plugins/ckan/package/Tag.ts deleted file mode 100644 index 33ddd7a6..00000000 --- a/dataset/plugins/ckan/package/Tag.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * CKAN Tag interface - */ -export interface CkanTag { - /** - * Tag identifier - */ - id: string - - /** - * Tag name - */ - name: string - - /** - * Tag display name - */ - display_name: string -} diff --git a/dataset/plugins/ckan/package/convert/fromCkan.spec.ts b/dataset/plugins/ckan/package/convert/fromCkan.spec.ts deleted file mode 100644 index 5eeb0a27..00000000 --- a/dataset/plugins/ckan/package/convert/fromCkan.spec.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, expect, it } from "vitest" -import type { CkanPackage } from "../Package.ts" -import ckanPackageFixture from "../fixtures/ckan-package.json" with { - type: "json", -} -import { convertPackageFromCkan } from "./fromCkan.ts" - -describe("convertPackageFromCkan", () => { - it("converts a CKAN package to a Frictionless Data Package", () => { - const ckanPackage = ckanPackageFixture as CkanPackage - - const result = convertPackageFromCkan(ckanPackage) - - expect(result.name).toEqual(ckanPackage.name) - expect(result.title).toEqual(ckanPackage.title) - expect(result.description).toEqual(ckanPackage.notes) - expect(result.version).toEqual(ckanPackage.version) - expect(result.created).toEqual(ckanPackage.metadata_created) - - expect(result.licenses).toHaveLength(1) - if (result.licenses && result.licenses.length > 0 && result.licenses[0]) { - const license = result.licenses[0] - if (ckanPackage.license_id) - expect(license.name).toEqual(ckanPackage.license_id) - if (ckanPackage.license_title) - expect(license.title).toEqual(ckanPackage.license_title) - if (ckanPackage.license_url) - expect(license.path).toEqual(ckanPackage.license_url) - } - - expect(result.contributors).toHaveLength(2) - if (result.contributors) { - const author = result.contributors.find(c => c.role === "author") - expect(author).toBeTruthy() - if (author) { - expect(author.title).toEqual(ckanPackage.author) - expect(author.email).toEqual(ckanPackage.author_email) - } - - const maintainer = result.contributors.find(c => c.role === "maintainer") - expect(maintainer).toBeTruthy() - if (maintainer) { - expect(maintainer.title).toEqual(ckanPackage.maintainer) - expect(maintainer.email).toEqual(ckanPackage.maintainer_email) - } - } - - expect(result.keywords).toHaveLength(ckanPackage.tags.length) - expect(result.keywords).toEqual(ckanPackage.tags.map(tag => tag.name)) - - expect(result.resources).toHaveLength(ckanPackage.resources.length) - - const firstCkanResource = ckanPackage.resources[0] - const firstResource = result.resources[0] - - expect(firstCkanResource).toBeDefined() - expect(firstResource).toBeDefined() - - if (firstResource && firstCkanResource) { - expect(firstResource.path).toEqual(firstCkanResource.url) - expect(firstResource.name).toMatch(/^sample[-_]linked[-_]csv$/) - expect(firstResource.description).toEqual(firstCkanResource.description) - expect(firstResource.format).toEqual( - firstCkanResource.format?.toLowerCase(), - ) - expect(firstResource.mediatype).toEqual(firstCkanResource.mimetype) - } - }) - - it("handles empty resources array", () => { - const ckanPackage: CkanPackage = { - ...(ckanPackageFixture as CkanPackage), - resources: [], - } - - const result = convertPackageFromCkan(ckanPackage) - - expect(result.resources).toEqual([]) - }) - - it("handles undefined optional properties", () => { - const ckanPackage: Partial = { - resources: [], - tags: [], - } - - const result = convertPackageFromCkan(ckanPackage as CkanPackage) - - expect(result.name).toBeUndefined() - expect(result.title).toBeUndefined() - expect(result.description).toBeUndefined() - expect(result.version).toBeUndefined() - expect(result.created).toBeUndefined() - expect(result.licenses).toBeUndefined() - expect(result.contributors).toBeUndefined() - expect(result.keywords).toBeUndefined() - expect(result.resources).toEqual([]) - }) -}) diff --git a/dataset/plugins/ckan/package/convert/fromCkan.ts b/dataset/plugins/ckan/package/convert/fromCkan.ts deleted file mode 100644 index 51a91495..00000000 --- a/dataset/plugins/ckan/package/convert/fromCkan.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { Contributor, Package } from "@dpkit/metadata" -import type { License } from "@dpkit/metadata" -import { convertResourceFromCkan } from "../../resource/index.ts" -import type { CkanPackage } from "../Package.ts" - -export function convertPackageFromCkan(ckanPackage: CkanPackage): Package { - const datapackage: Package = { - name: ckanPackage.name, - resources: [], - } - - if (ckanPackage.title) { - datapackage.title = ckanPackage.title - } - - if (ckanPackage.notes) { - datapackage.description = ckanPackage.notes - } - - if (ckanPackage.version) { - datapackage.version = ckanPackage.version - } - - if (ckanPackage.resources && ckanPackage.resources.length > 0) { - datapackage.resources = ckanPackage.resources.map(resource => - convertResourceFromCkan(resource), - ) - } - - if (ckanPackage.license_id) { - const license: License = { - name: ckanPackage.license_id, - } - - if (ckanPackage.license_title) { - license.title = ckanPackage.license_title - } - - if (ckanPackage.license_url) { - license.path = ckanPackage.license_url - } - - datapackage.licenses = [license] - } - - const contributors: Contributor[] = [] - - if (ckanPackage.author) { - const authorContributor: Contributor = { - title: ckanPackage.author, - role: "author", - } - - if (ckanPackage.author_email) { - authorContributor.email = ckanPackage.author_email - } - - contributors.push(authorContributor) - } - - if (ckanPackage.maintainer) { - const maintainerContributor: Contributor = { - title: ckanPackage.maintainer, - role: "maintainer", - } - - if (ckanPackage.maintainer_email) { - maintainerContributor.email = ckanPackage.maintainer_email - } - - contributors.push(maintainerContributor) - } - - if (contributors.length > 0) { - datapackage.contributors = contributors - } - - if (ckanPackage.tags && ckanPackage.tags.length > 0) { - datapackage.keywords = ckanPackage.tags.map(tag => tag.name) - } - - if (ckanPackage.metadata_created) { - datapackage.created = ckanPackage.metadata_created - } - - return datapackage -} diff --git a/dataset/plugins/ckan/package/convert/toCkan.spec.ts b/dataset/plugins/ckan/package/convert/toCkan.spec.ts deleted file mode 100644 index ce15d915..00000000 --- a/dataset/plugins/ckan/package/convert/toCkan.spec.ts +++ /dev/null @@ -1,196 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { describe, expect, it } from "vitest" -import type { CkanPackage } from "../Package.ts" -import ckanPackageFixture from "../fixtures/ckan-package.json" with { - type: "json", -} -import { convertPackageFromCkan } from "./fromCkan.ts" -import { convertPackageToCkan } from "./toCkan.ts" - -describe("convertPackageToCkan", () => { - it("converts a Frictionless Data Package to a CKAN package", () => { - const dataPackage: Package = { - name: "test-package", - title: "Test Package", - description: "This is a test package", - version: "1.0.0", - licenses: [ - { - name: "cc-by", - title: "Creative Commons Attribution", - path: "http://www.opendefinition.org/licenses/cc-by", - }, - ], - contributors: [ - { - title: "Test Author", - email: "author@example.com", - role: "author", - }, - { - title: "Test Maintainer", - email: "maintainer@example.com", - role: "maintainer", - }, - ], - keywords: ["test", "sample", "data"], - created: "2023-01-01T00:00:00Z", - resources: [ - { - name: "test-resource", - path: "https://example.com/data.csv", - format: "csv", - mediatype: "text/csv", - description: "Test resource", - bytes: 1024, - hash: "md5:1234567890abcdef", - }, - ], - } - - const result = convertPackageToCkan(dataPackage) - - expect(result.name).toEqual(dataPackage.name) - expect(result.title).toEqual(dataPackage.title) - expect(result.notes).toEqual(dataPackage.description) - expect(result.version).toEqual(dataPackage.version) - - if ( - dataPackage.licenses && - dataPackage.licenses.length > 0 && - dataPackage.licenses[0] - ) { - const license = dataPackage.licenses[0] - if (license.name) expect(result.license_id).toEqual(license.name) - if (license.title) expect(result.license_title).toEqual(license.title) - if (license.path) expect(result.license_url).toEqual(license.path) - } - - if (dataPackage.contributors && dataPackage.contributors.length >= 2) { - const author = dataPackage.contributors.find(c => c.role === "author") - const maintainer = dataPackage.contributors.find( - c => c.role === "maintainer", - ) - - if (author) { - expect(result.author).toEqual(author.title) - expect(result.author_email).toEqual(author.email) - } - - if (maintainer) { - expect(result.maintainer).toEqual(maintainer.title) - expect(result.maintainer_email).toEqual(maintainer.email) - } - } - - if (dataPackage.keywords && dataPackage.keywords.length > 0) { - expect(result.tags).toHaveLength(dataPackage.keywords.length) - dataPackage.keywords.forEach((keyword, index) => { - const tag = result.tags?.[index] - if (tag && keyword) { - expect(tag.name).toEqual(keyword) - expect(tag.display_name).toEqual(keyword) - } - }) - } - - expect(result.resources).toHaveLength(dataPackage.resources.length) - - expect(dataPackage.resources.length).toBeGreaterThan(0) - expect(result.resources?.length).toBeGreaterThan(0) - - if (dataPackage.resources.length > 0 && result.resources.length > 0) { - const firstResource = dataPackage.resources[0] - const firstCkanResource = result.resources[0] - - expect(firstCkanResource).toBeDefined() - expect(firstResource).toBeDefined() - - if (firstResource && firstCkanResource) { - expect(firstCkanResource.description).toEqual(firstResource.description) - expect(firstCkanResource.format).toEqual( - firstResource.format?.toUpperCase(), - ) - expect(firstCkanResource.mimetype).toEqual(firstResource.mediatype) - expect(firstCkanResource.size).toEqual(firstResource.bytes) - expect(firstCkanResource.hash).toEqual(firstResource.hash) - } - } - }) - - it("handles empty resources array", () => { - const dataPackage: Package = { - name: "test-package", - resources: [], - } - - const result = convertPackageToCkan(dataPackage) - - expect(result.resources).toEqual([]) - }) - - it("handles undefined optional properties", () => { - const dataPackage: Package = { - resources: [], - } - - const result = convertPackageToCkan(dataPackage) - - expect(result.name).toBeUndefined() - expect(result.title).toBeUndefined() - expect(result.notes).toBeUndefined() - expect(result.version).toBeUndefined() - expect(result.metadata_created).toBeUndefined() - expect(result.license_id).toBeUndefined() - expect(result.license_title).toBeUndefined() - expect(result.license_url).toBeUndefined() - expect(result.author).toBeUndefined() - expect(result.author_email).toBeUndefined() - expect(result.maintainer).toBeUndefined() - expect(result.maintainer_email).toBeUndefined() - expect(result.tags).toEqual([]) - expect(result.resources).toEqual([]) - }) - - it("performs a round-trip conversion (CKAN → DP → CKAN)", () => { - const originalCkanPackage = ckanPackageFixture as CkanPackage - - const dataPackage = convertPackageFromCkan(originalCkanPackage) - - const resultCkanPackage = convertPackageToCkan(dataPackage) - - expect(resultCkanPackage.name).toEqual(originalCkanPackage.name) - expect(resultCkanPackage.title).toEqual(originalCkanPackage.title) - expect(resultCkanPackage.notes).toEqual(originalCkanPackage.notes) - expect(resultCkanPackage.version).toEqual(originalCkanPackage.version) - - expect(resultCkanPackage.license_id).toEqual(originalCkanPackage.license_id) - expect(resultCkanPackage.license_title).toEqual( - originalCkanPackage.license_title, - ) - expect(resultCkanPackage.license_url).toEqual( - originalCkanPackage.license_url, - ) - - expect(resultCkanPackage.author).toEqual(originalCkanPackage.author) - expect(resultCkanPackage.author_email).toEqual( - originalCkanPackage.author_email, - ) - expect(resultCkanPackage.maintainer).toEqual(originalCkanPackage.maintainer) - expect(resultCkanPackage.maintainer_email).toEqual( - originalCkanPackage.maintainer_email, - ) - - expect(resultCkanPackage.resources.length).toBeGreaterThan(0) - - expect(resultCkanPackage.tags.length).toEqual( - originalCkanPackage.tags.length, - ) - originalCkanPackage.tags.forEach(originalTag => { - const matchingTag = resultCkanPackage.tags.find( - tag => tag.name === originalTag.name, - ) - expect(matchingTag).toBeTruthy() - }) - }) -}) diff --git a/dataset/plugins/ckan/package/convert/toCkan.ts b/dataset/plugins/ckan/package/convert/toCkan.ts deleted file mode 100644 index a1c58826..00000000 --- a/dataset/plugins/ckan/package/convert/toCkan.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import type { SetRequired } from "type-fest" -import type { CkanResource } from "../../resource/Resource.ts" -import { convertResourceToCkan } from "../../resource/index.ts" -import type { CkanPackage } from "../Package.ts" -import type { CkanTag } from "../Tag.ts" - -export function convertPackageToCkan(dataPackage: Package) { - const ckanPackage: SetRequired, "resources" | "tags"> = { - resources: [], - tags: [], - } - - if (dataPackage.name) ckanPackage.name = dataPackage.name - if (dataPackage.title) ckanPackage.title = dataPackage.title - if (dataPackage.description) ckanPackage.notes = dataPackage.description - if (dataPackage.version) ckanPackage.version = dataPackage.version - - if (dataPackage.licenses && dataPackage.licenses.length > 0) { - const license = dataPackage.licenses[0] - - if (license?.name) ckanPackage.license_id = license.name - if (license?.title) ckanPackage.license_title = license.title - if (license?.path) ckanPackage.license_url = license.path - } - - if (dataPackage.contributors) { - const author = dataPackage.contributors.find(c => c.role === "author") - if (author) { - ckanPackage.author = author.title - if (author.email) ckanPackage.author_email = author.email - } - - const maintainer = dataPackage.contributors.find( - c => c.role === "maintainer", - ) - if (maintainer) { - ckanPackage.maintainer = maintainer.title - if (maintainer.email) ckanPackage.maintainer_email = maintainer.email - } - } - - if (dataPackage.resources && dataPackage.resources.length > 0) { - ckanPackage.resources = dataPackage.resources - .map(resource => convertResourceToCkan(resource)) - .filter((resource): resource is CkanResource => resource !== undefined) - } - - if (dataPackage.keywords && dataPackage.keywords.length > 0) { - ckanPackage.tags = dataPackage.keywords.map(keyword => ({ - name: keyword, - display_name: keyword, - })) as CkanTag[] - } - - return ckanPackage -} diff --git a/dataset/plugins/ckan/package/fixtures/ckan-package.json b/dataset/plugins/ckan/package/fixtures/ckan-package.json deleted file mode 100644 index 87a8b44a..00000000 --- a/dataset/plugins/ckan/package/fixtures/ckan-package.json +++ /dev/null @@ -1,308 +0,0 @@ -{ - "author": "Test Author", - "author_email": "test@email.com", - "creator_user_id": "47c7f1b1-0ef5-4d7b-b43c-811c51c9e349", - "id": "c322307a-b871-44fe-a602-32ee8437ff04", - "isopen": true, - "license_id": "cc-by", - "license_title": "Creative Commons Attribution", - "license_url": "http://www.opendefinition.org/licenses/cc-by", - "maintainer": "Test Maintainer", - "maintainer_email": "test@email.com", - "metadata_created": "2021-04-09T11:39:37.657233", - "metadata_modified": "2022-05-20T09:20:43.998956", - "name": "sample-dataset-1", - "notes": "A CKAN Dataset is a collection of data resources (such as files), together with a description and other information (what is known as metadata), at a fixed URL. \r\n\r\n", - "num_resources": 9, - "num_tags": 8, - "organization": { - "id": "1fa89238-ee96-4439-a885-22d15244d070", - "name": "sample-organization", - "title": "Sample Organization", - "type": "organization", - "description": "This is a sample organization.", - "image_url": "2022-05-20-084702.929838siurana.jpg", - "created": "2021-04-09T14:27:17.753798", - "is_organization": true, - "approval_status": "approved", - "state": "active" - }, - "owner_org": "1fa89238-ee96-4439-a885-22d15244d070", - "private": false, - "state": "active", - "title": "Sample Dataset", - "type": "dataset", - "url": "", - "version": "1.0", - "groups": [ - { - "description": "", - "display_name": "Test Group", - "id": "5d423f6b-137e-4d15-a156-868763fa7a64", - "image_display_url": "https://demo.ckan.org/uploads/group/2021-04-21-153504.571229064c7c.png", - "name": "test-group", - "title": "Test Group" - } - ], - "resources": [ - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-09T14:31:09.032858", - "datastore_active": true, - "description": "This is a sample resource added via url.", - "format": "CSV", - "hash": "", - "id": "e687245d-7835-44b0-8ed3-0827de123895", - "last_modified": null, - "metadata_modified": "2021-04-09T14:31:09.021596", - "mimetype": "text/csv", - "mimetype_inner": null, - "name": "sample-linked.csv", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 0, - "resource_type": null, - "size": null, - "state": "active", - "url": "https://raw.githubusercontent.com/datopian/CKAN_Demo_Datasets/main/resources/org1_sample.csv", - "url_type": null - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-09T14:31:45.092631", - "datastore_active": true, - "description": "Sample csv (uploaded).", - "format": "CSV", - "hash": "", - "id": "b53c9e72-6b59-4cda-8c0c-7d6a51dad12a", - "last_modified": "2021-04-09T16:13:57.353205", - "metadata_modified": "2021-04-09T16:13:57.367140", - "mimetype": "application/csv", - "mimetype_inner": null, - "name": "sample.csv", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 1, - "resource_type": null, - "size": 6731, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/b53c9e72-6b59-4cda-8c0c-7d6a51dad12a/download/sample.csv", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-09T16:21:17.140402", - "datastore_active": true, - "description": "Sample views for csv.", - "format": "CSV", - "hash": "", - "id": "9ce6650b-6ff0-4a52-9b10-09cfc29bbd7e", - "last_modified": "2021-04-09T16:21:17.106693", - "metadata_modified": "2021-04-13T12:41:06.751746", - "mimetype": null, - "mimetype_inner": null, - "name": "views.csv", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 2, - "resource_type": null, - "size": 32773, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/9ce6650b-6ff0-4a52-9b10-09cfc29bbd7e/download/co2-mm-mlo_csv.csv", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-09T14:49:24.711541", - "datastore_active": false, - "description": "Sample pdf file.", - "format": "PDF", - "hash": "", - "id": "8aa53505-3b7f-4b9c-9b54-cf674eadc3f1", - "last_modified": "2021-04-09T16:11:46.261373", - "metadata_modified": "2021-04-13T12:39:41.141419", - "mimetype": null, - "mimetype_inner": null, - "name": "sample.pdf", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 3, - "resource_type": null, - "size": 712352, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/8aa53505-3b7f-4b9c-9b54-cf674eadc3f1/download/sample.pdf", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-09T16:17:05.189302", - "datastore_active": false, - "description": "Sample txt file.", - "format": "TXT", - "hash": "", - "id": "0185907b-2812-437f-9c64-eae24771ef5f", - "last_modified": "2021-04-09T16:17:05.136426", - "metadata_modified": "2021-04-13T12:39:24.524530", - "mimetype": null, - "mimetype_inner": null, - "name": "sample.txt", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 4, - "resource_type": null, - "size": 85, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/0185907b-2812-437f-9c64-eae24771ef5f/download/sample.txt", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-13T12:19:02.178513", - "datastore_active": false, - "description": "Sample GeoJSON resource for the list of countries.\r\n\r\nResource taken from https://openlayers.org/ \r\nLicensed under the 2-Clause BSD (https://www.tldrlegal.com/l/freebsd)", - "format": "GeoJSON", - "hash": "", - "id": "ecd4a62d-998b-46e4-8a64-cadac2125c64", - "last_modified": "2021-04-13T12:19:01.921374", - "metadata_modified": "2021-04-13T12:29:29.067536", - "mimetype": null, - "mimetype_inner": null, - "name": "sample.geojson", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 5, - "resource_type": null, - "size": 255943, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/ecd4a62d-998b-46e4-8a64-cadac2125c64/download/countries.geojson", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2021-04-13T12:32:17.823512", - "datastore_active": false, - "description": "Sample kml file for Earthquakes of magnitude 5 in 2012.\r\n\r\nResource taken from https://openlayers.org/ \r\nLicensed under the 2-Clause BSD (https://www.tldrlegal.com/l/freebsd)\r\n", - "format": "KML", - "hash": "", - "id": "048333ab-9608-42dc-901b-a7dd9fca3dda", - "last_modified": "2021-04-13T12:32:17.769578", - "metadata_modified": "2021-04-13T12:40:09.731123", - "mimetype": null, - "mimetype_inner": null, - "name": "sample.kml", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 6, - "resource_type": null, - "size": 474000, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/048333ab-9608-42dc-901b-a7dd9fca3dda/download/2012_earthquakes_mag5.kml", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2022-04-11T18:35:24.435997", - "datastore_active": false, - "description": "", - "format": "JPEG", - "hash": "", - "id": "b6c22c1d-e789-490d-b935-989093bbb173", - "last_modified": "2022-04-11T18:35:24.381740", - "metadata_modified": "2022-05-20T09:03:31.574365", - "mimetype": "image/png", - "mimetype_inner": null, - "name": "avoid-crowds-when-buying-materials-social-media-post.jpeg", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 7, - "resource_type": null, - "size": 444695, - "state": "active", - "url": "https://demo.ckan.org/dataset/c322307a-b871-44fe-a602-32ee8437ff04/resource/b6c22c1d-e789-490d-b935-989093bbb173/download/avoid-crowds-when-buying-materials-social-media-post.jpeg", - "url_type": "upload" - }, - { - "cache_last_updated": null, - "cache_url": null, - "created": "2022-05-20T09:03:31.617635", - "datastore_active": false, - "description": "", - "format": "WMS", - "hash": "", - "id": "664e5e2c-bd7d-4972-a245-a747f7d61cc9", - "last_modified": null, - "metadata_modified": "2022-05-20T09:03:42.450846", - "mimetype": null, - "mimetype_inner": null, - "name": "Sample WMS", - "package_id": "c322307a-b871-44fe-a602-32ee8437ff04", - "position": 8, - "resource_type": null, - "size": null, - "state": "active", - "url": "https://geoserveis.icgc.cat/icc_mapesbase/wms/service?", - "url_type": null - } - ], - "tags": [ - { - "display_name": "csv", - "id": "b5e651dd-8f42-445c-b9c4-2f09a3268427", - "name": "csv", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "economy", - "id": "0c4f9ad5-a372-4bda-a59b-e560cf264b0f", - "name": "economy", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "geojson", - "id": "a91d1c52-bd37-40fc-a1f2-6610ac7f39f3", - "name": "geojson", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "kml", - "id": "f29e8c38-be23-430b-bae0-7898f59d0089", - "name": "kml", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "pdf", - "id": "a4ba9601-bfa5-4322-8a62-38a83f2348c1", - "name": "pdf", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "sample", - "id": "1e6aa0c9-0dfc-448a-8c13-b5fd8ab2fefd", - "name": "sample", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "txt", - "id": "19e5f926-d769-49cf-a82c-80870baa3528", - "name": "txt", - "state": "active", - "vocabulary_id": null - }, - { - "display_name": "wms", - "id": "1279dad1-9397-48e1-94b8-0999bac75aee", - "name": "wms", - "state": "active", - "vocabulary_id": null - } - ], - "extras": [], - "relationships_as_subject": [], - "relationships_as_object": [] -} diff --git a/dataset/plugins/ckan/package/fixtures/data.csv b/dataset/plugins/ckan/package/fixtures/data.csv deleted file mode 100644 index d888cb89..00000000 --- a/dataset/plugins/ckan/package/fixtures/data.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name,value -1,test,100 -2,example,200 diff --git a/dataset/plugins/ckan/package/fixtures/generated/load.spec.ts.snap b/dataset/plugins/ckan/package/fixtures/generated/load.spec.ts.snap deleted file mode 100644 index 0c226733..00000000 --- a/dataset/plugins/ckan/package/fixtures/generated/load.spec.ts.snap +++ /dev/null @@ -1,261 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`loadPackageFromCkan > should load a package 1`] = ` -{ - "contributors": [ - { - "role": "author", - "title": "Natural History Museum", - }, - ], - "created": "2022-03-29T17:19:40.385253", - "description": "The records in this dataset provide a high-level summary of the Natural History Museum's collection and encompass the material managed by its Life Sciences, Earth Sciences and Library & Archives departments. The data has been gathered, structured and published with the intention of providing some level of visibility into the scope, nature and composition of the Museum's holdings, the majority of which have yet to be digitised. The 2,800+ records in this dataset represent the majority of the NHM collection, approximately 78 million items, and should be treated as an index or discovery tool: they are an indicative view of the Museum's holdings, a source of high-level figures and a signpost towards the relevant curatorial team to contact for more in-depth information. - -## Descriptive fields - -Each record in the **JtD collection-level records** table describes a group of objects, referred to as a 'collection unit'. Collection units can range in size from a handful of objects to several million and are defined by a set of high-level characteristics common to the objects within them (e.g., taxonomy, straitgraphy, item type, preservation method). Controlled vocabularies and external authority lists underpin descriptive fields wherever possible: see **Data dictionary** for field-level definitions and guidance for all fields used in this dataset. - -## Collection metrics - -The count fields indicate the scale of each collection unit in terms of the estimated quantity of 'items' (specimen-level or equivalent) and 'curatorial units' (objects physically managed during curation activities, such as jars or boxes) it holds. The 'reporting count' field should be used when aggregating multiple units and is the default collection size metric used by the Museum both internally and externally. - -Assessment scores are allocated by curatorial teams and are an attempt to indicate subjective or mutable qualities of each collection unit, such as outreach potential, scope and significance, physical accessibility and level of information available. See **Data dictionary** and **Join the Dots User Manual** for scoring definitions and examples. Scores are updated on an annual basis and are best viewed as a benchmark of the state of a collection unit/group of units: they are intended to allow us to track the impact of collection care, development and management activities over time and are not intended for use as a comparative measure between different areas of the collection. - -## Join the Dots Framework - -The criteria underpinning collection units and the methodology by which they are quantified and assessed is defined in the Join the Dots (JtD) cross–disciplinary collection assessment framework, developed and used at the NHM since 2018. For details on the full JtD framework see **Join the Dots User Manual** and the summary paper referenced below. Note that not all fields referenced in the JtD specification are included in this dataset: a subset is presented to avoid compromising the security of the collection and to provide a level of detail relevant to an external audience. - -## Further references - -- [NHM main website collection pages](https://www.nhm.ac.uk/our-science/collections.html) -- [Library & Archives Discovery Layer](https://www.nhm.ac.uk/our-science/departments-and-staff/library-and-archives.html) -- [Archives Catalogues](https://www.nhm.ac.uk/CalmView/default.aspx) -- [Specimen-level collection dataset](https://doi.org/10.5519/0002965) -- [Join the Dots: assessing a collection of 80 million items at The Natural History Museum, London, 2022](https://www.tandfonline.com/doi/full/10.1080/09647775.2021.2023900) - currently paywalled: contact the dataset curator for a copy", - "keywords": [ - "collection descriptions", - "collections assessment", - "curation", - "natural history collections", - ], - "licenses": [ - { - "name": "odc-by", - "path": "http://www.opendefinition.org/licenses/odc-by", - "title": "Open Data Commons Attribution License", - }, - ], - "name": "join-the-dots-collection-level-descriptions", - "resources": [ - { - "bytes": 896648, - "ckan:key": undefined, - "ckan:url": undefined, - "description": "Collection-level records describing the holdings of the Natural History Museum as of April 2025. See the dataset description and the other resources in this repository for more detail on navigating and interpreting this data: - -- [Data dictionary](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/09b2f07b-9e77-44b7-8645-ddcba1266629) -- [Join the Dots user manual](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/6920d9c9-6ea4-415c-95c8-54512a70346c) - ", - "format": "csv", - "mediatype": "text/csv", - "name": "jtd_collection-level_records", - "path": "https://data.nhm.ac.uk/dataset/78941769-a224-41d2-bdd5-527e9bbd9757/resource/660de433-447b-40ad-9aaf-77c98f8c2ba9/download/jtd_cld_public_dataset_20250425.csv", - "schema": { - "fields": [ - { - "name": "Collection unit ID", - "type": "number", - }, - { - "name": "Department", - "type": "string", - }, - { - "name": "Division", - "type": "string", - }, - { - "name": "Section", - "type": "string", - }, - { - "name": "Collection unit name", - "type": "string", - }, - { - "name": "Taxon", - "type": "string", - }, - { - "name": "Taxon rank", - "type": "string", - }, - { - "name": "Taxon ID", - "type": "string", - }, - { - "name": "Taxon ID source", - "type": "string", - }, - { - "name": "Informal taxon", - "type": "string", - }, - { - "name": "Type collection", - "type": "boolean", - }, - { - "name": "Geographic origin", - "type": "string", - }, - { - "name": "Earliest geological period", - "type": "string", - }, - { - "name": "Latest geological period", - "type": "string", - }, - { - "name": "Item type", - "type": "string", - }, - { - "name": "Preservation method", - "type": "string", - }, - { - "name": "Curatorial unit type", - "type": "string", - }, - { - "name": "Bibliographic level", - "type": "string", - }, - { - "name": "Fond reference", - "type": "string", - }, - { - "name": "Reporting count", - "type": "number", - }, - { - "name": "Reporting metric used", - "type": "string", - }, - { - "name": "Item count", - "type": "number", - }, - { - "name": "Item count confidence", - "type": "string", - }, - { - "name": "Curatorial unit count", - "type": "number", - }, - { - "name": "Curatorial unit count confidence", - "type": "string", - }, - { - "name": "C1: Physical accessibility", - "type": "number", - }, - { - "name": "S1: Strategy/mission/research", - "type": "number", - }, - { - "name": "S2: Scope and depth", - "type": "number", - }, - { - "name": "S3: Significance by comparison", - "type": "number", - }, - { - "name": "S4: Usage", - "type": "number", - }, - { - "name": "I1: Digitisation", - "type": "number", - }, - { - "name": "I2: Identification", - "type": "number", - }, - { - "name": "I4: Development potential", - "type": "number", - }, - { - "name": "O1: Education suitability", - "type": "number", - }, - { - "name": "O3: Exhibition suitability", - "type": "number", - }, - ], - }, - "type": "table", - }, - { - "bytes": 5868, - "ckan:key": undefined, - "ckan:url": undefined, - "description": "Description of the fields found in [JtD collection-level records](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/660de433-447b-40ad-9aaf-77c98f8c2ba9)", - "format": "csv", - "mediatype": "text/csv", - "name": "data_dictionary", - "path": "https://data.nhm.ac.uk/dataset/78941769-a224-41d2-bdd5-527e9bbd9757/resource/09b2f07b-9e77-44b7-8645-ddcba1266629/download/jtd-collection-level-records-data-dictionary.csv", - "schema": { - "fields": [ - { - "name": "fieldname", - "type": "string", - }, - { - "name": "description", - "type": "string", - }, - { - "name": "datatype", - "type": "string", - }, - { - "name": "constraints", - "type": "string", - }, - { - "name": "nullable", - "type": "boolean", - }, - { - "name": "Documentation reference", - "type": "string", - }, - ], - }, - "type": "table", - }, - { - "bytes": 8108818, - "ckan:key": undefined, - "ckan:url": undefined, - "description": "Documentation produced for NHM staff in 2018: a thorough overview of the Join the Dots data, processes and rationale. Some elements of the JtD framework have been amended slightly since this document was assembled, but it remains a useful reference for navigating the collection-level dataset presented in this package. ", - "format": "pdf", - "mediatype": "application/pdf", - "name": "join_the_dots_user_manual", - "path": "https://data.nhm.ac.uk/dataset/78941769-a224-41d2-bdd5-527e9bbd9757/resource/6920d9c9-6ea4-415c-95c8-54512a70346c/download/join-the-dots-user-manual.pdf", - }, - ], - "title": "Join the Dots collection-level descriptions", -} -`; diff --git a/dataset/plugins/ckan/package/fixtures/generated/loadPackageFromCkan-should-load-a-package_3615031657/recording.har b/dataset/plugins/ckan/package/fixtures/generated/loadPackageFromCkan-should-load-a-package_3615031657/recording.har deleted file mode 100644 index 3ad969c1..00000000 --- a/dataset/plugins/ckan/package/fixtures/generated/loadPackageFromCkan-should-load-a-package_3615031657/recording.har +++ /dev/null @@ -1,278 +0,0 @@ -{ - "log": { - "_recordingName": "loadPackageFromCkan-should load a package", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "a4062babfd90a257009beac9691adb4d", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 52, - "cookies": [], - "headers": [ - { - "name": "content-type", - "value": "application/json" - } - ], - "headersSize": 98, - "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"id\":\"join-the-dots-collection-level-descriptions\"}" - }, - "queryString": [], - "url": "https://data.nhm.ac.uk/api/3/action/package_show" - }, - "response": { - "bodySize": 9616, - "content": { - "mimeType": "application/json;charset=utf-8", - "size": 9616, - "text": "{\"help\": \"https://data.nhm.ac.uk/api/3/action/help_show?name=package_show\", \"success\": true, \"result\": {\"author\": \"Natural History Museum\", \"author_email\": null, \"creator_user_id\": \"adf402e6-f82f-4545-8bbe-68634a349107\", \"dataset_category\": [\"Collections\"], \"id\": \"78941769-a224-41d2-bdd5-527e9bbd9757\", \"isopen\": true, \"license_id\": \"odc-by\", \"license_title\": \"Open Data Commons Attribution License\", \"license_url\": \"http://www.opendefinition.org/licenses/odc-by\", \"maintainer\": null, \"maintainer_email\": null, \"metadata_created\": \"2022-03-29T17:19:40.385253\", \"metadata_modified\": \"2025-04-29T14:13:19.793337\", \"name\": \"join-the-dots-collection-level-descriptions\", \"notes\": \"The records in this dataset provide a high-level summary of the Natural History Museum's collection and encompass the material managed by its Life Sciences, Earth Sciences and Library & Archives departments. The data has been gathered, structured and published with the intention of providing some level of visibility into the scope, nature and composition of the Museum's holdings, the majority of which have yet to be digitised. The 2,800+ records in this dataset represent the majority of the NHM collection, approximately 78 million items, and should be treated as an index or discovery tool: they are an indicative view of the Museum's holdings, a source of high-level figures and a signpost towards the relevant curatorial team to contact for more in-depth information.\\r\\n \\r\\n## Descriptive fields\\r\\n \\r\\nEach record in the **JtD collection-level records** table describes a group of objects, referred to as a 'collection unit'. Collection units can range in size from a handful of objects to several million and are defined by a set of high-level characteristics common to the objects within them (e.g., taxonomy, straitgraphy, item type, preservation method). Controlled vocabularies and external authority lists underpin descriptive fields wherever possible: see **Data dictionary** for field-level definitions and guidance for all fields used in this dataset. \\r\\n\\r\\n## Collection metrics\\r\\n\\r\\nThe count fields indicate the scale of each collection unit in terms of the estimated quantity of 'items' (specimen-level or equivalent) and 'curatorial units' (objects physically managed during curation activities, such as jars or boxes) it holds. The 'reporting count' field should be used when aggregating multiple units and is the default collection size metric used by the Museum both internally and externally. \\r\\n\\r\\nAssessment scores are allocated by curatorial teams and are an attempt to indicate subjective or mutable qualities of each collection unit, such as outreach potential, scope and significance, physical accessibility and level of information available. See **Data dictionary** and **Join the Dots User Manual** for scoring definitions and examples. Scores are updated on an annual basis and are best viewed as a benchmark of the state of a collection unit/group of units: they are intended to allow us to track the impact of collection care, development and management activities over time and are not intended for use as a comparative measure between different areas of the collection. \\r\\n\\r\\n## Join the Dots Framework\\r\\n\\r\\nThe criteria underpinning collection units and the methodology by which they are quantified and assessed is defined in the Join the Dots (JtD) cross\\u2013disciplinary collection assessment framework, developed and used at the NHM since 2018. For details on the full JtD framework see **Join the Dots User Manual** and the summary paper referenced below. Note that not all fields referenced in the JtD specification are included in this dataset: a subset is presented to avoid compromising the security of the collection and to provide a level of detail relevant to an external audience.\\r\\n\\r\\n## Further references \\r\\n\\r\\n- [NHM main website collection pages](https://www.nhm.ac.uk/our-science/collections.html)\\r\\n- [Library & Archives Discovery Layer](https://www.nhm.ac.uk/our-science/departments-and-staff/library-and-archives.html)\\r\\n- [Archives Catalogues](https://www.nhm.ac.uk/CalmView/default.aspx)\\r\\n- [Specimen-level collection dataset](https://doi.org/10.5519/0002965)\\r\\n- [Join the Dots: assessing a collection of 80 million items at The Natural History Museum, London, 2022](https://www.tandfonline.com/doi/full/10.1080/09647775.2021.2023900) - currently paywalled: contact the dataset curator for a copy\", \"num_resources\": 3, \"num_tags\": 5, \"organization\": {\"id\": \"7854c918-d7eb-4341-96e9-3adfb8d636a0\", \"name\": \"nhm\", \"title\": \"Natural History Museum\", \"type\": \"organization\", \"description\": \"\", \"image_url\": \"\", \"created\": \"2014-12-08T16:35:24.247079\", \"is_organization\": true, \"approval_status\": \"approved\", \"state\": \"active\"}, \"owner_org\": \"7854c918-d7eb-4341-96e9-3adfb8d636a0\", \"private\": false, \"spatial\": \"\", \"state\": \"active\", \"temporal_extent\": \"2018 - 2024\", \"title\": \"Join the Dots collection-level descriptions\", \"type\": \"dataset\", \"update_frequency\": \"annual\", \"url\": null, \"version\": null, \"resources\": [{\"_image_delimiter\": \"\", \"_image_field\": \"\", \"_image_licence\": \"\", \"_latitude_field\": \"\", \"_longitude_field\": \"\", \"_subtitle_field\": \"\", \"_title_field\": \"Collection unit name\", \"cache_last_updated\": null, \"cache_url\": null, \"created\": \"2022-03-29T17:27:59.198498\", \"datastore_active\": true, \"description\": \"Collection-level records describing the holdings of the Natural History Museum as of April 2025. See the dataset description and the other resources in this repository for more detail on navigating and interpreting this data:\\r\\n\\r\\n- [Data dictionary](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/09b2f07b-9e77-44b7-8645-ddcba1266629)\\r\\n- [Join the Dots user manual](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/6920d9c9-6ea4-415c-95c8-54512a70346c)\\r\\n \", \"format\": \"CSV\", \"hash\": \"\", \"id\": \"660de433-447b-40ad-9aaf-77c98f8c2ba9\", \"last_modified\": \"2025-04-29T14:13:19.775868\", \"linked_specimen\": \"\", \"metadata_modified\": \"2025-04-29T14:13:19.801405\", \"mimetype\": \"text/csv\", \"mimetype_inner\": null, \"name\": \"JtD collection-level records\", \"package_id\": \"78941769-a224-41d2-bdd5-527e9bbd9757\", \"position\": 0, \"resource_group\": \"\", \"resource_type\": null, \"size\": 896648, \"state\": \"active\", \"url\": \"https://data.nhm.ac.uk/dataset/78941769-a224-41d2-bdd5-527e9bbd9757/resource/660de433-447b-40ad-9aaf-77c98f8c2ba9/download/jtd_cld_public_dataset_20250425.csv\", \"url_type\": \"upload\"}, {\"_image_delimiter\": \"\", \"_image_field\": \"None\", \"_image_licence\": \"\", \"_latitude_field\": \"None\", \"_longitude_field\": \"None\", \"_title_field\": \"fieldname\", \"cache_last_updated\": null, \"cache_url\": null, \"created\": \"2022-04-01T15:30:41.146689\", \"datastore_active\": true, \"description\": \"Description of the fields found in [JtD collection-level records](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/660de433-447b-40ad-9aaf-77c98f8c2ba9)\", \"format\": \"CSV\", \"hash\": \"\", \"id\": \"09b2f07b-9e77-44b7-8645-ddcba1266629\", \"last_modified\": \"2022-04-01T16:22:52.158232\", \"linked_specimen\": \"\", \"metadata_modified\": \"2022-04-01T17:44:24.791180\", \"mimetype\": \"text/csv\", \"mimetype_inner\": null, \"name\": \"Data dictionary\", \"package_id\": \"78941769-a224-41d2-bdd5-527e9bbd9757\", \"position\": 1, \"resource_group\": \"\", \"resource_type\": null, \"size\": 5868, \"state\": \"active\", \"url\": \"https://data.nhm.ac.uk/dataset/78941769-a224-41d2-bdd5-527e9bbd9757/resource/09b2f07b-9e77-44b7-8645-ddcba1266629/download/jtd-collection-level-records-data-dictionary.csv\", \"url_type\": \"upload\"}, {\"cache_last_updated\": null, \"cache_url\": null, \"created\": \"2022-04-01T12:24:09.228702\", \"datastore_active\": false, \"description\": \"Documentation produced for NHM staff in 2018: a thorough overview of the Join the Dots data, processes and rationale. Some elements of the JtD framework have been amended slightly since this document was assembled, but it remains a useful reference for navigating the collection-level dataset presented in this package. \", \"format\": \"PDF\", \"hash\": \"\", \"id\": \"6920d9c9-6ea4-415c-95c8-54512a70346c\", \"last_modified\": \"2022-04-01T12:24:09.178476\", \"linked_specimen\": \"\", \"metadata_modified\": \"2022-04-01T15:34:49.147048\", \"mimetype\": \"application/pdf\", \"mimetype_inner\": null, \"name\": \"Join the Dots User Manual\", \"package_id\": \"78941769-a224-41d2-bdd5-527e9bbd9757\", \"position\": 2, \"resource_group\": \"\", \"resource_type\": null, \"size\": 8108818, \"state\": \"active\", \"url\": \"https://data.nhm.ac.uk/dataset/78941769-a224-41d2-bdd5-527e9bbd9757/resource/6920d9c9-6ea4-415c-95c8-54512a70346c/download/join-the-dots-user-manual.pdf\", \"url_type\": \"upload\"}], \"tags\": [{\"display_name\": \"collection descriptions\", \"id\": \"1d22818a-9eb5-409e-b4e4-284b631d0df7\", \"name\": \"collection descriptions\", \"state\": \"active\", \"vocabulary_id\": null}, {\"display_name\": \"collections assessment\", \"id\": \"82aa6e78-2fee-4a5e-b919-9afc5829b025\", \"name\": \"collections assessment\", \"state\": \"active\", \"vocabulary_id\": null}, {\"display_name\": \"curation\", \"id\": \"c26cdbe4-0be0-4c2b-80cd-3ce03114988a\", \"name\": \"curation\", \"state\": \"active\", \"vocabulary_id\": null}, {\"display_name\": \"natural history collections\", \"id\": \"8a42e0bd-732c-43a7-8391-929dbe37232f\", \"name\": \"natural history collections\", \"state\": \"active\", \"vocabulary_id\": null}], \"groups\": [], \"relationships_as_subject\": [], \"relationships_as_object\": [], \"doi\": \"10.5519/hay2d5yo\", \"doi_status\": true, \"domain\": \"data.nhm.ac.uk\", \"doi_date_published\": \"2022-04-01\", \"doi_publisher\": \"Natural History Museum\"}}" - }, - "cookies": [], - "headers": [ - { - "name": "cache-control", - "value": "public, max-age=0, must-revalidate" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "frame-ancestors 'self';" - }, - { - "name": "content-type", - "value": "application/json;charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "server", - "value": "nginx/1.14.0 (Ubuntu)" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "x-frame-options", - "value": "SAMEORIGIN" - } - ], - "headersSize": 322, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:25.398Z", - "time": 1371, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 1371 - } - }, - { - "_id": "e2f7f125d55b402974faee5611ed1776", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 64, - "cookies": [], - "headers": [ - { - "name": "content-type", - "value": "application/json" - } - ], - "headersSize": 102, - "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"resource_id\":\"660de433-447b-40ad-9aaf-77c98f8c2ba9\",\"limit\":0}" - }, - "queryString": [], - "url": "https://data.nhm.ac.uk/api/3/action/datastore_search" - }, - "response": { - "bodySize": 2514, - "content": { - "mimeType": "application/json;charset=utf-8", - "size": 2514, - "text": "{\"help\": \"https://data.nhm.ac.uk/api/3/action/help_show?name=datastore_search\", \"success\": true, \"result\": {\"total\": 2916, \"records\": [], \"facets\": {}, \"fields\": [{\"id\": \"_id\", \"type\": \"string\"}, {\"id\": \"Collection unit ID\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"Department\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Division\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Section\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Collection unit name\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Taxon\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Taxon rank\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Taxon ID\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Taxon ID source\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Informal taxon\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Type collection\", \"type\": \"boolean\", \"sortable\": true}, {\"id\": \"Geographic origin\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Earliest geological period\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Latest geological period\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Item type\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Preservation method\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Curatorial unit type\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Bibliographic level\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Fond reference\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Reporting count\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"Reporting metric used\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Item count\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"Item count confidence\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"Curatorial unit count\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"Curatorial unit count confidence\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"C1: Physical accessibility\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"S1: Strategy/mission/research\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"S2: Scope and depth\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"S3: Significance by comparison\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"S4: Usage\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"I1: Digitisation\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"I2: Identification\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"I4: Development potential\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"O1: Education suitability\", \"type\": \"number\", \"sortable\": true}, {\"id\": \"O3: Exhibition suitability\", \"type\": \"number\", \"sortable\": true}], \"after\": null}}" - }, - "cookies": [], - "headers": [ - { - "name": "cache-control", - "value": "public, max-age=0, must-revalidate" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "frame-ancestors 'self';" - }, - { - "name": "content-type", - "value": "application/json;charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "server", - "value": "nginx/1.14.0 (Ubuntu)" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "x-frame-options", - "value": "SAMEORIGIN" - } - ], - "headersSize": 322, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:26.770Z", - "time": 532, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 532 - } - }, - { - "_id": "6d41833a9fe7552f4c5c629553ac2aea", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 64, - "cookies": [], - "headers": [ - { - "name": "content-type", - "value": "application/json" - } - ], - "headersSize": 102, - "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"resource_id\":\"09b2f07b-9e77-44b7-8645-ddcba1266629\",\"limit\":0}" - }, - "queryString": [], - "url": "https://data.nhm.ac.uk/api/3/action/datastore_search" - }, - "response": { - "bodySize": 569, - "content": { - "mimeType": "application/json;charset=utf-8", - "size": 569, - "text": "{\"help\": \"https://data.nhm.ac.uk/api/3/action/help_show?name=datastore_search\", \"success\": true, \"result\": {\"total\": 35, \"records\": [], \"facets\": {}, \"fields\": [{\"id\": \"_id\", \"type\": \"string\"}, {\"id\": \"fieldname\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"description\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"datatype\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"constraints\", \"type\": \"string\", \"sortable\": true}, {\"id\": \"nullable\", \"type\": \"boolean\", \"sortable\": true}, {\"id\": \"Documentation reference\", \"type\": \"string\", \"sortable\": true}], \"after\": null}}" - }, - "cookies": [], - "headers": [ - { - "name": "cache-control", - "value": "public, max-age=0, must-revalidate" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "frame-ancestors 'self';" - }, - { - "name": "content-type", - "value": "application/json;charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "server", - "value": "nginx/1.14.0 (Ubuntu)" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "x-frame-options", - "value": "SAMEORIGIN" - } - ], - "headersSize": 322, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:27.303Z", - "time": 189, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 189 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/dataset/plugins/ckan/package/index.ts b/dataset/plugins/ckan/package/index.ts deleted file mode 100644 index 95cf5829..00000000 --- a/dataset/plugins/ckan/package/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export type { CkanPackage } from "./Package.ts" -export type { CkanOrganization } from "./Organization.ts" -export type { CkanTag } from "./Tag.ts" -export { loadPackageFromCkan } from "./load.ts" -export { savePackageToCkan } from "./save.ts" -export { convertPackageFromCkan } from "./convert/fromCkan.ts" -export { convertPackageToCkan } from "./convert/toCkan.ts" diff --git a/dataset/plugins/ckan/package/load.spec.ts b/dataset/plugins/ckan/package/load.spec.ts deleted file mode 100644 index 03ee0e39..00000000 --- a/dataset/plugins/ckan/package/load.spec.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromCkan } from "./load.ts" - -useRecording() - -describe("loadPackageFromCkan", () => { - it("should load a package", async () => { - const dataPackage = await loadPackageFromCkan( - "https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions", - ) - - expect(dataPackage).toMatchSnapshot() - }) -}) diff --git a/dataset/plugins/ckan/package/load.ts b/dataset/plugins/ckan/package/load.ts deleted file mode 100644 index 0f012b57..00000000 --- a/dataset/plugins/ckan/package/load.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { mergePackages } from "../../../package/index.ts" -import { makeCkanApiRequest } from "../ckan/index.ts" -import type { CkanPackage } from "./Package.ts" -import { convertPackageFromCkan } from "./convert/fromCkan.ts" - -/** - * Load a package from a CKAN instance - * @param props Object containing the URL to the CKAN package - * @returns Package object and cleanup function - */ -export async function loadPackageFromCkan(datasetUrl: string) { - const packageId = extractPackageId(datasetUrl) - if (!packageId) { - throw new Error(`Failed to extract package ID from URL: ${datasetUrl}`) - } - - const ckanPackage = await makeCkanApiRequest({ - ckanUrl: datasetUrl, - action: "package_show", - payload: { id: packageId }, - }) - - for (const resource of ckanPackage.resources) { - const resourceId = resource.id - if (["CSV", "XLS", "XLSX"].includes(resource.format)) { - const schema = await loadCkanSchema({ datasetUrl, resourceId }) - if (schema) { - resource.schema = schema - } - } - } - - const systemPackage = convertPackageFromCkan(ckanPackage) - const userPackagePath = systemPackage.resources - .filter(resource => resource["ckan:key"] === "datapackage.json") - .map(resource => resource["ckan:url"]) - .at(0) - - const datapackage = await mergePackages({ systemPackage, userPackagePath }) - datapackage.resources = datapackage.resources.map(resource => { - // TODO: remove these keys completely - return { ...resource, "ckan:key": undefined, "ckan:url": undefined } - }) - - return datapackage -} - -/** - * Extract package ID from URL - * - * Examples: - * - https://hri.fi/data/en_GB/dataset/helsingin-kaupungin-verkkosivustojen-kavijaanalytiikka - * - https://www.opendata.dk/city-of-copenhagen/parkeringszoner-information - * - https://open.africa/dataset/pib-annual-senegal - * - https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions - */ -function extractPackageId(datasetUrl: string) { - const url = new URL(datasetUrl) - const pathParts = url.pathname.split("/").filter(Boolean) - return pathParts.at(-1) -} - -/** - * Fetch resource schema data from CKAN datastore - */ -async function loadCkanSchema(options: { - datasetUrl: string - resourceId: string -}) { - try { - // For some reason, datastore_info doesn't work - // So we use data fetching endpoint that also returns the schema - const result = await makeCkanApiRequest({ - ckanUrl: options.datasetUrl, - action: "datastore_search", - payload: { resource_id: options.resourceId, limit: 0 }, - }) - - // @ts-ignore - const fields = result.fields.filter( - (field: any) => field.id !== "_id" && field.id !== "_full_text", - ) - - return { fields } - } catch (error) { - return undefined - } -} diff --git a/dataset/plugins/ckan/package/save.spec.ts b/dataset/plugins/ckan/package/save.spec.ts deleted file mode 100644 index b32abc1a..00000000 --- a/dataset/plugins/ckan/package/save.spec.ts +++ /dev/null @@ -1,404 +0,0 @@ -import { relative } from "node:path" -import type { Package } from "@dpkit/metadata" -import { loadPackageDescriptor } from "@dpkit/metadata" -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { savePackageToCkan } from "./save.ts" - -describe("savePackageToCkan", () => { - const getFixturePath = (name: string) => - relative(process.cwd(), `${import.meta.dirname}/fixtures/${name}`) - - const mockPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - path: getFixturePath("data.csv"), - format: "csv", - bytes: 100, - }, - ], - } - - const mockOptions = { - apiKey: "test-api-key", - ckanUrl: "https://ckan.example.com", - ownerOrg: "test-org", - datasetName: "test-dataset", - } - - const originalFetch = globalThis.fetch - let fetchMock: ReturnType - - beforeEach(() => { - fetchMock = vi.fn() - // @ts-ignore - globalThis.fetch = fetchMock - }) - - afterEach(() => { - globalThis.fetch = originalFetch - vi.resetAllMocks() - }) - - it.skip("should save a package", async () => { - const dataPackage = await loadPackageDescriptor( - "core/package/fixtures/package.json", - ) - - const result = await savePackageToCkan(dataPackage, { - ckanUrl: "http://localhost:5000/", - apiKey: - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiJ1T0Y0VUNybTU5Y0dzdlk3ejhreF9CeC02R0w4RDBOdW9QS0J0WkJFXzlJIiwiaWF0IjoxNzQ3OTI0NDg5fQ.ioGiLlZkm24xHQRBas5X5ig5eU7u_fIjkl4oifGnLaA", - datasetName: "test", - ownerOrg: "test", - }) - - expect(result).toBeDefined() - }) - - it("creates a package in CKAN with correct API calls", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - name: "test-dataset", - url: "https://ckan.example.com/dataset/test-dataset", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-1", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-1", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-2", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-2", - }, - }), - }) - - const result = await savePackageToCkan(mockPackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(3) - - const packageCreateCall = fetchMock.mock.calls[0] - expect(packageCreateCall).toBeDefined() - if (!packageCreateCall) return - - expect(packageCreateCall[0]).toEqual( - "https://ckan.example.com/api/3/action/package_create", - ) - expect(packageCreateCall[1]).toMatchObject({ - method: "POST", - headers: { - Authorization: "test-api-key", - "Content-Type": "application/json", - }, - }) - - const packagePayload = JSON.parse(packageCreateCall[1].body) - expect(packagePayload.name).toEqual("test-dataset") - expect(packagePayload.owner_org).toEqual("test-org") - expect(packagePayload.title).toEqual("Test Package") - expect(packagePayload.notes).toEqual("A test package") - expect(packagePayload.resources).toEqual([]) - - expect(result).toEqual({ - path: "https://ckan.example.com/dataset/test-dataset", - datasetUrl: "https://ckan.example.com/dataset/test-dataset", - }) - }) - - it("creates resources with file uploads", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - name: "test-dataset", - url: "https://ckan.example.com/dataset/test-dataset", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-1", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-1", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-2", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-2", - }, - }), - }) - - await savePackageToCkan(mockPackage, mockOptions) - - const resourceCreateCall = fetchMock.mock.calls[1] - expect(resourceCreateCall).toBeDefined() - if (!resourceCreateCall) return - - expect(resourceCreateCall[0]).toEqual( - "https://ckan.example.com/api/3/action/resource_create", - ) - expect(resourceCreateCall[1]).toMatchObject({ - method: "POST", - headers: { - Authorization: "test-api-key", - }, - }) - - const formData = resourceCreateCall[1].body - expect(formData).toBeInstanceOf(FormData) - expect(formData.get("package_id")).toEqual("test-dataset") - expect(formData.get("name")).toEqual("data.csv") - }) - - it("creates datapackage.json resource", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - name: "test-dataset", - url: "https://ckan.example.com/dataset/test-dataset", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-1", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-1", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-2", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-2", - }, - }), - }) - - await savePackageToCkan(mockPackage, mockOptions) - - const datapackageCreateCall = fetchMock.mock.calls[2] - expect(datapackageCreateCall).toBeDefined() - if (!datapackageCreateCall) return - - expect(datapackageCreateCall[0]).toEqual( - "https://ckan.example.com/api/3/action/resource_create", - ) - - const formData = datapackageCreateCall[1].body - expect(formData).toBeInstanceOf(FormData) - expect(formData.get("package_id")).toEqual("test-dataset") - expect(formData.get("name")).toEqual("datapackage.json") - - const uploadBlob = formData.get("upload") - expect(uploadBlob).toBeInstanceOf(Blob) - }) - - it("handles API errors from package_create", async () => { - fetchMock.mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: "Bad Request", - text: () => Promise.resolve("Invalid package data"), - }) - - await expect(savePackageToCkan(mockPackage, mockOptions)).rejects.toThrow( - "CKAN API error: 400 Bad Request", - ) - }) - - it("handles API errors from resource_create", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - name: "test-dataset", - url: "https://ckan.example.com/dataset/test-dataset", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: false, - status: 500, - statusText: "Internal Server Error", - text: () => Promise.resolve("Failed to create resource"), - }) - - await expect(savePackageToCkan(mockPackage, mockOptions)).rejects.toThrow( - "CKAN API error: 500 Internal Server Error", - ) - }) - - it("handles CKAN API success: false responses", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: false, - error: { - message: "Package already exists", - }, - }), - }) - - await expect(savePackageToCkan(mockPackage, mockOptions)).rejects.toThrow( - "CKAN API error", - ) - }) - - it("handles packages with multiple resources", async () => { - const multiResourcePackage: Package = { - ...mockPackage, - resources: [ - { - name: "resource-1", - path: getFixturePath("data.csv"), - format: "csv", - }, - { - name: "resource-2", - path: getFixturePath("data.csv"), - format: "json", - }, - ], - } - - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - name: "test-dataset", - url: "https://ckan.example.com/dataset/test-dataset", - }, - }), - }) - - await savePackageToCkan(multiResourcePackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(4) - - const secondResourceCall = fetchMock.mock.calls[2] - expect(secondResourceCall).toBeDefined() - if (!secondResourceCall) return - - expect(secondResourceCall[0]).toEqual( - "https://ckan.example.com/api/3/action/resource_create", - ) - }) - - it("handles packages with no resources", async () => { - const emptyPackage: Package = { - ...mockPackage, - resources: [], - } - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - name: "test-dataset", - url: "https://ckan.example.com/dataset/test-dataset", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { - id: "resource-1", - url: "https://ckan.example.com/dataset/test-dataset/resource/resource-1", - }, - }), - }) - - const result = await savePackageToCkan(emptyPackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(result.datasetUrl).toEqual( - "https://ckan.example.com/dataset/test-dataset", - ) - }) - - it("passes API key in Authorization header", async () => { - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - success: true, - result: { name: "test-dataset" }, - }), - }) - - await savePackageToCkan(mockPackage, { - ...mockOptions, - apiKey: "custom-api-key", - }) - - const firstCall = fetchMock.mock.calls[0] - expect(firstCall).toBeDefined() - if (!firstCall) return - - const headers = firstCall[1].headers - expect(headers.Authorization).toEqual("custom-api-key") - }) -}) diff --git a/dataset/plugins/ckan/package/save.ts b/dataset/plugins/ckan/package/save.ts deleted file mode 100644 index c07639e7..00000000 --- a/dataset/plugins/ckan/package/save.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { blob } from "node:stream/consumers" -import type { Descriptor, Package } from "@dpkit/metadata" -import { - convertPackageToDescriptor, - getFilename, - getFormat, - stringifyDescriptor, -} from "@dpkit/metadata" -import { getPackageBasepath } from "../../../package/index.ts" -import { saveResourceFiles } from "../../../resource/index.ts" -import { loadFileStream } from "../../../stream/index.ts" -import { makeCkanApiRequest } from "../ckan/index.ts" -import type { CkanResource } from "../resource/index.ts" -import { convertResourceToCkan } from "../resource/index.ts" -import { convertPackageToCkan } from "./convert/toCkan.ts" - -export async function savePackageToCkan( - dataPackage: Package, - options: { - apiKey: string - ckanUrl: string - ownerOrg: string - datasetName: string - }, -) { - const { apiKey, ckanUrl, ownerOrg, datasetName } = options - - const basepath = getPackageBasepath(dataPackage) - const ckanPackage = convertPackageToCkan(dataPackage) - - const payload = { - ...ckanPackage, - name: datasetName, - owner_org: ownerOrg, - resources: [], - } - - const result = await makeCkanApiRequest({ - action: "package_create", - payload, - ckanUrl: ckanUrl, - apiKey: apiKey, - }) - - const url = new URL(ckanUrl) - url.pathname = `/dataset/${result.name}` - - const resourceDescriptors: Descriptor[] = [] - for (const resource of dataPackage.resources) { - resourceDescriptors.push( - await saveResourceFiles(resource, { - basepath, - withRemote: true, - withoutFolders: true, - saveFile: async options => { - const filename = getFilename(options.normalizedPath) - const ckanResource = convertResourceToCkan(resource) - - const payload = { - ...ckanResource, - package_id: datasetName, - name: options.denormalizedPath, - format: getFormat(filename)?.toUpperCase(), - } - - const upload = { - name: options.denormalizedPath, - data: await blob(await loadFileStream(options.normalizedPath)), - } - - const result = await makeCkanApiRequest({ - action: "resource_create", - payload, - upload, - ckanUrl, - apiKey, - }) - - return result.url - }, - }), - ) - } - - const descriptor = { - ...convertPackageToDescriptor(dataPackage, { basepath }), - resources: resourceDescriptors, - } - - for (const denormalizedPath of ["datapackage.json"]) { - const payload = { - package_id: datasetName, - name: denormalizedPath, - } - - const upload = { - name: denormalizedPath, - data: new Blob([stringifyDescriptor(descriptor)]), - } - - await makeCkanApiRequest({ - action: "resource_create", - payload, - upload, - ckanUrl, - apiKey, - }) - } - - return { - path: result.url, - datasetUrl: url.toString(), - } -} diff --git a/dataset/plugins/ckan/plugin.spec.ts b/dataset/plugins/ckan/plugin.spec.ts deleted file mode 100644 index a5efd47c..00000000 --- a/dataset/plugins/ckan/plugin.spec.ts +++ /dev/null @@ -1,126 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/load.ts" -import { CkanPlugin } from "./plugin.ts" - -vi.mock("./package/load.ts", () => ({ - loadPackageFromCkan: vi.fn(), -})) - -describe("CkanPlugin", () => { - let plugin: CkanPlugin - let mockLoadPackageFromCkan: ReturnType - - beforeEach(() => { - plugin = new CkanPlugin() - mockLoadPackageFromCkan = vi.mocked(packageModule.loadPackageFromCkan) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from ckan url with /dataset/ path", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromCkan.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://data.example.com/dataset/test-dataset", - ) - - expect(mockLoadPackageFromCkan).toHaveBeenCalledWith( - "https://data.example.com/dataset/test-dataset", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for urls without /dataset/", async () => { - const result = await plugin.loadPackage("https://example.com/data") - - expect(mockLoadPackageFromCkan).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local paths", async () => { - const result = await plugin.loadPackage("./data") - - expect(mockLoadPackageFromCkan).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for github urls", async () => { - const result = await plugin.loadPackage("https://github.com/owner/repo") - - expect(mockLoadPackageFromCkan).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle ckan urls with additional path segments", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromCkan.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://data.example.com/dataset/test-dataset/resource/123", - ) - - expect(mockLoadPackageFromCkan).toHaveBeenCalledWith( - "https://data.example.com/dataset/test-dataset/resource/123", - ) - expect(result).toEqual(mockPackage) - }) - - it("should handle ckan urls with query parameters", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromCkan.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://data.example.com/dataset/test-dataset?id=456", - ) - - expect(mockLoadPackageFromCkan).toHaveBeenCalledWith( - "https://data.example.com/dataset/test-dataset?id=456", - ) - expect(result).toEqual(mockPackage) - }) - - it("should handle http ckan urls", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromCkan.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "http://data.example.com/dataset/test-dataset", - ) - - expect(mockLoadPackageFromCkan).toHaveBeenCalledWith( - "http://data.example.com/dataset/test-dataset", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for zenodo urls", async () => { - const result = await plugin.loadPackage("https://zenodo.org/record/123") - - expect(mockLoadPackageFromCkan).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for urls with dataset in query params only", async () => { - const result = await plugin.loadPackage( - "https://example.com/api?name=dataset", - ) - - expect(mockLoadPackageFromCkan).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/ckan/plugin.ts b/dataset/plugins/ckan/plugin.ts deleted file mode 100644 index 4bd30f22..00000000 --- a/dataset/plugins/ckan/plugin.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { isRemotePath } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" -import { loadPackageFromCkan } from "./package/load.ts" - -export class CkanPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isCkan = getIsCkan(source) - if (!isCkan) return undefined - - const dataPackage = await loadPackageFromCkan(source) - return dataPackage - } -} - -function getIsCkan(path: string) { - const isRemote = isRemotePath(path) - if (!isRemote) return false - - return path.includes("/dataset/") -} diff --git a/dataset/plugins/ckan/resource/Resource.ts b/dataset/plugins/ckan/resource/Resource.ts deleted file mode 100644 index ea788a40..00000000 --- a/dataset/plugins/ckan/resource/Resource.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { SetRequired } from "type-fest" -import type { CkanSchema } from "../schema/index.ts" - -/** - * CKAN Resource interface - */ -export interface CkanResource { - /** - * Resource identifier - */ - id: string - - /** - * Resource URL - */ - url: string - - /** - * Resource name - */ - name: string - - /** - * Resource creation timestamp - */ - created: string - - /** - * Resource description - */ - description: string - - /** - * Resource format - */ - format: string - - /** - * Resource hash - */ - hash: string - - /** - * Resource last modification timestamp - */ - last_modified: string - - /** - * Resource metadata modification timestamp - */ - metadata_modified: string - - /** - * Resource MIME type - */ - mimetype: string - - /** - * Resource size in bytes - */ - size: number - - /** - * Resource schema - */ - schema?: CkanSchema -} - -export type NewCkanResource = SetRequired, "url" | "name"> diff --git a/dataset/plugins/ckan/resource/convert/fromCkan.ts b/dataset/plugins/ckan/resource/convert/fromCkan.ts deleted file mode 100644 index 4fa9aeeb..00000000 --- a/dataset/plugins/ckan/resource/convert/fromCkan.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { getFilename } from "@dpkit/metadata" -import { convertSchemaFromCkan } from "../../schema/index.ts" -import type { CkanResource } from "../Resource.ts" - -export function convertResourceFromCkan(ckanResource: CkanResource): Resource { - const resource: Resource = { - name: convertName(ckanResource.name), - path: ckanResource.url, - "ckan:key": getFilename(ckanResource.url), - "ckan:url": ckanResource.url, - } - - if (ckanResource.description) { - resource.description = ckanResource.description - } - - if (ckanResource.format) { - resource.format = ckanResource.format.toLowerCase() - } - - if (ckanResource.mimetype) { - resource.mediatype = ckanResource.mimetype - } - - if (ckanResource.size) { - resource.bytes = ckanResource.size - } - - if (ckanResource.hash) { - resource.hash = ckanResource.hash - } - - if (ckanResource.schema) { - resource.type = "table" - resource.schema = convertSchemaFromCkan(ckanResource.schema) - } - - return resource -} - -function convertName(name: string): string { - return name - .replace(/[\s\.\(\)\/\\,]+/g, "_") - .toLowerCase() - .replace(/[^a-z0-9_\-]/g, "") - .replace(/^(\d)/, "_$1") - .slice(0, 100) -} diff --git a/dataset/plugins/ckan/resource/convert/toCkan.ts b/dataset/plugins/ckan/resource/convert/toCkan.ts deleted file mode 100644 index 0175d6fe..00000000 --- a/dataset/plugins/ckan/resource/convert/toCkan.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import type { CkanResource } from "../Resource.ts" - -export function convertResourceToCkan(resource: Resource) { - const ckanResource: Partial = {} - - if (resource.description) { - ckanResource.description = resource.description - } - - if (resource.format) { - ckanResource.format = resource.format.toUpperCase() - } - - if (resource.mediatype) { - ckanResource.mimetype = resource.mediatype - } - - if (resource.bytes) { - ckanResource.size = resource.bytes - } - - if (resource.hash) { - ckanResource.hash = resource.hash - } - - return ckanResource -} diff --git a/dataset/plugins/ckan/resource/index.ts b/dataset/plugins/ckan/resource/index.ts deleted file mode 100644 index 5537809d..00000000 --- a/dataset/plugins/ckan/resource/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type { CkanResource } from "./Resource.ts" -export { convertResourceFromCkan } from "./convert/fromCkan.ts" -export { convertResourceToCkan } from "./convert/toCkan.ts" diff --git a/dataset/plugins/ckan/schema/Field.ts b/dataset/plugins/ckan/schema/Field.ts deleted file mode 100644 index 8a47a7d0..00000000 --- a/dataset/plugins/ckan/schema/Field.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * CKAN Field interface - */ -export interface CkanField { - /** - * Field identifier - */ - id: string - - /** - * Field data type - */ - type: string - - /** - * Additional field information - */ - info?: CkanFieldInfo -} - -/** - * CKAN Field Info interface - */ -export interface CkanFieldInfo { - /** - * Human-readable field label - */ - label: string - - /** - * Additional notes about the field - */ - notes: string - - /** - * Field type override - */ - type_override: string -} diff --git a/dataset/plugins/ckan/schema/Schema.ts b/dataset/plugins/ckan/schema/Schema.ts deleted file mode 100644 index 2397efb2..00000000 --- a/dataset/plugins/ckan/schema/Schema.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { CkanField } from "./Field.ts" - -/** - * CKAN Schema interface - */ -export interface CkanSchema { - /** - * List of fields - */ - fields: CkanField[] -} diff --git a/dataset/plugins/ckan/schema/convert/fixtures/ckan-schema.json b/dataset/plugins/ckan/schema/convert/fixtures/ckan-schema.json deleted file mode 100644 index 94977029..00000000 --- a/dataset/plugins/ckan/schema/convert/fixtures/ckan-schema.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "fields": [ - { - "id": "id", - "type": "int", - "info": { - "label": "ID", - "notes": "Unique identifier", - "type_override": "int" - } - }, - { - "id": "name", - "type": "text", - "info": { - "label": "Name", - "notes": "Person's full name", - "type_override": "text" - } - }, - { - "id": "age", - "type": "int" - }, - { - "id": "score", - "type": "numeric", - "info": { - "label": "Score", - "notes": "Test score", - "type_override": "numeric" - } - }, - { - "id": "is_active", - "type": "bool" - }, - { - "id": "birth_date", - "type": "date", - "info": { - "label": "Birth Date", - "notes": "Date of birth", - "type_override": "date" - } - }, - { - "id": "start_time", - "type": "time" - }, - { - "id": "created_at", - "type": "timestamp", - "info": { - "label": "Created At", - "notes": "Timestamp when record was created", - "type_override": "timestamp" - } - }, - { - "id": "metadata", - "type": "json" - }, - { - "id": "tags", - "type": "array", - "info": { - "label": "Tags", - "notes": "List of tags", - "type_override": "array" - } - }, - { - "id": "string_field", - "type": "string" - }, - { - "id": "integer_field", - "type": "integer" - }, - { - "id": "number_field", - "type": "number" - }, - { - "id": "float_field", - "type": "float" - }, - { - "id": "boolean_field", - "type": "boolean" - }, - { - "id": "datetime_field", - "type": "datetime" - }, - { - "id": "object_field", - "type": "object" - }, - { - "id": "unknown_field", - "type": "unknown_type" - }, - { - "id": "override_field", - "type": "text", - "info": { - "label": "Override Field", - "notes": "Field with type override", - "type_override": "int" - } - } - ] -} diff --git a/dataset/plugins/ckan/schema/convert/fromCkan.spec.ts b/dataset/plugins/ckan/schema/convert/fromCkan.spec.ts deleted file mode 100644 index 05f93776..00000000 --- a/dataset/plugins/ckan/schema/convert/fromCkan.spec.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { describe, expect, it } from "vitest" -import type { CkanSchema } from "../Schema.ts" -import ckanSchemaFixture from "./fixtures/ckan-schema.json" with { - type: "json", -} -import { convertSchemaFromCkan } from "./fromCkan.ts" - -describe("convertSchemaFromCkan", () => { - it("converts a CKAN schema to a Frictionless schema", () => { - const ckanSchema = ckanSchemaFixture as CkanSchema - - const result = convertSchemaFromCkan(ckanSchema) - - expect(result.fields).toHaveLength(ckanSchema.fields.length) - - const idField = result.fields.find(f => f.name === "id") - expect(idField).toBeDefined() - if (idField) { - expect(idField.type).toEqual("integer") - expect(idField.title).toEqual("ID") - expect(idField.description).toEqual("Unique identifier") - } - - const nameField = result.fields.find(f => f.name === "name") - expect(nameField).toBeDefined() - if (nameField) { - expect(nameField.type).toEqual("string") - expect(nameField.title).toEqual("Name") - expect(nameField.description).toEqual("Person's full name") - } - - const ageField = result.fields.find(f => f.name === "age") - expect(ageField).toBeDefined() - if (ageField) { - expect(ageField.type).toEqual("integer") - expect(ageField.title).toBeUndefined() - expect(ageField.description).toBeUndefined() - } - - const scoreField = result.fields.find(f => f.name === "score") - expect(scoreField).toBeDefined() - if (scoreField) { - expect(scoreField.type).toEqual("number") - expect(scoreField.title).toEqual("Score") - expect(scoreField.description).toEqual("Test score") - } - - const isActiveField = result.fields.find(f => f.name === "is_active") - expect(isActiveField).toBeDefined() - if (isActiveField) { - expect(isActiveField.type).toEqual("boolean") - } - - const birthDateField = result.fields.find(f => f.name === "birth_date") - expect(birthDateField).toBeDefined() - if (birthDateField) { - expect(birthDateField.type).toEqual("date") - expect(birthDateField.title).toEqual("Birth Date") - expect(birthDateField.description).toEqual("Date of birth") - } - - const startTimeField = result.fields.find(f => f.name === "start_time") - expect(startTimeField).toBeDefined() - if (startTimeField) { - expect(startTimeField.type).toEqual("time") - } - - const createdAtField = result.fields.find(f => f.name === "created_at") - expect(createdAtField).toBeDefined() - if (createdAtField) { - expect(createdAtField.type).toEqual("datetime") - expect(createdAtField.title).toEqual("Created At") - expect(createdAtField.description).toEqual( - "Timestamp when record was created", - ) - } - - const metadataField = result.fields.find(f => f.name === "metadata") - expect(metadataField).toBeDefined() - if (metadataField) { - expect(metadataField.type).toEqual("object") - } - - const tagsField = result.fields.find(f => f.name === "tags") - expect(tagsField).toBeDefined() - if (tagsField) { - expect(tagsField.type).toEqual("array") - expect(tagsField.title).toEqual("Tags") - expect(tagsField.description).toEqual("List of tags") - } - }) - - it("converts CKAN type aliases to Frictionless types", () => { - const ckanSchema = ckanSchemaFixture as CkanSchema - - const result = convertSchemaFromCkan(ckanSchema) - - const stringField = result.fields.find(f => f.name === "string_field") - expect(stringField?.type).toEqual("string") - - const integerField = result.fields.find(f => f.name === "integer_field") - expect(integerField?.type).toEqual("integer") - - const numberField = result.fields.find(f => f.name === "number_field") - expect(numberField?.type).toEqual("number") - - const floatField = result.fields.find(f => f.name === "float_field") - expect(floatField?.type).toEqual("number") - - const booleanField = result.fields.find(f => f.name === "boolean_field") - expect(booleanField?.type).toEqual("boolean") - - const datetimeField = result.fields.find(f => f.name === "datetime_field") - expect(datetimeField?.type).toEqual("datetime") - - const objectField = result.fields.find(f => f.name === "object_field") - expect(objectField?.type).toEqual("object") - }) - - it("handles unknown field types by converting to 'any'", () => { - const ckanSchema = ckanSchemaFixture as CkanSchema - - const result = convertSchemaFromCkan(ckanSchema) - - const unknownField = result.fields.find(f => f.name === "unknown_field") - expect(unknownField).toBeDefined() - if (unknownField) { - expect(unknownField.type).toEqual("any") - } - }) - - it("respects type_override in field info", () => { - const ckanSchema = ckanSchemaFixture as CkanSchema - - const result = convertSchemaFromCkan(ckanSchema) - - const overrideField = result.fields.find(f => f.name === "override_field") - expect(overrideField).toBeDefined() - if (overrideField) { - expect(overrideField.type).toEqual("integer") - expect(overrideField.title).toEqual("Override Field") - expect(overrideField.description).toEqual("Field with type override") - } - }) - - it("handles empty fields array", () => { - const ckanSchema: CkanSchema = { - fields: [], - } - - const result = convertSchemaFromCkan(ckanSchema) - - expect(result.fields).toEqual([]) - }) - - it("handles fields without info object", () => { - const ckanSchema: CkanSchema = { - fields: [ - { - id: "simple_field", - type: "text", - }, - ], - } - - const result = convertSchemaFromCkan(ckanSchema) - - expect(result.fields).toHaveLength(1) - const field = result.fields[0] - expect(field).toBeDefined() - if (field) { - expect(field.name).toEqual("simple_field") - expect(field.type).toEqual("string") - expect(field.title).toBeUndefined() - expect(field.description).toBeUndefined() - } - }) - - it("handles case insensitivity in type conversion", () => { - const ckanSchema: CkanSchema = { - fields: [ - { id: "field1", type: "TEXT" }, - { id: "field2", type: "INT" }, - { id: "field3", type: "BOOL" }, - { id: "field4", type: "TIMESTAMP" }, - ], - } - - const result = convertSchemaFromCkan(ckanSchema) - - expect(result.fields[0]?.type).toEqual("string") - expect(result.fields[1]?.type).toEqual("integer") - expect(result.fields[2]?.type).toEqual("boolean") - expect(result.fields[3]?.type).toEqual("datetime") - }) -}) diff --git a/dataset/plugins/ckan/schema/convert/fromCkan.ts b/dataset/plugins/ckan/schema/convert/fromCkan.ts deleted file mode 100644 index 1332b0d5..00000000 --- a/dataset/plugins/ckan/schema/convert/fromCkan.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { Field, Schema } from "@dpkit/metadata" -import type { - ArrayField, - BooleanField, - DateField, - DatetimeField, - IntegerField, - NumberField, - ObjectField, - StringField, - TimeField, -} from "@dpkit/metadata" -import type { CkanField } from "../Field.ts" -import type { CkanSchema } from "../Schema.ts" - -export function convertSchemaFromCkan(ckanSchema: CkanSchema): Schema { - const fields = ckanSchema.fields.map(convertField) - - return { fields } -} - -function convertField(ckanField: CkanField) { - const { id, type, info } = ckanField - - const field: Partial = { - name: id, - } - - if (info) { - if (info.label) field.title = info.label - if (info.notes) field.description = info.notes - } - - const fieldType = (info?.type_override || type).toLowerCase() - switch (fieldType) { - case "text": - case "string": - return { ...field, type: "string" } as StringField - case "int": - case "integer": - return { ...field, type: "integer" } as IntegerField - case "numeric": - case "number": - case "float": - return { ...field, type: "number" } as NumberField - case "bool": - case "boolean": - return { ...field, type: "boolean" } as BooleanField - case "date": - return { ...field, type: "date" } as DateField - case "time": - return { ...field, type: "time" } as TimeField - case "timestamp": - case "datetime": - return { ...field, type: "datetime" } as DatetimeField - case "json": - case "object": - return { ...field, type: "object" } as ObjectField - case "array": - return { ...field, type: "array" } as ArrayField - default: - return { ...field, type: "any" } as Field - } -} diff --git a/dataset/plugins/ckan/schema/convert/toCkan.spec.ts b/dataset/plugins/ckan/schema/convert/toCkan.spec.ts deleted file mode 100644 index 2a2c1d0f..00000000 --- a/dataset/plugins/ckan/schema/convert/toCkan.spec.ts +++ /dev/null @@ -1,322 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import { describe, expect, it } from "vitest" -import type { CkanSchema } from "../Schema.ts" -import ckanSchemaFixture from "./fixtures/ckan-schema.json" with { - type: "json", -} -import { convertSchemaFromCkan } from "./fromCkan.ts" -import { convertSchemaToCkan } from "./toCkan.ts" - -describe("convertSchemaToCkan", () => { - it("converts a Frictionless schema to a CKAN schema", () => { - const schema: Schema = { - fields: [ - { - name: "id", - type: "integer", - title: "ID", - description: "Unique identifier", - }, - { - name: "name", - type: "string", - title: "Name", - description: "Person's full name", - }, - { - name: "age", - type: "integer", - }, - { - name: "score", - type: "number", - title: "Score", - description: "Test score", - }, - { - name: "is_active", - type: "boolean", - }, - { - name: "birth_date", - type: "date", - title: "Birth Date", - description: "Date of birth", - }, - { - name: "start_time", - type: "time", - }, - { - name: "created_at", - type: "datetime", - title: "Created At", - description: "Timestamp when record was created", - }, - { - name: "metadata", - type: "object", - }, - { - name: "tags", - type: "array", - title: "Tags", - description: "List of tags", - }, - { - name: "location", - type: "geopoint", - }, - { - name: "boundary", - type: "geojson", - }, - ], - } - - const result = convertSchemaToCkan(schema) - - expect(result.fields).toHaveLength(schema.fields.length) - - const idField = result.fields.find(f => f.id === "id") - expect(idField).toBeDefined() - if (idField) { - expect(idField.type).toEqual("int") - expect(idField.info).toBeDefined() - expect(idField.info?.label).toEqual("ID") - expect(idField.info?.notes).toEqual("Unique identifier") - expect(idField.info?.type_override).toEqual("int") - } - - const nameField = result.fields.find(f => f.id === "name") - expect(nameField).toBeDefined() - if (nameField) { - expect(nameField.type).toEqual("text") - expect(nameField.info).toBeDefined() - expect(nameField.info?.label).toEqual("Name") - expect(nameField.info?.notes).toEqual("Person's full name") - expect(nameField.info?.type_override).toEqual("text") - } - - const ageField = result.fields.find(f => f.id === "age") - expect(ageField).toBeDefined() - if (ageField) { - expect(ageField.type).toEqual("int") - expect(ageField.info).toBeUndefined() - } - - const scoreField = result.fields.find(f => f.id === "score") - expect(scoreField).toBeDefined() - if (scoreField) { - expect(scoreField.type).toEqual("numeric") - expect(scoreField.info).toBeDefined() - expect(scoreField.info?.label).toEqual("Score") - expect(scoreField.info?.notes).toEqual("Test score") - expect(scoreField.info?.type_override).toEqual("numeric") - } - - const isActiveField = result.fields.find(f => f.id === "is_active") - expect(isActiveField).toBeDefined() - if (isActiveField) { - expect(isActiveField.type).toEqual("bool") - expect(isActiveField.info).toBeUndefined() - } - - const birthDateField = result.fields.find(f => f.id === "birth_date") - expect(birthDateField).toBeDefined() - if (birthDateField) { - expect(birthDateField.type).toEqual("date") - expect(birthDateField.info).toBeDefined() - expect(birthDateField.info?.label).toEqual("Birth Date") - expect(birthDateField.info?.notes).toEqual("Date of birth") - expect(birthDateField.info?.type_override).toEqual("date") - } - - const startTimeField = result.fields.find(f => f.id === "start_time") - expect(startTimeField).toBeDefined() - if (startTimeField) { - expect(startTimeField.type).toEqual("time") - expect(startTimeField.info).toBeUndefined() - } - - const createdAtField = result.fields.find(f => f.id === "created_at") - expect(createdAtField).toBeDefined() - if (createdAtField) { - expect(createdAtField.type).toEqual("timestamp") - expect(createdAtField.info).toBeDefined() - expect(createdAtField.info?.label).toEqual("Created At") - expect(createdAtField.info?.notes).toEqual( - "Timestamp when record was created", - ) - expect(createdAtField.info?.type_override).toEqual("timestamp") - } - - const metadataField = result.fields.find(f => f.id === "metadata") - expect(metadataField).toBeDefined() - if (metadataField) { - expect(metadataField.type).toEqual("json") - expect(metadataField.info).toBeUndefined() - } - - const tagsField = result.fields.find(f => f.id === "tags") - expect(tagsField).toBeDefined() - if (tagsField) { - expect(tagsField.type).toEqual("array") - expect(tagsField.info).toBeDefined() - expect(tagsField.info?.label).toEqual("Tags") - expect(tagsField.info?.notes).toEqual("List of tags") - expect(tagsField.info?.type_override).toEqual("array") - } - - const locationField = result.fields.find(f => f.id === "location") - expect(locationField).toBeDefined() - if (locationField) { - expect(locationField.type).toEqual("geopoint") - } - - const boundaryField = result.fields.find(f => f.id === "boundary") - expect(boundaryField).toBeDefined() - if (boundaryField) { - expect(boundaryField.type).toEqual("geojson") - } - }) - - it("handles fields with only title", () => { - const schema: Schema = { - fields: [ - { - name: "field1", - type: "string", - title: "Field 1", - }, - ], - } - - const result = convertSchemaToCkan(schema) - - expect(result.fields).toHaveLength(1) - const field = result.fields[0] - expect(field).toBeDefined() - if (field) { - expect(field.id).toEqual("field1") - expect(field.type).toEqual("text") - expect(field.info).toBeDefined() - expect(field.info?.label).toEqual("Field 1") - expect(field.info?.notes).toBeUndefined() - expect(field.info?.type_override).toEqual("text") - } - }) - - it("handles fields with only description", () => { - const schema: Schema = { - fields: [ - { - name: "field1", - type: "string", - description: "Field 1 description", - }, - ], - } - - const result = convertSchemaToCkan(schema) - - expect(result.fields).toHaveLength(1) - const field = result.fields[0] - expect(field).toBeDefined() - if (field) { - expect(field.id).toEqual("field1") - expect(field.type).toEqual("text") - expect(field.info).toBeDefined() - expect(field.info?.label).toBeUndefined() - expect(field.info?.notes).toEqual("Field 1 description") - expect(field.info?.type_override).toEqual("text") - } - }) - - it("handles fields without title or description", () => { - const schema: Schema = { - fields: [ - { - name: "simple_field", - type: "string", - }, - ], - } - - const result = convertSchemaToCkan(schema) - - expect(result.fields).toHaveLength(1) - const field = result.fields[0] - expect(field).toBeDefined() - if (field) { - expect(field.id).toEqual("simple_field") - expect(field.type).toEqual("text") - expect(field.info).toBeUndefined() - } - }) - - it("handles empty fields array", () => { - const schema: Schema = { - fields: [], - } - - const result = convertSchemaToCkan(schema) - - expect(result.fields).toEqual([]) - }) - - it("converts unknown field types to 'text'", () => { - const schema: Schema = { - fields: [ - { - name: "unknown_field", - type: "unknown" as any, - }, - ], - } - - const result = convertSchemaToCkan(schema) - - expect(result.fields).toHaveLength(1) - const field = result.fields[0] - expect(field).toBeDefined() - if (field) { - expect(field.type).toEqual("text") - } - }) - - it("performs a round-trip conversion (CKAN → Frictionless → CKAN)", () => { - const originalCkanSchema = ckanSchemaFixture as CkanSchema - - const frictionlessSchema = convertSchemaFromCkan(originalCkanSchema) - - const resultCkanSchema = convertSchemaToCkan(frictionlessSchema) - - expect(resultCkanSchema.fields).toHaveLength( - originalCkanSchema.fields.length, - ) - - originalCkanSchema.fields.forEach(originalField => { - const resultField = resultCkanSchema.fields.find( - f => f.id === originalField.id, - ) - expect(resultField).toBeDefined() - - if (resultField && originalField) { - expect(resultField.id).toEqual(originalField.id) - - if (originalField.info) { - expect(resultField.info).toBeDefined() - if (resultField.info) { - if (originalField.info.label) { - expect(resultField.info.label).toEqual(originalField.info.label) - } - if (originalField.info.notes) { - expect(resultField.info.notes).toEqual(originalField.info.notes) - } - } - } - } - }) - }) -}) diff --git a/dataset/plugins/ckan/schema/convert/toCkan.ts b/dataset/plugins/ckan/schema/convert/toCkan.ts deleted file mode 100644 index 889c5f2e..00000000 --- a/dataset/plugins/ckan/schema/convert/toCkan.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { Field, Schema } from "@dpkit/metadata" -import type { CkanField, CkanFieldInfo } from "../Field.ts" -import type { CkanSchema } from "../Schema.ts" - -export function convertSchemaToCkan(schema: Schema): CkanSchema { - const fields = schema.fields.map(convertField) - - return { fields } -} - -function convertField(field: Field): CkanField { - const { name, title, description, type } = field - - const ckanField: CkanField = { - id: name, - type: convertType(type), - } - - if (title || description) { - const fieldInfo: CkanFieldInfo = {} as CkanFieldInfo - - if (title) fieldInfo.label = title - if (description) fieldInfo.notes = description - - fieldInfo.type_override = convertType(type) - - ckanField.info = fieldInfo - } - - return ckanField -} - -function convertType(type?: string): string { - switch (type) { - case "string": - return "text" - case "integer": - return "int" - case "number": - return "numeric" - case "boolean": - return "bool" - case "date": - return "date" - case "time": - return "time" - case "datetime": - return "timestamp" - case "object": - return "json" - case "array": - return "array" - case "geopoint": - return "geopoint" - case "geojson": - return "geojson" - default: - return "text" - } -} diff --git a/dataset/plugins/ckan/schema/index.ts b/dataset/plugins/ckan/schema/index.ts deleted file mode 100644 index c2b6aa4d..00000000 --- a/dataset/plugins/ckan/schema/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type { CkanSchema } from "./Schema.ts" -export type { CkanField } from "./Field.ts" -export { convertSchemaFromCkan } from "./convert/fromCkan.ts" -export { convertSchemaToCkan } from "./convert/toCkan.ts" diff --git a/dataset/plugins/datahub/index.ts b/dataset/plugins/datahub/index.ts deleted file mode 100644 index 8e03d380..00000000 --- a/dataset/plugins/datahub/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./package/index.ts" -export * from "./plugin.ts" diff --git a/dataset/plugins/datahub/package/index.ts b/dataset/plugins/datahub/package/index.ts deleted file mode 100644 index 548ee727..00000000 --- a/dataset/plugins/datahub/package/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { loadPackageFromDatahub } from "./load.ts" diff --git a/dataset/plugins/datahub/package/load.spec.ts b/dataset/plugins/datahub/package/load.spec.ts deleted file mode 100644 index b2a801a4..00000000 --- a/dataset/plugins/datahub/package/load.spec.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromDatahub } from "./load.ts" - -useRecording() - -describe.skip("loadPackageFromDatahub", () => { - it("should load a package", async () => { - const dataPackage = await loadPackageFromDatahub( - "https://datahub.io/core/eu-emissions-trading-system#readme", - ) - - expect(dataPackage).toMatchSnapshot() - }) -}) diff --git a/dataset/plugins/datahub/package/load.ts b/dataset/plugins/datahub/package/load.ts deleted file mode 100644 index a7a80ff0..00000000 --- a/dataset/plugins/datahub/package/load.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { loadPackageDescriptor } from "@dpkit/metadata" - -export async function loadPackageFromDatahub(datasetUrl: string) { - const url = new URL(datasetUrl) - - url.pathname = `${url.pathname}/datapackage.json` - url.search = "" - url.hash = "" - - return loadPackageDescriptor(url.toString()) -} diff --git a/dataset/plugins/datahub/plugin.spec.ts b/dataset/plugins/datahub/plugin.spec.ts deleted file mode 100644 index 197f4c5b..00000000 --- a/dataset/plugins/datahub/plugin.spec.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/index.ts" -import { DatahubPlugin } from "./plugin.ts" - -vi.mock("./package/index.ts", () => ({ - loadPackageFromDatahub: vi.fn(), -})) - -describe("DatahubPlugin", () => { - let plugin: DatahubPlugin - let mockLoadPackageFromDatahub: ReturnType - - beforeEach(() => { - plugin = new DatahubPlugin() - mockLoadPackageFromDatahub = vi.mocked(packageModule.loadPackageFromDatahub) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from datahub.io url", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromDatahub.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://datahub.io/core/country-codes", - ) - - expect(mockLoadPackageFromDatahub).toHaveBeenCalledWith( - "https://datahub.io/core/country-codes", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for non-datahub urls", async () => { - const result = await plugin.loadPackage("https://example.com/data") - - expect(mockLoadPackageFromDatahub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local paths", async () => { - const result = await plugin.loadPackage("./data") - - expect(mockLoadPackageFromDatahub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for github urls", async () => { - const result = await plugin.loadPackage("https://github.com/owner/repo") - - expect(mockLoadPackageFromDatahub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle datahub urls with multiple path segments", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromDatahub.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://datahub.io/core/gdp/datapackage.json", - ) - - expect(mockLoadPackageFromDatahub).toHaveBeenCalledWith( - "https://datahub.io/core/gdp/datapackage.json", - ) - expect(result).toEqual(mockPackage) - }) - - it("should handle datahub urls with query parameters", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromDatahub.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://datahub.io/core/country-codes?format=json", - ) - - expect(mockLoadPackageFromDatahub).toHaveBeenCalledWith( - "https://datahub.io/core/country-codes?format=json", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for http non-datahub urls", async () => { - const result = await plugin.loadPackage("http://example.com/data") - - expect(mockLoadPackageFromDatahub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for zenodo urls", async () => { - const result = await plugin.loadPackage("https://zenodo.org/record/123") - - expect(mockLoadPackageFromDatahub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for subdomain datahub urls", async () => { - const result = await plugin.loadPackage("https://test.datahub.io/data") - - expect(mockLoadPackageFromDatahub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/datahub/plugin.ts b/dataset/plugins/datahub/plugin.ts deleted file mode 100644 index 37c8101b..00000000 --- a/dataset/plugins/datahub/plugin.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { isRemotePath } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" -import { loadPackageFromDatahub } from "./package/index.ts" - -export class DatahubPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isDatahub = getIsDatahub(source) - if (!isDatahub) return undefined - - const dataPackage = await loadPackageFromDatahub(source) - return dataPackage - } -} - -function getIsDatahub(path: string) { - const isRemote = isRemotePath(path) - if (!isRemote) return false - - return new URL(path).hostname === "datahub.io" -} diff --git a/dataset/plugins/descriptor/index.ts b/dataset/plugins/descriptor/index.ts deleted file mode 100644 index 3959722b..00000000 --- a/dataset/plugins/descriptor/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./plugin.ts" diff --git a/dataset/plugins/descriptor/plugin.spec.ts b/dataset/plugins/descriptor/plugin.spec.ts deleted file mode 100644 index b2676ce0..00000000 --- a/dataset/plugins/descriptor/plugin.spec.ts +++ /dev/null @@ -1,227 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import * as metadataModule from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { DescriptorPlugin } from "./plugin.ts" - -vi.mock("@dpkit/metadata", async () => { - const actual = await vi.importActual("@dpkit/metadata") - return { - ...actual, - loadPackageDescriptor: vi.fn(), - savePackageDescriptor: vi.fn(), - } -}) - -describe("DescriptorPlugin", () => { - let plugin: DescriptorPlugin - let mockLoadPackageDescriptor: ReturnType - let mockSavePackageDescriptor: ReturnType - - beforeEach(() => { - plugin = new DescriptorPlugin() - mockLoadPackageDescriptor = vi.mocked(metadataModule.loadPackageDescriptor) - mockSavePackageDescriptor = vi.mocked(metadataModule.savePackageDescriptor) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from local datapackage.json file", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageDescriptor.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("./datapackage.json") - - expect(mockLoadPackageDescriptor).toHaveBeenCalledWith( - "./datapackage.json", - ) - expect(result).toEqual(mockPackage) - }) - - it("should load package from local json file", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageDescriptor.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("./package.json") - - expect(mockLoadPackageDescriptor).toHaveBeenCalledWith("./package.json") - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for remote json urls", async () => { - const result = await plugin.loadPackage( - "https://example.com/datapackage.json", - ) - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for http remote json urls", async () => { - const result = await plugin.loadPackage( - "http://example.com/datapackage.json", - ) - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local csv files", async () => { - const result = await plugin.loadPackage("./data.csv") - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local xlsx files", async () => { - const result = await plugin.loadPackage("./data.xlsx") - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local parquet files", async () => { - const result = await plugin.loadPackage("./data.parquet") - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle absolute paths", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageDescriptor.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("/absolute/path/datapackage.json") - - expect(mockLoadPackageDescriptor).toHaveBeenCalledWith( - "/absolute/path/datapackage.json", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for github urls", async () => { - const result = await plugin.loadPackage( - "https://github.com/owner/repo/datapackage.json", - ) - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for zenodo urls", async () => { - const result = await plugin.loadPackage("https://zenodo.org/record/123") - - expect(mockLoadPackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("savePackage", () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - - it("should save package to local datapackage.json file", async () => { - mockSavePackageDescriptor.mockResolvedValue(undefined) - - const result = await plugin.savePackage(mockPackage, { - target: "./datapackage.json", - }) - - expect(mockSavePackageDescriptor).toHaveBeenCalledWith(mockPackage, { - path: "./datapackage.json", - }) - expect(result).toEqual({ path: "./datapackage.json" }) - }) - - it("should save package with absolute path", async () => { - mockSavePackageDescriptor.mockResolvedValue(undefined) - - const result = await plugin.savePackage(mockPackage, { - target: "/absolute/path/datapackage.json", - }) - - expect(mockSavePackageDescriptor).toHaveBeenCalledWith(mockPackage, { - path: "/absolute/path/datapackage.json", - }) - expect(result).toEqual({ path: "/absolute/path/datapackage.json" }) - }) - - it("should return undefined for remote urls", async () => { - const result = await plugin.savePackage(mockPackage, { - target: "https://example.com/datapackage.json", - }) - - expect(mockSavePackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local json files not named datapackage.json", async () => { - const result = await plugin.savePackage(mockPackage, { - target: "./package.json", - }) - - expect(mockSavePackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local csv files", async () => { - const result = await plugin.savePackage(mockPackage, { - target: "./data.csv", - }) - - expect(mockSavePackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local xlsx files", async () => { - const result = await plugin.savePackage(mockPackage, { - target: "./data.xlsx", - }) - - expect(mockSavePackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for http urls", async () => { - const result = await plugin.savePackage(mockPackage, { - target: "http://example.com/datapackage.json", - }) - - expect(mockSavePackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should ignore withRemote option for local files", async () => { - mockSavePackageDescriptor.mockResolvedValue(undefined) - - const result = await plugin.savePackage(mockPackage, { - target: "./datapackage.json", - withRemote: true, - }) - - expect(mockSavePackageDescriptor).toHaveBeenCalledWith(mockPackage, { - path: "./datapackage.json", - }) - expect(result).toEqual({ path: "./datapackage.json" }) - }) - - it("should return undefined for local directories", async () => { - const result = await plugin.savePackage(mockPackage, { - target: "./data", - }) - - expect(mockSavePackageDescriptor).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/descriptor/plugin.ts b/dataset/plugins/descriptor/plugin.ts deleted file mode 100644 index 10e1e2b3..00000000 --- a/dataset/plugins/descriptor/plugin.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { inferFormat } from "@dpkit/metadata" -import type { Package } from "@dpkit/metadata" -import { isRemotePath } from "@dpkit/metadata" -import { loadPackageDescriptor } from "@dpkit/metadata" -import { savePackageDescriptor } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" - -export class DescriptorPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isLocalJson = await getIsLocalJson(source) - if (!isLocalJson) return undefined - - const dataPackage = await loadPackageDescriptor(source) - return dataPackage - } - - async savePackage( - dataPackage: Package, - options: { target: string; withRemote?: boolean }, - ) { - const isLocalJson = await getIsLocalJson(options.target) - if (!isLocalJson) return undefined - - if (!options.target.endsWith("datapackage.json")) { - return undefined - } - - await savePackageDescriptor(dataPackage, { path: options.target }) - - return { path: options.target } - } -} - -async function getIsLocalJson(path: string) { - const isRemote = isRemotePath(path) - const format = inferFormat({ path }) - return !isRemote && format === "json" -} diff --git a/dataset/plugins/folder/index.ts b/dataset/plugins/folder/index.ts deleted file mode 100644 index 8e03d380..00000000 --- a/dataset/plugins/folder/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./package/index.ts" -export * from "./plugin.ts" diff --git a/dataset/plugins/folder/package/index.ts b/dataset/plugins/folder/package/index.ts deleted file mode 100644 index 9efd23e2..00000000 --- a/dataset/plugins/folder/package/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadPackageFromFolder } from "./load.ts" -export { savePackageToFolder } from "./save.ts" diff --git a/dataset/plugins/folder/package/load.spec.ts b/dataset/plugins/folder/package/load.spec.ts deleted file mode 100644 index 29994bfa..00000000 --- a/dataset/plugins/folder/package/load.spec.ts +++ /dev/null @@ -1,205 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it } from "vitest" -import { getTempFilePath, writeTempFile } from "../../../file/index.ts" -import { loadPackageFromFolder } from "./load.ts" -import { savePackageToFolder } from "./save.ts" - -describe("loadPackageFromFolder", () => { - let tempFolderPath: string - - beforeEach(() => { - tempFolderPath = getTempFilePath() - }) - - it("should load a basic package from folder", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "empty-resource", - data: [], - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.name).toBe("test-package") - expect(loadedPackage.resources).toHaveLength(1) - }) - - it("should load package with metadata", async () => { - const originalPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test data package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage.name).toBe("test-package") - expect(loadedPackage.title).toBe("Test Package") - expect(loadedPackage.description).toBe("A test data package") - expect(loadedPackage.version).toBe("1.0.0") - }) - - it("should load package with inline data resources", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ], - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.resources).toHaveLength(1) - expect(loadedPackage.resources[0]?.name).toBe("test-resource") - expect(loadedPackage.resources[0]?.data).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should load package with file resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.resources).toHaveLength(1) - expect(loadedPackage.resources[0]?.name).toBe("test-resource") - expect(loadedPackage.resources[0]?.format).toBe("csv") - }) - - it("should load package with schema", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage.resources[0]?.schema).toBeDefined() - const schema = loadedPackage.resources[0]?.schema - expect(typeof schema === "object" && "fields" in schema).toBe(true) - if (typeof schema === "object" && "fields" in schema) { - expect(schema.fields).toHaveLength(2) - } - }) - - it("should load package with multiple resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "resource-1", - path: csvPath, - format: "csv", - }, - { - name: "resource-2", - data: [{ id: 1, value: 100 }], - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.name).toBe("test-package") - expect(loadedPackage.resources).toHaveLength(2) - expect(loadedPackage.resources[0]?.name).toBe("resource-1") - expect(loadedPackage.resources[1]?.name).toBe("resource-2") - }) - - it("should load package with dialect", async () => { - const csvContent = "id;name\n1;alice\n2;bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - dialect: { - delimiter: ";", - }, - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const loadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(loadedPackage.resources[0]?.dialect).toBeDefined() - const dialect = loadedPackage.resources[0]?.dialect - expect(typeof dialect === "object" && "delimiter" in dialect).toBe(true) - if (typeof dialect === "object" && "delimiter" in dialect) { - expect(dialect.delimiter).toBe(";") - } - }) - - it("should throw error for non-existent folder", async () => { - const nonExistentPath = "/non/existent/folder" - - await expect(loadPackageFromFolder(nonExistentPath)).rejects.toThrow() - }) - - it("should throw error for folder without datapackage.json", async () => { - const emptyFolderPath = getTempFilePath() - const fs = await import("node:fs/promises") - await fs.mkdir(emptyFolderPath, { recursive: true }) - - await expect(loadPackageFromFolder(emptyFolderPath)).rejects.toThrow() - }) -}) diff --git a/dataset/plugins/folder/package/load.ts b/dataset/plugins/folder/package/load.ts deleted file mode 100644 index 703b8461..00000000 --- a/dataset/plugins/folder/package/load.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { join } from "node:path" -import { loadPackageDescriptor } from "@dpkit/metadata" - -export async function loadPackageFromFolder(folderPath: string) { - return loadPackageDescriptor(join(folderPath, "datapackage.json")) -} diff --git a/dataset/plugins/folder/package/save.spec.ts b/dataset/plugins/folder/package/save.spec.ts deleted file mode 100644 index c8e81d42..00000000 --- a/dataset/plugins/folder/package/save.spec.ts +++ /dev/null @@ -1,389 +0,0 @@ -import { access, readFile } from "node:fs/promises" -import { join } from "node:path" -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it } from "vitest" -import { getTempFilePath, writeTempFile } from "../../../file/index.ts" -import { loadPackageFromFolder } from "./load.ts" -import { savePackageToFolder } from "./save.ts" - -describe("savePackageToFolder", () => { - let tempFolderPath: string - - beforeEach(() => { - tempFolderPath = getTempFilePath() - }) - - it("should save a basic package to folder", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save package with metadata", async () => { - const dataPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save package with inline data resources", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ], - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save package with file resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save package with multiple resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "resource-1", - path: csvPath, - format: "csv", - }, - { - name: "resource-2", - data: [{ id: 1, value: 100 }], - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save package with schema", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save package with dialect", async () => { - const csvContent = "id;name\n1;alice\n2;bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - dialect: { - delimiter: ";", - }, - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - await expect(access(descriptorPath)).resolves.toBeUndefined() - }) - - it("should save and reload package with same structure", async () => { - const originalPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const reloadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(reloadedPackage).toBeDefined() - expect(reloadedPackage.name).toBe("test-package") - expect(reloadedPackage.title).toBe("Test Package") - expect(reloadedPackage.description).toBe("A test package") - expect(reloadedPackage.resources).toHaveLength(1) - expect(reloadedPackage.resources[0]?.name).toBe("test-resource") - }) - - it("should save and reload package preserving metadata", async () => { - const originalPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - keywords: ["test", "package"], - resources: [ - { - name: "test-resource", - data: [{ id: 1 }], - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const reloadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(reloadedPackage.name).toBe("test-package") - expect(reloadedPackage.title).toBe("Test Package") - expect(reloadedPackage.version).toBe("1.0.0") - expect(reloadedPackage.keywords).toEqual(["test", "package"]) - }) - - it("should save and reload package with schema", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const reloadedPackage = await loadPackageFromFolder(tempFolderPath) - - const schema = reloadedPackage.resources[0]?.schema - expect(schema).toBeDefined() - expect(typeof schema === "object" && "fields" in schema).toBe(true) - if (typeof schema === "object" && "fields" in schema) { - expect(schema.fields).toHaveLength(2) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") - } - }) - - it("should save and reload package with file resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const reloadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(reloadedPackage.resources).toHaveLength(1) - expect(reloadedPackage.resources[0]?.name).toBe("test-resource") - expect(reloadedPackage.resources[0]?.format).toBe("csv") - }) - - it("should throw error when saving to existing folder", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - const fs = await import("node:fs/promises") - await fs.mkdir(tempFolderPath, { recursive: true }) - await fs.writeFile(join(tempFolderPath, "existing.txt"), "content") - - await expect( - savePackageToFolder(dataPackage, { folderPath: tempFolderPath }), - ).rejects.toThrow() - }) - - it("should create valid folder structure", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1 }], - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - const reloadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(reloadedPackage).toMatchObject({ - name: "test-package", - resources: [ - { - name: "test-resource", - }, - ], - }) - }) - - it("should save package with multiple file resources", async () => { - const csv1Content = "id,name\n1,alice" - const csv2Content = "id,value\n1,100" - const csv1Path = await writeTempFile(csv1Content) - const csv2Path = await writeTempFile(csv2Content) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "resource-1", - path: csv1Path, - format: "csv", - }, - { - name: "resource-2", - path: csv2Path, - format: "csv", - }, - ], - } - - await savePackageToFolder(originalPackage, { folderPath: tempFolderPath }) - const reloadedPackage = await loadPackageFromFolder(tempFolderPath) - - expect(reloadedPackage.resources).toHaveLength(2) - expect(reloadedPackage.resources[0]?.name).toBe("resource-1") - expect(reloadedPackage.resources[1]?.name).toBe("resource-2") - }) - - it("should create datapackage.json in folder", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1 }], - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - const descriptorContent = await readFile(descriptorPath, "utf-8") - const descriptor = JSON.parse(descriptorContent) - - expect(descriptor.name).toBe("test-package") - expect(descriptor.resources).toHaveLength(1) - }) - - it("should copy file resources to folder", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToFolder(dataPackage, { folderPath: tempFolderPath }) - - const descriptorPath = join(tempFolderPath, "datapackage.json") - const descriptorContent = await readFile(descriptorPath, "utf-8") - const descriptor = JSON.parse(descriptorContent) - - const resourcePath = descriptor.resources[0].path - const resourceFilePath = join(tempFolderPath, resourcePath) - const resourceContent = await readFile(resourceFilePath, "utf-8") - - expect(resourceContent).toBe(csvContent) - }) -}) diff --git a/dataset/plugins/folder/package/save.ts b/dataset/plugins/folder/package/save.ts deleted file mode 100644 index e5d51bad..00000000 --- a/dataset/plugins/folder/package/save.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { join } from "node:path" -import { convertPackageToDescriptor, saveDescriptor } from "@dpkit/metadata" -import type { Descriptor, Package } from "@dpkit/metadata" -import { assertLocalPathVacant, copyFile } from "../../../file/index.ts" -import { createFolder } from "../../../folder/index.ts" -import { getPackageBasepath } from "../../../package/index.ts" -import { saveResourceFiles } from "../../../resource/index.ts" - -export async function savePackageToFolder( - dataPackage: Package, - options: { - folderPath: string - withRemote?: boolean - }, -) { - const basepath = getPackageBasepath(dataPackage) - const { folderPath, withRemote } = options - - await assertLocalPathVacant(folderPath) - await createFolder(folderPath) - - const resourceDescriptors: Descriptor[] = [] - for (const resource of dataPackage.resources) { - resourceDescriptors.push( - await saveResourceFiles(resource, { - basepath, - withRemote, - saveFile: async options => { - await copyFile({ - sourcePath: options.normalizedPath, - targetPath: join(folderPath, options.denormalizedPath), - }) - - return options.denormalizedPath - }, - }), - ) - } - - const descriptor = { - ...convertPackageToDescriptor(dataPackage, { basepath }), - resources: resourceDescriptors, - } - - await saveDescriptor(descriptor, { - path: join(folderPath, "datapackage.json"), - }) - - return descriptor -} diff --git a/dataset/plugins/folder/plugin.spec.ts b/dataset/plugins/folder/plugin.spec.ts deleted file mode 100644 index 3226a8e3..00000000 --- a/dataset/plugins/folder/plugin.spec.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/index.ts" -import { FolderPlugin } from "./plugin.ts" - -vi.mock("./package/index.ts", () => ({ - loadPackageFromFolder: vi.fn(), -})) - -describe("FolderPlugin", () => { - let plugin: FolderPlugin - let mockLoadPackageFromFolder: ReturnType - - beforeEach(() => { - plugin = new FolderPlugin() - mockLoadPackageFromFolder = vi.mocked(packageModule.loadPackageFromFolder) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from local directory", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromFolder.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage(".") - - expect(mockLoadPackageFromFolder).toHaveBeenCalledWith(".") - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for remote paths", async () => { - const result = await plugin.loadPackage("http://example.com/data") - - expect(mockLoadPackageFromFolder).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for https paths", async () => { - const result = await plugin.loadPackage("https://example.com/data") - - expect(mockLoadPackageFromFolder).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for github urls", async () => { - const result = await plugin.loadPackage( - "https://github.com/owner/repo/data", - ) - - expect(mockLoadPackageFromFolder).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for ftp paths", async () => { - const result = await plugin.loadPackage("ftp://example.com/data") - - expect(mockLoadPackageFromFolder).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for zenodo urls", async () => { - const result = await plugin.loadPackage("https://zenodo.org/record/123") - - expect(mockLoadPackageFromFolder).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/folder/plugin.ts b/dataset/plugins/folder/plugin.ts deleted file mode 100644 index 58f6f7f5..00000000 --- a/dataset/plugins/folder/plugin.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { stat } from "node:fs/promises" -import { isRemotePath } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" -import { loadPackageFromFolder } from "./package/index.ts" - -export class FolderPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isFolder = await getIsFolder(source) - if (!isFolder) return undefined - - const dataPackage = await loadPackageFromFolder(source) - return dataPackage - } - - // TOOD: implement savePackage? -} - -async function getIsFolder(path: string) { - const isRemote = isRemotePath(path) - if (isRemote) return false - - return (await stat(path)).isDirectory() -} diff --git a/dataset/plugins/github/github/index.ts b/dataset/plugins/github/github/index.ts deleted file mode 100644 index 72213bd6..00000000 --- a/dataset/plugins/github/github/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { makeGithubApiRequest } from "./request.ts" diff --git a/dataset/plugins/github/github/path.ts b/dataset/plugins/github/github/path.ts deleted file mode 100644 index 510e2120..00000000 --- a/dataset/plugins/github/github/path.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function normalizeFileLink(link: string) { - return link.replace("/api/", "/").replace(/\/content$/, "") -} diff --git a/dataset/plugins/github/github/request.ts b/dataset/plugins/github/github/request.ts deleted file mode 100644 index d070066f..00000000 --- a/dataset/plugins/github/github/request.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { Descriptor } from "@dpkit/metadata" - -/** - * Makes a request to the Github API - */ -export async function makeGithubApiRequest(options: { - endpoint: string - method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" - payload?: Descriptor - apiKey?: string - upload?: { - name: string - data: Blob - path?: string // Path within repository - } -}) { - const { endpoint, method = "GET", payload, upload, apiKey } = options - - let body: string | FormData | undefined - const headers: Record = {} - - if (apiKey) { - headers["Authorization"] = `Bearer ${apiKey}` - } - - // Create full API URL - const baseUrl = "https://api.github.com" - const url = `${baseUrl}${endpoint}` - - if (upload) { - body = new FormData() - body.append("file", upload.data, upload.name) - - if (payload) { - for (const [key, value] of Object.entries(payload)) { - if (typeof value === "object" && value !== null) { - body.append(key, JSON.stringify(value)) - } else { - body.append(key, String(value)) - } - } - } - } else if (payload) { - body = JSON.stringify(payload) - headers["Content-Type"] = "application/json" - } - - const response = await fetch(url, { - method, - headers, - body, - }) - - if (!response.ok || response.status === 204) { - const errorText = await response.text() - throw new Error( - `Github API error: ${response.status} ${response.statusText}\n${errorText}`, - ) - } - - return (await response.json()) as T -} diff --git a/dataset/plugins/github/index.ts b/dataset/plugins/github/index.ts deleted file mode 100644 index ae6326f6..00000000 --- a/dataset/plugins/github/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./package/index.ts" -export * from "./resource/index.ts" -export * from "./plugin.ts" diff --git a/dataset/plugins/github/package/License.ts b/dataset/plugins/github/package/License.ts deleted file mode 100644 index 886c0692..00000000 --- a/dataset/plugins/github/package/License.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * GitHub repository license - */ -export interface GithubLicense { - /** - * License key - */ - key: string - - /** - * License name - */ - name: string - - /** - * License SPDX ID - */ - spdx_id: string - - /** - * License URL - */ - url: string -} diff --git a/dataset/plugins/github/package/Owner.ts b/dataset/plugins/github/package/Owner.ts deleted file mode 100644 index f2982576..00000000 --- a/dataset/plugins/github/package/Owner.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * GitHub repository owner - */ -export interface GithubOwner { - /** - * Owner login name - */ - login: string - - /** - * Owner ID - */ - id: number - - /** - * Owner avatar URL - */ - avatar_url: string - - /** - * Owner URL - */ - html_url: string - - /** - * Owner type (User/Organization) - */ - type: "User" | "Organization" -} diff --git a/dataset/plugins/github/package/Package.ts b/dataset/plugins/github/package/Package.ts deleted file mode 100644 index 82bdc2e4..00000000 --- a/dataset/plugins/github/package/Package.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { GithubResource } from "../resource/index.ts" -import type { GithubLicense } from "./License.ts" -import type { GithubOwner } from "./Owner.ts" - -/** - * Github repository as a package - */ -export interface GithubPackage { - /** - * Repository identifier - */ - id: number - - /** - * Repository name - */ - name: string - - /** - * Repository full name (owner/name) - */ - full_name: string - - /** - * Repository owner - */ - owner: GithubOwner - - /** - * Repository description - */ - description: string | null - - /** - * Repository creation date - */ - created_at: string - - /** - * Repository update date - */ - updated_at: string - - /** - * Repository homepage URL - */ - homepage: string | null - - /** - * Repository size in KB - */ - size: number - - /** - * Repository stars count - */ - stargazers_count: number - - /** - * Repository watchers count - */ - watchers_count: number - - /** - * Repository language - */ - language: string | null - - /** - * Repository license - */ - license: GithubLicense | null - - /** - * Repository default branch - */ - default_branch: string - - /** - * Repository topics - */ - topics: string[] - - /** - * Repository is private - */ - private: boolean - - /** - * Repository is archived - */ - archived: boolean - - /** - * Repository URLs - */ - html_url: string - git_url: string - ssh_url: string - clone_url: string - - /** - * Repository resources - */ - resources?: GithubResource[] -} diff --git a/dataset/plugins/github/package/convert/fromGithub.ts b/dataset/plugins/github/package/convert/fromGithub.ts deleted file mode 100644 index 098fbcc2..00000000 --- a/dataset/plugins/github/package/convert/fromGithub.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { Contributor, License, Package } from "@dpkit/metadata" -import { convertResourceFromGithub } from "../../resource/index.ts" -import type { GithubPackage } from "../Package.ts" - -export function convertPackageFromGithub( - githubPackage: GithubPackage, -): Package { - const datapackage: Package = { - name: githubPackage.name, - resources: [], - } - - if (githubPackage.description) { - datapackage.description = githubPackage.description - } - - datapackage.title = githubPackage.full_name - - if (githubPackage.homepage) { - datapackage.homepage = githubPackage.homepage - } - - if (githubPackage.license) { - const license: License = { - name: githubPackage.license.spdx_id || githubPackage.license.key, - } - - if (githubPackage.license.name) { - license.title = githubPackage.license.name - } - - if (githubPackage.license.url) { - license.path = githubPackage.license.url - } - - datapackage.licenses = [license] - } - - if (githubPackage.owner) { - const contributor: Contributor = { - title: githubPackage.owner.login, - role: - githubPackage.owner.type === "Organization" ? "publisher" : "author", - path: githubPackage.owner.html_url, - } - - datapackage.contributors = [contributor] - } - - if (githubPackage.resources && githubPackage.resources.length > 0) { - datapackage.resources = githubPackage.resources - .filter(resource => !resource.path.startsWith(".")) - .filter(resource => resource.type === "blob") - .map(resource => - convertResourceFromGithub(resource, { - defaultBranch: githubPackage.default_branch, - }), - ) - } - - if (githubPackage.topics && githubPackage.topics.length > 0) { - datapackage.keywords = githubPackage.topics - } - - if (githubPackage.created_at) { - datapackage.created = githubPackage.created_at - } - - return datapackage -} diff --git a/dataset/plugins/github/package/fixtures/data.csv b/dataset/plugins/github/package/fixtures/data.csv deleted file mode 100644 index d888cb89..00000000 --- a/dataset/plugins/github/package/fixtures/data.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name,value -1,test,100 -2,example,200 diff --git a/dataset/plugins/github/package/fixtures/generated/load.spec.ts.snap b/dataset/plugins/github/package/fixtures/generated/load.spec.ts.snap deleted file mode 100644 index fadcca50..00000000 --- a/dataset/plugins/github/package/fixtures/generated/load.spec.ts.snap +++ /dev/null @@ -1,203 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`loadPackageFromGithub > should load a package 1`] = ` -{ - "contributors": [ - { - "path": "https://github.com/roll", - "role": "author", - "title": "roll", - }, - ], - "created": "2019-11-11T10:48:02Z", - "name": "data", - "resources": [ - { - "bytes": 6, - "format": "md", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:31cd35605bbee1d3b3f297d5b0462028a46a0380", - "name": "readme", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/README.md", - }, - { - "bytes": 67920590, - "format": "zip", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:7c32a3955b961c3fe7646fac499177ab5e5af524", - "name": "data", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/data.zip", - }, - { - "bytes": 192, - "format": "json", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:8905b138ed93ebb9e78945126ac2a423d31babc9", - "name": "meta", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/meta.json", - }, - { - "bytes": 1000000, - "format": "csv", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:f79b42b814066ec87cc734d932c64733c47444df", - "name": "table1", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table1.csv", - }, - { - "bytes": 10000000, - "format": "csv", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:f352795df6534642cc0a9fe6d9b2b3f8fa80135a", - "name": "table10", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table10.csv", - }, - { - "bytes": 100000000, - "format": "csv", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:734520644f4616825209c0c7a0750e5887b93a76", - "name": "table100", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table100.csv", - }, - { - "bytes": 5000000, - "format": "csv", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:306b2975e0e469a2eec853b1a4856acb25a1dab6", - "name": "table5", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table5.csv", - }, - { - "bytes": 50000000, - "format": "csv", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:eca005a30f75202d7ccdadbbb6789a9ae0389555", - "name": "table50", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table50.csv", - }, - { - "bytes": 513, - "format": "json", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:4564795a02255050972182a43590b853dee8b8b5", - "name": "package", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/package.json", - }, - { - "bytes": 1952, - "format": "js", - "github:key": undefined, - "github:url": undefined, - "hash": "sha1:59f443ca5062f3f84937490509d6cc25571fb853", - "name": "generate", - "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/scripts/generate.js", - }, - ], - "title": "roll/data", -} -`; - -exports[`loadPackageFromGithub > should merge datapackage.json if present 1`] = ` -{ - "$schema": undefined, - "contributors": [ - { - "email": "rufus.pollock@okfn.org", - "role": undefined, - "roles": [ - "maintainer", - ], - "title": "Rufus Pollock", - }, - { - "email": "kdkusano@gmail.com", - "title": "Kristofer D. Kusano", - }, - ], - "created": "2021-07-07T14:09:49Z", - "description": "ISO 4217 List of Currencies and Currency Codes", - "homepage": "http://www.iso.org/iso/currency_codes", - "keywords": [ - "iso", - "iso-4217", - "currency", - "codes", - ], - "licenses": [ - { - "name": "ODC-PDDL-1.0", - "path": "http://opendatacommons.org/licenses/pddl/", - "title": "Open Data Commons Public Domain Dedication and License v1.0", - }, - ], - "name": "currency-codes", - "resources": [ - { - "$schema": undefined, - "github:key": undefined, - "github:url": undefined, - "mimetype": "text/csv", - "name": "codes-all", - "path": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/data/codes-all.csv", - "schema": { - "$schema": undefined, - "fields": [ - { - "description": "Country or region name", - "name": "Entity", - "type": "string", - }, - { - "description": "Name of the currency", - "name": "Currency", - "type": "string", - }, - { - "description": "3 digit alphabetic code for the currency", - "name": "AlphabeticCode", - "title": "Alphabetic Code", - "type": "string", - }, - { - "description": "3 digit numeric code", - "name": "NumericCode", - "title": "Numeric Code", - "type": "number", - }, - { - "description": "", - "name": "MinorUnit", - "title": "Minor Unit", - "type": "string", - }, - { - "description": "Date currency withdrawn (values can be ranges or months", - "name": "WithdrawalDate", - "title": "Withdrawal Date", - "type": "string", - }, - ], - }, - "size": "16863", - }, - ], - "sources": [ - { - "email": "office@currency-iso.org", - "name": "SIX Interbank Clearing Ltd (on behalf of ISO)", - "title": "SIX Interbank Clearing Ltd (on behalf of ISO)", - }, - ], - "title": "ISO 4217 Currency Codes", -} -`; diff --git a/dataset/plugins/github/package/fixtures/generated/loadPackageFromGithub-should-load-a-package_1044819575/recording.har b/dataset/plugins/github/package/fixtures/generated/loadPackageFromGithub-should-load-a-package_1044819575/recording.har deleted file mode 100644 index eb7d49e9..00000000 --- a/dataset/plugins/github/package/fixtures/generated/loadPackageFromGithub-should-load-a-package_1044819575/recording.har +++ /dev/null @@ -1,311 +0,0 @@ -{ - "log": { - "_recordingName": "loadPackageFromGithub-should load a package", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "af6e29ae2e06d9b1710553904135742d", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 57, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://api.github.com/repos/roll/data" - }, - "response": { - "bodySize": 1236, - "content": { - "mimeType": "application/json; charset=utf-8", - "size": 1236, - "text": "{\"id\":220959516,\"node_id\":\"MDEwOlJlcG9zaXRvcnkyMjA5NTk1MTY=\",\"name\":\"data\",\"full_name\":\"roll/data\",\"private\":false,\"owner\":{\"login\":\"roll\",\"id\":557395,\"node_id\":\"MDQ6VXNlcjU1NzM5NQ==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/557395?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/roll\",\"html_url\":\"https://github.com/roll\",\"followers_url\":\"https://api.github.com/users/roll/followers\",\"following_url\":\"https://api.github.com/users/roll/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/roll/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/roll/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/roll/subscriptions\",\"organizations_url\":\"https://api.github.com/users/roll/orgs\",\"repos_url\":\"https://api.github.com/users/roll/repos\",\"events_url\":\"https://api.github.com/users/roll/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/roll/received_events\",\"type\":\"User\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/roll/data\",\"description\":null,\"fork\":false,\"url\":\"https://api.github.com/repos/roll/data\",\"forks_url\":\"https://api.github.com/repos/roll/data/forks\",\"keys_url\":\"https://api.github.com/repos/roll/data/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/roll/data/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/roll/data/teams\",\"hooks_url\":\"https://api.github.com/repos/roll/data/hooks\",\"issue_events_url\":\"https://api.github.com/repos/roll/data/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/roll/data/events\",\"assignees_url\":\"https://api.github.com/repos/roll/data/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/roll/data/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/roll/data/tags\",\"blobs_url\":\"https://api.github.com/repos/roll/data/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/roll/data/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/roll/data/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/roll/data/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/roll/data/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/roll/data/languages\",\"stargazers_url\":\"https://api.github.com/repos/roll/data/stargazers\",\"contributors_url\":\"https://api.github.com/repos/roll/data/contributors\",\"subscribers_url\":\"https://api.github.com/repos/roll/data/subscribers\",\"subscription_url\":\"https://api.github.com/repos/roll/data/subscription\",\"commits_url\":\"https://api.github.com/repos/roll/data/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/roll/data/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/roll/data/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/roll/data/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/roll/data/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/roll/data/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/roll/data/merges\",\"archive_url\":\"https://api.github.com/repos/roll/data/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/roll/data/downloads\",\"issues_url\":\"https://api.github.com/repos/roll/data/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/roll/data/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/roll/data/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/roll/data/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/roll/data/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/roll/data/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/roll/data/deployments\",\"created_at\":\"2019-11-11T10:48:02Z\",\"updated_at\":\"2020-05-14T07:12:23Z\",\"pushed_at\":\"2020-05-14T07:12:13Z\",\"git_url\":\"git://github.com/roll/data.git\",\"ssh_url\":\"git@github.com:roll/data.git\",\"clone_url\":\"https://github.com/roll/data.git\",\"svn_url\":\"https://github.com/roll/data\",\"homepage\":null,\"size\":536228,\"stargazers_count\":1,\"watchers_count\":1,\"language\":\"JavaScript\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":0,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":0,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":0,\"open_issues\":0,\"watchers\":1,\"default_branch\":\"master\",\"temp_clone_token\":null,\"network_count\":0,\"subscribers_count\":0}" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "access-control-expose-headers", - "value": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset" - }, - { - "name": "cache-control", - "value": "public, max-age=60, s-maxage=60" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "1236" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'" - }, - { - "name": "content-type", - "value": "application/json; charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:25 GMT" - }, - { - "name": "etag", - "value": "W/\"cabe7e42ada08dc05ec06db40df679feb6e58d41f59407b819ccc4e49326c490\"" - }, - { - "name": "last-modified", - "value": "Thu, 14 May 2020 07:12:23 GMT" - }, - { - "name": "referrer-policy", - "value": "origin-when-cross-origin, strict-origin-when-cross-origin" - }, - { - "name": "server", - "value": "github.com" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000; includeSubdomains; preload" - }, - { - "name": "vary", - "value": "Accept,Accept-Encoding, Accept, X-Requested-With" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-api-version-selected", - "value": "2022-11-28" - }, - { - "name": "x-github-media-type", - "value": "github.v3; format=json" - }, - { - "name": "x-github-request-id", - "value": "2359:39D094:F0DA73:E1E475:68945A85" - }, - { - "name": "x-ratelimit-limit", - "value": "60" - }, - { - "name": "x-ratelimit-remaining", - "value": "59" - }, - { - "name": "x-ratelimit-reset", - "value": "1754556565" - }, - { - "name": "x-ratelimit-resource", - "value": "core" - }, - { - "name": "x-ratelimit-used", - "value": "1" - }, - { - "name": "x-xss-protection", - "value": "0" - } - ], - "headersSize": 1294, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:25.318Z", - "time": 520, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 520 - } - }, - { - "_id": "5c1796f31beae7f63ee90a22a252053d", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 86, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [ - { - "name": "recursive", - "value": "1" - } - ], - "url": "https://api.github.com/repos/roll/data/git/trees/master?recursive=1" - }, - "response": { - "bodySize": 798, - "content": { - "mimeType": "application/json; charset=utf-8", - "size": 798, - "text": "{\"sha\":\"8094ea97a76e9435f90a9a73398754d7ec94cfac\",\"url\":\"https://api.github.com/repos/roll/data/git/trees/8094ea97a76e9435f90a9a73398754d7ec94cfac\",\"tree\":[{\"path\":\".gitignore\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"1f91d773a9b5e361a699bc10eff921c841831612\",\"size\":787,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/1f91d773a9b5e361a699bc10eff921c841831612\"},{\"path\":\".prettierrc\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"36a4660e1d068962321affe174c604c8b5bfa6d5\",\"size\":90,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/36a4660e1d068962321affe174c604c8b5bfa6d5\"},{\"path\":\"README.md\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"31cd35605bbee1d3b3f297d5b0462028a46a0380\",\"size\":6,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/31cd35605bbee1d3b3f297d5b0462028a46a0380\"},{\"path\":\"data\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"6bec63ab526a9fb256f4681c48b9f760b5675eba\",\"url\":\"https://api.github.com/repos/roll/data/git/trees/6bec63ab526a9fb256f4681c48b9f760b5675eba\"},{\"path\":\"data/data.zip\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"7c32a3955b961c3fe7646fac499177ab5e5af524\",\"size\":67920590,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/7c32a3955b961c3fe7646fac499177ab5e5af524\"},{\"path\":\"data/meta.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"8905b138ed93ebb9e78945126ac2a423d31babc9\",\"size\":192,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/8905b138ed93ebb9e78945126ac2a423d31babc9\"},{\"path\":\"data/table1.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"f79b42b814066ec87cc734d932c64733c47444df\",\"size\":1000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/f79b42b814066ec87cc734d932c64733c47444df\"},{\"path\":\"data/table10.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"f352795df6534642cc0a9fe6d9b2b3f8fa80135a\",\"size\":10000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/f352795df6534642cc0a9fe6d9b2b3f8fa80135a\"},{\"path\":\"data/table100.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"734520644f4616825209c0c7a0750e5887b93a76\",\"size\":100000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/734520644f4616825209c0c7a0750e5887b93a76\"},{\"path\":\"data/table5.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"306b2975e0e469a2eec853b1a4856acb25a1dab6\",\"size\":5000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/306b2975e0e469a2eec853b1a4856acb25a1dab6\"},{\"path\":\"data/table50.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"eca005a30f75202d7ccdadbbb6789a9ae0389555\",\"size\":50000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/eca005a30f75202d7ccdadbbb6789a9ae0389555\"},{\"path\":\"package.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"4564795a02255050972182a43590b853dee8b8b5\",\"size\":513,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/4564795a02255050972182a43590b853dee8b8b5\"},{\"path\":\"scripts\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"43a35f56f388fae0badbb2c5ce00489a982e796d\",\"url\":\"https://api.github.com/repos/roll/data/git/trees/43a35f56f388fae0badbb2c5ce00489a982e796d\"},{\"path\":\"scripts/generate.js\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"59f443ca5062f3f84937490509d6cc25571fb853\",\"size\":1952,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/59f443ca5062f3f84937490509d6cc25571fb853\"}],\"truncated\":false}" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "access-control-expose-headers", - "value": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset" - }, - { - "name": "cache-control", - "value": "public, max-age=60, s-maxage=60" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "798" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'" - }, - { - "name": "content-type", - "value": "application/json; charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:25 GMT" - }, - { - "name": "etag", - "value": "W/\"8b0b83664974e44f10a88c20b559e755b40b0569eb248c3d35681217d7acdd44\"" - }, - { - "name": "last-modified", - "value": "Thu, 14 May 2020 07:12:23 GMT" - }, - { - "name": "referrer-policy", - "value": "origin-when-cross-origin, strict-origin-when-cross-origin" - }, - { - "name": "server", - "value": "github.com" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000; includeSubdomains; preload" - }, - { - "name": "vary", - "value": "Accept,Accept-Encoding, Accept, X-Requested-With" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-api-version-selected", - "value": "2022-11-28" - }, - { - "name": "x-github-media-type", - "value": "github.v3; format=json" - }, - { - "name": "x-github-request-id", - "value": "2359:39D094:F0DC2F:E1E5F1:68945A85" - }, - { - "name": "x-ratelimit-limit", - "value": "60" - }, - { - "name": "x-ratelimit-remaining", - "value": "58" - }, - { - "name": "x-ratelimit-reset", - "value": "1754556565" - }, - { - "name": "x-ratelimit-resource", - "value": "core" - }, - { - "name": "x-ratelimit-used", - "value": "2" - }, - { - "name": "x-xss-protection", - "value": "0" - } - ], - "headersSize": 1293, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:25.841Z", - "time": 281, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 281 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/dataset/plugins/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har b/dataset/plugins/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har deleted file mode 100644 index e1ba3813..00000000 --- a/dataset/plugins/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har +++ /dev/null @@ -1,594 +0,0 @@ -{ - "log": { - "_recordingName": "loadPackageFromGithub-should merge datapackage.json if present", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "1a1b6f9f2e0280a7960e568ae456fc69", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 67, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://api.github.com/repos/roll/currency-codes" - }, - "response": { - "bodySize": 15526, - "content": { - "mimeType": "application/json; charset=utf-8", - "size": 15526, - "text": "{\"id\":383820850,\"node_id\":\"MDEwOlJlcG9zaXRvcnkzODM4MjA4NTA=\",\"name\":\"currency-codes\",\"full_name\":\"roll/currency-codes\",\"private\":false,\"owner\":{\"login\":\"roll\",\"id\":557395,\"node_id\":\"MDQ6VXNlcjU1NzM5NQ==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/557395?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/roll\",\"html_url\":\"https://github.com/roll\",\"followers_url\":\"https://api.github.com/users/roll/followers\",\"following_url\":\"https://api.github.com/users/roll/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/roll/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/roll/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/roll/subscriptions\",\"organizations_url\":\"https://api.github.com/users/roll/orgs\",\"repos_url\":\"https://api.github.com/users/roll/repos\",\"events_url\":\"https://api.github.com/users/roll/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/roll/received_events\",\"type\":\"User\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/roll/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":true,\"url\":\"https://api.github.com/repos/roll/currency-codes\",\"forks_url\":\"https://api.github.com/repos/roll/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/roll/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/roll/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/roll/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/roll/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/roll/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/roll/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/roll/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/roll/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/roll/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/roll/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/roll/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/roll/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/roll/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/roll/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/roll/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/roll/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/roll/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/roll/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/roll/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/roll/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/roll/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/roll/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/roll/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/roll/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/roll/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/roll/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/roll/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/roll/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/roll/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/roll/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/roll/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/roll/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/roll/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/roll/currency-codes/deployments\",\"created_at\":\"2021-07-07T14:09:49Z\",\"updated_at\":\"2021-07-07T14:47:03Z\",\"pushed_at\":\"2021-07-07T14:47:01Z\",\"git_url\":\"git://github.com/roll/currency-codes.git\",\"ssh_url\":\"git@github.com:roll/currency-codes.git\",\"clone_url\":\"https://github.com/roll/currency-codes.git\",\"svn_url\":\"https://github.com/roll/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":87,\"stargazers_count\":0,\"watchers_count\":0,\"language\":\"Shell\",\"has_issues\":false,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":0,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":0,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":0,\"open_issues\":0,\"watchers\":0,\"default_branch\":\"master\",\"temp_clone_token\":null,\"parent\":{\"id\":6696644,\"node_id\":\"MDEwOlJlcG9zaXRvcnk2Njk2NjQ0\",\"name\":\"currency-codes\",\"full_name\":\"datasets/currency-codes\",\"private\":false,\"owner\":{\"login\":\"datasets\",\"id\":1643515,\"node_id\":\"MDEyOk9yZ2FuaXphdGlvbjE2NDM1MTU=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/1643515?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/datasets\",\"html_url\":\"https://github.com/datasets\",\"followers_url\":\"https://api.github.com/users/datasets/followers\",\"following_url\":\"https://api.github.com/users/datasets/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/datasets/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/datasets/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/datasets/subscriptions\",\"organizations_url\":\"https://api.github.com/users/datasets/orgs\",\"repos_url\":\"https://api.github.com/users/datasets/repos\",\"events_url\":\"https://api.github.com/users/datasets/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/datasets/received_events\",\"type\":\"Organization\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/datasets/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":false,\"url\":\"https://api.github.com/repos/datasets/currency-codes\",\"forks_url\":\"https://api.github.com/repos/datasets/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/datasets/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/datasets/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/datasets/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/datasets/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/datasets/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/datasets/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/datasets/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/datasets/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/datasets/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/datasets/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/datasets/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/datasets/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/datasets/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/datasets/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/datasets/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/datasets/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/datasets/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/datasets/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/datasets/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/datasets/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/datasets/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/datasets/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/datasets/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/datasets/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/datasets/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/datasets/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/datasets/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/datasets/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/datasets/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/datasets/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/datasets/currency-codes/deployments\",\"created_at\":\"2012-11-14T23:19:47Z\",\"updated_at\":\"2025-08-02T21:09:44Z\",\"pushed_at\":\"2025-06-01T01:47:03Z\",\"git_url\":\"git://github.com/datasets/currency-codes.git\",\"ssh_url\":\"git@github.com:datasets/currency-codes.git\",\"clone_url\":\"https://github.com/datasets/currency-codes.git\",\"svn_url\":\"https://github.com/datasets/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":80,\"stargazers_count\":162,\"watchers_count\":162,\"language\":\"Shell\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":177,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":3,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":177,\"open_issues\":3,\"watchers\":162,\"default_branch\":\"main\"},\"source\":{\"id\":6696644,\"node_id\":\"MDEwOlJlcG9zaXRvcnk2Njk2NjQ0\",\"name\":\"currency-codes\",\"full_name\":\"datasets/currency-codes\",\"private\":false,\"owner\":{\"login\":\"datasets\",\"id\":1643515,\"node_id\":\"MDEyOk9yZ2FuaXphdGlvbjE2NDM1MTU=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/1643515?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/datasets\",\"html_url\":\"https://github.com/datasets\",\"followers_url\":\"https://api.github.com/users/datasets/followers\",\"following_url\":\"https://api.github.com/users/datasets/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/datasets/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/datasets/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/datasets/subscriptions\",\"organizations_url\":\"https://api.github.com/users/datasets/orgs\",\"repos_url\":\"https://api.github.com/users/datasets/repos\",\"events_url\":\"https://api.github.com/users/datasets/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/datasets/received_events\",\"type\":\"Organization\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/datasets/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":false,\"url\":\"https://api.github.com/repos/datasets/currency-codes\",\"forks_url\":\"https://api.github.com/repos/datasets/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/datasets/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/datasets/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/datasets/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/datasets/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/datasets/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/datasets/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/datasets/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/datasets/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/datasets/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/datasets/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/datasets/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/datasets/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/datasets/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/datasets/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/datasets/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/datasets/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/datasets/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/datasets/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/datasets/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/datasets/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/datasets/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/datasets/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/datasets/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/datasets/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/datasets/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/datasets/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/datasets/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/datasets/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/datasets/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/datasets/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/datasets/currency-codes/deployments\",\"created_at\":\"2012-11-14T23:19:47Z\",\"updated_at\":\"2025-08-02T21:09:44Z\",\"pushed_at\":\"2025-06-01T01:47:03Z\",\"git_url\":\"git://github.com/datasets/currency-codes.git\",\"ssh_url\":\"git@github.com:datasets/currency-codes.git\",\"clone_url\":\"https://github.com/datasets/currency-codes.git\",\"svn_url\":\"https://github.com/datasets/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":80,\"stargazers_count\":162,\"watchers_count\":162,\"language\":\"Shell\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":177,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":3,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":177,\"open_issues\":3,\"watchers\":162,\"default_branch\":\"main\"},\"network_count\":177,\"subscribers_count\":0}" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "access-control-expose-headers", - "value": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset" - }, - { - "name": "cache-control", - "value": "public, max-age=60, s-maxage=60" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'" - }, - { - "name": "content-type", - "value": "application/json; charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "etag", - "value": "W/\"5910a4781b626f2a41e1824960503e8a43c24b955cb667a2217c33eb1c444ee8\"" - }, - { - "name": "last-modified", - "value": "Wed, 07 Jul 2021 14:47:03 GMT" - }, - { - "name": "referrer-policy", - "value": "origin-when-cross-origin, strict-origin-when-cross-origin" - }, - { - "name": "server", - "value": "github.com" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000; includeSubdomains; preload" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "vary", - "value": "Accept,Accept-Encoding, Accept, X-Requested-With" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-api-version-selected", - "value": "2022-11-28" - }, - { - "name": "x-github-media-type", - "value": "github.v3; format=json" - }, - { - "name": "x-github-request-id", - "value": "2359:39D094:F0DE0F:E1E7C5:68945A85" - }, - { - "name": "x-ratelimit-limit", - "value": "60" - }, - { - "name": "x-ratelimit-remaining", - "value": "57" - }, - { - "name": "x-ratelimit-reset", - "value": "1754556565" - }, - { - "name": "x-ratelimit-resource", - "value": "core" - }, - { - "name": "x-ratelimit-used", - "value": "3" - }, - { - "name": "x-xss-protection", - "value": "0" - } - ], - "headersSize": 1300, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:26.129Z", - "time": 335, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 335 - } - }, - { - "_id": "392264ee4148eff756557a8163d81440", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 96, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [ - { - "name": "recursive", - "value": "1" - } - ], - "url": "https://api.github.com/repos/roll/currency-codes/git/trees/master?recursive=1" - }, - "response": { - "bodySize": 874, - "content": { - "mimeType": "application/json; charset=utf-8", - "size": 874, - "text": "{\"sha\":\"e855efbcd7344e9ad8d89a8a1a31979678d67b17\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/e855efbcd7344e9ad8d89a8a1a31979678d67b17\",\"tree\":[{\"path\":\".github\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"0c5ecb959cab3cebda3cf98e4bab0950a6242be3\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/0c5ecb959cab3cebda3cf98e4bab0950a6242be3\"},{\"path\":\".github/frictionless.yaml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"2bf0512efed4418e9264928a2bffa63af6ab1944\",\"size\":455,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/2bf0512efed4418e9264928a2bffa63af6ab1944\"},{\"path\":\".github/workflows\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"842fcd20ad09704fbd17ccaf960e01a8b611cd7a\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/842fcd20ad09704fbd17ccaf960e01a8b611cd7a\"},{\"path\":\".github/workflows/frictionless.yaml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"aac9b28f30362e3e4c6e950a9bdd17e01280cd83\",\"size\":304,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/aac9b28f30362e3e4c6e950a9bdd17e01280cd83\"},{\"path\":\".gitignore\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"7b14861d4f9b884288afd38fe4e7469789e9b34c\",\"size\":21,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/7b14861d4f9b884288afd38fe4e7469789e9b34c\"},{\"path\":\".travis.yml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"67c7e5588ed57eb0014cd8b3e6540000037dc7ba\",\"size\":294,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/67c7e5588ed57eb0014cd8b3e6540000037dc7ba\"},{\"path\":\"README.md\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"9c36869b88dfad9b9d610734966270b7e883b707\",\"size\":1752,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/9c36869b88dfad9b9d610734966270b7e883b707\"},{\"path\":\"archive\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"39b08adf12bb3f3adc170b3a77f338f8e02b63d1\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/39b08adf12bb3f3adc170b3a77f338f8e02b63d1\"},{\"path\":\"archive/table_a1.xml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"42609efee29fb5a7d98935b806ea135e3f455653\",\"size\":47281,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/42609efee29fb5a7d98935b806ea135e3f455653\"},{\"path\":\"archive/table_a2.xml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"b2a569c403e6917aa466d0973d3b3f677821530f\",\"size\":29349,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/b2a569c403e6917aa466d0973d3b3f677821530f\"},{\"path\":\"data\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"0cf8433969ede15de089c1127cb6937d2ae13e3e\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/0cf8433969ede15de089c1127cb6937d2ae13e3e\"},{\"path\":\"data/codes-all.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"2c803df30c1d09e1c7f5de6661ca38f8e627d65b\",\"size\":17472,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/2c803df30c1d09e1c7f5de6661ca38f8e627d65b\"},{\"path\":\"datapackage.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"dfca466367e0b3d23294df83b72101cb5b084c09\",\"size\":2019,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/dfca466367e0b3d23294df83b72101cb5b084c09\"},{\"path\":\"scripts\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"ff0526e9a5e2d76f31d8038cbea148067203776a\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/ff0526e9a5e2d76f31d8038cbea148067203776a\"},{\"path\":\"scripts/runall.sh\",\"mode\":\"100755\",\"type\":\"blob\",\"sha\":\"4632c9933cd6c76331301efa71dc5cd183c6dc89\",\"size\":3098,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/4632c9933cd6c76331301efa71dc5cd183c6dc89\"}],\"truncated\":false}" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "access-control-expose-headers", - "value": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset" - }, - { - "name": "cache-control", - "value": "public, max-age=60, s-maxage=60" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "874" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'" - }, - { - "name": "content-type", - "value": "application/json; charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "etag", - "value": "W/\"4635290695f3b6591e9e38e1e37992e8a758b746f29fb6288f64ac5ea26b5d2d\"" - }, - { - "name": "last-modified", - "value": "Wed, 07 Jul 2021 14:47:03 GMT" - }, - { - "name": "referrer-policy", - "value": "origin-when-cross-origin, strict-origin-when-cross-origin" - }, - { - "name": "server", - "value": "github.com" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000; includeSubdomains; preload" - }, - { - "name": "vary", - "value": "Accept,Accept-Encoding, Accept, X-Requested-With" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-api-version-selected", - "value": "2022-11-28" - }, - { - "name": "x-github-media-type", - "value": "github.v3; format=json" - }, - { - "name": "x-github-request-id", - "value": "2359:39D094:F0E017:E1E96A:68945A86" - }, - { - "name": "x-ratelimit-limit", - "value": "60" - }, - { - "name": "x-ratelimit-remaining", - "value": "56" - }, - { - "name": "x-ratelimit-reset", - "value": "1754556565" - }, - { - "name": "x-ratelimit-resource", - "value": "core" - }, - { - "name": "x-ratelimit-used", - "value": "4" - }, - { - "name": "x-xss-protection", - "value": "0" - } - ], - "headersSize": 1293, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:26.465Z", - "time": 292, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 292 - } - }, - { - "_id": "9d2eca9b34eee81e983e0f36dc5e2abe", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 107, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/datapackage.json" - }, - "response": { - "bodySize": 679, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 679, - "text": "{\n \"name\": \"currency-codes\",\n \"title\": \"ISO 4217 Currency Codes\",\n \"licenses\": [\n {\n \"name\": \"ODC-PDDL-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ],\n \"keywords\": [ \"iso\", \"iso-4217\", \"currency\", \"codes\" ],\n \"homepage\": \"http://www.iso.org/iso/currency_codes\",\n \"sources\": [{\n \"name\": \"SIX Interbank Clearing Ltd (on behalf of ISO)\",\n \"email\": \"office@currency-iso.org\",\n \"title\": \"SIX Interbank Clearing Ltd (on behalf of ISO)\"\n }],\n \"contributors\": [\n {\n \"title\": \"Rufus Pollock\",\n \"email\": \"rufus.pollock@okfn.org\",\n \"role\": \"maintainer\"\n },\n {\n \"title\": \"Kristofer D. Kusano\",\n \"email\": \"kdkusano@gmail.com\"\n }\n ],\n \"resources\": [\n {\n \"path\": \"data/codes-all.csv\",\n \"name\": \"codes-all\",\n \"mimetype\": \"text/csv\",\n \"size\": \"16863\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"Entity\",\n \"type\": \"string\",\n \"description\": \"Country or region name\"\n },\n {\n \"name\": \"Currency\",\n \"type\": \"string\",\n \"description\": \"Name of the currency\"\n },\n {\n \"name\": \"AlphabeticCode\",\n \"title\": \"Alphabetic Code\",\n \"type\": \"string\",\n \"description\": \"3 digit alphabetic code for the currency\"\n },\n {\n \"name\": \"NumericCode\",\n \"title\": \"Numeric Code\",\n \"type\": \"number\",\n \"description\": \"3 digit numeric code\"\n },\n {\n \"name\": \"MinorUnit\",\n \"title\": \"Minor Unit\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"WithdrawalDate\",\n \"title\": \"Withdrawal Date\",\n \"type\": \"string\",\n \"description\": \"Date currency withdrawn (values can be ranges or months\"\n }\n ]\n }\n }\n ]\n}\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "679" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:27 GMT" - }, - { - "name": "etag", - "value": "W/\"18535666ca34d8d5f92e12776d3673e8e31c05c2be5c50a439cee1f3213ab9cb\"" - }, - { - "name": "expires", - "value": "Thu, 07 Aug 2025 07:54:27 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "485833d62b1c0cbf6673f946e8b2bed21876f2c6" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "0EAB:27570B:15BE0F9:191ED61:68945A86" - }, - { - "name": "x-served-by", - "value": "cache-lis1490057-LIS" - }, - { - "name": "x-timer", - "value": "S1754552967.823769,VS0,VE177" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 904, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:26.760Z", - "time": 346, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 346 - } - }, - { - "_id": "2fc9ef7063eaa13e9b46bdfb36e801d0", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 108, - "httpVersion": "HTTP/1.1", - "method": "HEAD", - "queryString": [], - "url": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/datapackage.json" - }, - "response": { - "bodySize": 679, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 679 - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "close" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "679" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Fri, 31 Oct 2025 07:31:18 GMT" - }, - { - "name": "etag", - "value": "W/\"18535666ca34d8d5f92e12776d3673e8e31c05c2be5c50a439cee1f3213ab9cb\"" - }, - { - "name": "expires", - "value": "Fri, 31 Oct 2025 07:36:18 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "e044ba96c2be9646f2b98f8df059609ba43674fd" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "44CE:1FAB18:CCC9A7:DF847E:690465C5" - }, - { - "name": "x-served-by", - "value": "cache-lis1490043-LIS" - }, - { - "name": "x-timer", - "value": "S1761895878.877871,VS0,VE185" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 897, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-31T07:31:17.615Z", - "time": 581, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 581 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/dataset/plugins/github/package/index.ts b/dataset/plugins/github/package/index.ts deleted file mode 100644 index ba2309ad..00000000 --- a/dataset/plugins/github/package/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type { GithubPackage } from "./Package.ts" -export type { GithubOwner } from "./Owner.ts" -export type { GithubLicense } from "./License.ts" -export { loadPackageFromGithub } from "./load.ts" -export { savePackageToGithub } from "./save.ts" -export { convertPackageFromGithub } from "./convert/fromGithub.ts" diff --git a/dataset/plugins/github/package/load.spec.ts b/dataset/plugins/github/package/load.spec.ts deleted file mode 100644 index 93585c2e..00000000 --- a/dataset/plugins/github/package/load.spec.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromGithub } from "./load.ts" - -useRecording() - -describe("loadPackageFromGithub", () => { - it("should load a package", async () => { - const datapackage = await loadPackageFromGithub( - "https://github.com/roll/data", - ) - - expect(datapackage).toMatchSnapshot() - }) - - it("should merge datapackage.json if present", async () => { - const datapackage = await loadPackageFromGithub( - "https://github.com/roll/currency-codes", - ) - - expect(datapackage).toMatchSnapshot() - }) -}) diff --git a/dataset/plugins/github/package/load.ts b/dataset/plugins/github/package/load.ts deleted file mode 100644 index 7a8533d7..00000000 --- a/dataset/plugins/github/package/load.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { mergePackages } from "../../../package/index.ts" -import { makeGithubApiRequest } from "../github/index.ts" -import type { GithubResource } from "../resource/index.ts" -import type { GithubPackage } from "./Package.ts" -import { convertPackageFromGithub } from "./convert/fromGithub.ts" - -/** - * Load a package from a Github repository - * @param props Object containing the URL to the Github repository - * @returns Package object - */ -export async function loadPackageFromGithub( - repoUrl: string, - options?: { - apiKey?: string - }, -) { - const { apiKey } = options ?? {} - - // Extract owner and repo from URL - const { owner, repo } = extractRepositoryInfo(repoUrl) - if (!owner || !repo) { - throw new Error(`Failed to extract repository info from URL: ${repoUrl}`) - } - - const githubPackage = await makeGithubApiRequest({ - endpoint: `/repos/${owner}/${repo}`, - apiKey, - }) - - const ref = githubPackage.default_branch - githubPackage.resources = ( - await makeGithubApiRequest<{ tree: GithubResource[] }>({ - endpoint: `/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`, - apiKey, - }) - ).tree - - const systemPackage = convertPackageFromGithub(githubPackage) - const userPackagePath = systemPackage.resources - .filter(resource => resource["github:key"] === "datapackage.json") - .map(resource => resource["github:url"]) - .at(0) - - const datapackage = await mergePackages({ systemPackage, userPackagePath }) - datapackage.resources = datapackage.resources.map(resource => { - // TODO: remove these keys completely - return { ...resource, "github:key": undefined, "github:url": undefined } - }) - - return datapackage -} - -/** - * Extract repository owner and name from URL - * - * Examples: - * - https://github.com/owner/repo - */ -function extractRepositoryInfo(repoUrl: string) { - const url = new URL(repoUrl) - const [owner, repo] = url.pathname.split("/").filter(Boolean) - return { owner, repo } -} diff --git a/dataset/plugins/github/package/save.spec.ts b/dataset/plugins/github/package/save.spec.ts deleted file mode 100644 index 72de7874..00000000 --- a/dataset/plugins/github/package/save.spec.ts +++ /dev/null @@ -1,654 +0,0 @@ -import { relative } from "node:path" -import type { Package } from "@dpkit/metadata" -import { loadPackageDescriptor } from "@dpkit/metadata" -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { savePackageToGithub } from "./save.ts" - -describe("savePackageToGithub", () => { - const getFixturePath = (name: string) => - relative(process.cwd(), `${import.meta.dirname}/fixtures/${name}`) - - const mockPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - path: getFixturePath("data.csv"), - format: "csv", - bytes: 100, - }, - ], - } - - const mockOptions = { - apiKey: "test-api-key", - repo: "test-repo", - } - - const originalFetch = globalThis.fetch - let fetchMock: ReturnType - - beforeEach(() => { - fetchMock = vi.fn() - // @ts-ignore - globalThis.fetch = fetchMock - }) - - afterEach(() => { - globalThis.fetch = originalFetch - vi.resetAllMocks() - }) - - it.skip("should save a package", async () => { - const dataPackage = await loadPackageDescriptor( - "core/package/fixtures/package.json", - ) - - const result = await savePackageToGithub(dataPackage, { - apiKey: "", - repo: "test", - }) - - console.log(result) - - expect(true).toBeDefined() - }) - - it("creates a repository in GitHub with correct API calls", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "data.csv", - path: "data.csv", - sha: "abc123", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "datapackage.json", - path: "datapackage.json", - sha: "def456", - }, - }), - }) - - const result = await savePackageToGithub(mockPackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(3) - - const repoCreateCall = fetchMock.mock.calls[0] - expect(repoCreateCall).toBeDefined() - if (!repoCreateCall) return - - expect(repoCreateCall[0]).toEqual("https://api.github.com/user/repos") - expect(repoCreateCall[1]).toMatchObject({ - method: "POST", - headers: { - Authorization: "Bearer test-api-key", - "Content-Type": "application/json", - }, - }) - - const repoPayload = JSON.parse(repoCreateCall[1].body) - expect(repoPayload.name).toEqual("test-repo") - expect(repoPayload.auto_init).toEqual(true) - - expect(result).toEqual({ - path: "https://raw.githubusercontent.com/test-user/test-repo/refs/heads/main/dataPackage.json", - repoUrl: "https://github.com/test-user/test-repo", - }) - }) - - it("creates a repository in an organization when org is specified", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-org/test-repo", - owner: { - login: "test-org", - id: 2, - avatar_url: "https://avatars.githubusercontent.com/u/2", - html_url: "https://github.com/test-org", - type: "Organization", - }, - html_url: "https://github.com/test-org/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-org/test-repo.git", - ssh_url: "git@github.com:test-org/test-repo.git", - clone_url: "https://github.com/test-org/test-repo.git", - }), - }) - - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "file", - path: "file", - sha: "abc123", - }, - }), - }) - - await savePackageToGithub(mockPackage, { - ...mockOptions, - org: "test-org", - }) - - const repoCreateCall = fetchMock.mock.calls[0] - expect(repoCreateCall).toBeDefined() - if (!repoCreateCall) return - - expect(repoCreateCall[0]).toEqual( - "https://api.github.com/orgs/test-org/repos", - ) - }) - - it("uploads resource files with base64 encoding", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "data.csv", - path: "data.csv", - sha: "abc123", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "datapackage.json", - path: "datapackage.json", - sha: "def456", - }, - }), - }) - - await savePackageToGithub(mockPackage, mockOptions) - - const fileUploadCall = fetchMock.mock.calls[1] - expect(fileUploadCall).toBeDefined() - if (!fileUploadCall) return - - expect(fileUploadCall[0]).toEqual( - "https://api.github.com/repos/test-user/test-repo/contents/data.csv", - ) - expect(fileUploadCall[1]).toMatchObject({ - method: "PUT", - headers: { - Authorization: "Bearer test-api-key", - "Content-Type": "application/json", - }, - }) - - const filePayload = JSON.parse(fileUploadCall[1].body) - expect(filePayload.path).toEqual("data.csv") - expect(filePayload.message).toEqual('Added file "data.csv"') - expect(filePayload.content).toBeDefined() - expect(typeof filePayload.content).toEqual("string") - }) - - it("uploads datapackage.json metadata file", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "data.csv", - path: "data.csv", - sha: "abc123", - }, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "datapackage.json", - path: "datapackage.json", - sha: "def456", - }, - }), - }) - - await savePackageToGithub(mockPackage, mockOptions) - - const datapackageUploadCall = fetchMock.mock.calls[2] - expect(datapackageUploadCall).toBeDefined() - if (!datapackageUploadCall) return - - expect(datapackageUploadCall[0]).toEqual( - "https://api.github.com/repos/test-user/test-repo/contents/datapackage.json", - ) - - const datapackagePayload = JSON.parse(datapackageUploadCall[1].body) - expect(datapackagePayload.path).toEqual("datapackage.json") - expect(datapackagePayload.message).toEqual('Added file "datapackage.json"') - expect(datapackagePayload.content).toBeDefined() - expect(typeof datapackagePayload.content).toEqual("string") - }) - - it("passes API key as Bearer token in Authorization header", async () => { - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - await savePackageToGithub(mockPackage, { - ...mockOptions, - apiKey: "custom-api-key", - }) - - const firstCall = fetchMock.mock.calls[0] - expect(firstCall).toBeDefined() - if (!firstCall) return - - const headers = firstCall[1].headers - expect(headers.Authorization).toEqual("Bearer custom-api-key") - }) - - it("handles API errors from repository creation", async () => { - fetchMock.mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: "Bad Request", - text: () => Promise.resolve("Repository name already exists"), - }) - - await expect(savePackageToGithub(mockPackage, mockOptions)).rejects.toThrow( - "Github API error: 400 Bad Request", - ) - }) - - it("handles API errors from file upload", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: false, - status: 500, - statusText: "Internal Server Error", - text: () => Promise.resolve("Failed to upload file"), - }) - - await expect(savePackageToGithub(mockPackage, mockOptions)).rejects.toThrow( - "Github API error: 500 Internal Server Error", - ) - }) - - it("handles packages with multiple resources", async () => { - const multiResourcePackage: Package = { - ...mockPackage, - resources: [ - { - name: "resource-1", - path: getFixturePath("data.csv"), - format: "csv", - }, - { - name: "resource-2", - path: getFixturePath("data.csv"), - format: "json", - }, - ], - } - - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - await savePackageToGithub(multiResourcePackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(4) - - const secondFileUploadCall = fetchMock.mock.calls[2] - expect(secondFileUploadCall).toBeDefined() - if (!secondFileUploadCall) return - - expect(secondFileUploadCall[0]).toContain("/contents/") - }) - - it("handles packages with no resources", async () => { - const emptyPackage: Package = { - ...mockPackage, - resources: [], - } - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "datapackage.json", - path: "datapackage.json", - sha: "def456", - }, - }), - }) - - const result = await savePackageToGithub(emptyPackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(result.repoUrl).toEqual("https://github.com/test-user/test-repo") - }) - - it("skips resources without path", async () => { - const packageWithoutPath: Package = { - ...mockPackage, - resources: [ - { - name: "resource-without-path", - format: "csv", - }, - ], - } - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - name: "test-repo", - full_name: "test-user/test-repo", - owner: { - login: "test-user", - id: 1, - avatar_url: "https://avatars.githubusercontent.com/u/1", - html_url: "https://github.com/test-user", - type: "User", - }, - html_url: "https://github.com/test-user/test-repo", - description: null, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - homepage: null, - size: 0, - stargazers_count: 0, - watchers_count: 0, - language: null, - license: null, - default_branch: "main", - topics: [], - private: false, - archived: false, - git_url: "git://github.com/test-user/test-repo.git", - ssh_url: "git@github.com:test-user/test-repo.git", - clone_url: "https://github.com/test-user/test-repo.git", - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - content: { - name: "datapackage.json", - path: "datapackage.json", - sha: "def456", - }, - }), - }) - - await savePackageToGithub(packageWithoutPath, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(2) - }) -}) diff --git a/dataset/plugins/github/package/save.ts b/dataset/plugins/github/package/save.ts deleted file mode 100644 index b2c28dbe..00000000 --- a/dataset/plugins/github/package/save.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Buffer } from "node:buffer" -import { buffer } from "node:stream/consumers" -import type { Descriptor, Package } from "@dpkit/metadata" -import { stringifyDescriptor } from "@dpkit/metadata" -import { convertPackageToDescriptor } from "@dpkit/metadata" -import { getPackageBasepath } from "../../../package/index.ts" -import { saveResourceFiles } from "../../../resource/index.ts" -import { loadFileStream } from "../../../stream/index.ts" -import { makeGithubApiRequest } from "../github/index.ts" -import type { GithubPackage } from "./Package.ts" - -/** - * Save a package to a Github repository - * @param options Object containing the package to save and Github details - * @returns Object with the repository URL - */ -export async function savePackageToGithub( - dataPackage: Package, - options: { - apiKey: string - repo: string - org?: string - }, -) { - const { apiKey, org, repo } = options - const basepath = getPackageBasepath(dataPackage) - - const githubPackage = await makeGithubApiRequest({ - endpoint: org ? `/orgs/${org}/repos` : "/user/repos", - payload: { name: repo, auto_init: true }, - method: "POST", - apiKey, - }) - - const resourceDescriptors: Descriptor[] = [] - for (const resource of dataPackage.resources) { - if (!resource.path) continue - - resourceDescriptors.push( - await saveResourceFiles(resource, { - basepath, - withRemote: false, - saveFile: async options => { - const stream = await loadFileStream(options.normalizedPath) - - const payload = { - path: options.denormalizedPath, - content: Buffer.from(await buffer(stream)).toString("base64"), - message: `Added file "${options.denormalizedPath}"`, - } - - await makeGithubApiRequest({ - endpoint: `/repos/${githubPackage.owner.login}/${repo}/contents/${options.denormalizedPath}`, - method: "PUT", - payload, - apiKey, - }) - - return options.denormalizedPath - }, - }), - ) - } - - const descriptor = { - ...convertPackageToDescriptor(dataPackage, { basepath }), - resources: resourceDescriptors, - } - - for (const denormalizedPath of ["datapackage.json"]) { - const payload = { - path: denormalizedPath, - message: `Added file "${denormalizedPath}"`, - content: Buffer.from(stringifyDescriptor(descriptor)).toString("base64"), - } - - await makeGithubApiRequest({ - endpoint: `/repos/${githubPackage.owner.login}/${repo}/contents/${denormalizedPath}`, - method: "PUT", - payload, - apiKey, - }) - } - - return { - path: `https://raw.githubusercontent.com/${githubPackage.owner.login}/${repo}/refs/heads/main/dataPackage.json`, - repoUrl: githubPackage.html_url, - } -} diff --git a/dataset/plugins/github/plugin.spec.ts b/dataset/plugins/github/plugin.spec.ts deleted file mode 100644 index c2a9aac5..00000000 --- a/dataset/plugins/github/plugin.spec.ts +++ /dev/null @@ -1,107 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/load.ts" -import { GithubPlugin } from "./plugin.ts" - -vi.mock("./package/load.ts", () => ({ - loadPackageFromGithub: vi.fn(), -})) - -describe("GithubPlugin", () => { - let plugin: GithubPlugin - let mockLoadPackageFromGithub: ReturnType - - beforeEach(() => { - plugin = new GithubPlugin() - mockLoadPackageFromGithub = vi.mocked(packageModule.loadPackageFromGithub) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from github.com url", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromGithub.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://github.com/owner/repo/data", - ) - - expect(mockLoadPackageFromGithub).toHaveBeenCalledWith( - "https://github.com/owner/repo/data", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for non-github urls", async () => { - const result = await plugin.loadPackage("https://example.com/data") - - expect(mockLoadPackageFromGithub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local paths", async () => { - const result = await plugin.loadPackage("./data") - - expect(mockLoadPackageFromGithub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for zenodo urls", async () => { - const result = await plugin.loadPackage("https://zenodo.org/record/123") - - expect(mockLoadPackageFromGithub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle github urls with paths", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromGithub.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://github.com/owner/repo/tree/main/data", - ) - - expect(mockLoadPackageFromGithub).toHaveBeenCalledWith( - "https://github.com/owner/repo/tree/main/data", - ) - expect(result).toEqual(mockPackage) - }) - - it("should handle github urls with query parameters", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromGithub.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://github.com/owner/repo?tab=readme", - ) - - expect(mockLoadPackageFromGithub).toHaveBeenCalledWith( - "https://github.com/owner/repo?tab=readme", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for http non-github urls", async () => { - const result = await plugin.loadPackage("http://example.com/data") - - expect(mockLoadPackageFromGithub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for gitlab urls", async () => { - const result = await plugin.loadPackage("https://gitlab.com/owner/repo") - - expect(mockLoadPackageFromGithub).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/github/plugin.ts b/dataset/plugins/github/plugin.ts deleted file mode 100644 index 256d97ec..00000000 --- a/dataset/plugins/github/plugin.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { isRemotePath } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" -import { loadPackageFromGithub } from "./package/load.ts" - -export class GithubPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isGithub = getIsGithub(source) - if (!isGithub) return undefined - - const dataPackage = await loadPackageFromGithub(source) - return dataPackage - } -} - -function getIsGithub(path: string) { - const isRemote = isRemotePath(path) - if (!isRemote) return false - - return new URL(path).hostname === "github.com" -} diff --git a/dataset/plugins/github/resource/Resource.ts b/dataset/plugins/github/resource/Resource.ts deleted file mode 100644 index 7498e17d..00000000 --- a/dataset/plugins/github/resource/Resource.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * GitHub repository file content - */ -export interface GithubResource { - /** - * File path within repository - */ - path: string - - /** - * File mode e.g. `100755` - */ - mode: string - - /** - * File type e.g. `blob` - */ - type: string - - /** - * File size in bytes - */ - size: number - - /** - * File SHA-1 - */ - sha: string - - /** - * File url on GitHub API - */ - url: string -} diff --git a/dataset/plugins/github/resource/convert/fromGithub.ts b/dataset/plugins/github/resource/convert/fromGithub.ts deleted file mode 100644 index 2b68625f..00000000 --- a/dataset/plugins/github/resource/convert/fromGithub.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { getFilename, getFormat, getName } from "@dpkit/metadata" -import type { GithubResource } from "../Resource.ts" - -export function convertResourceFromGithub( - githubResource: GithubResource, - options: { - defaultBranch: string - }, -) { - const path = convertPath({ - ...githubResource, - ref: options.defaultBranch, - }) - - const filename = getFilename(path) - const resource: Resource = { - path, - name: getName(filename) ?? githubResource.sha, - bytes: githubResource.size, - hash: `sha1:${githubResource.sha}`, - format: getFormat(filename), - "github:key": githubResource.path, - "github:url": path, - } - - return resource -} - -function convertPath(options: { - url: string - ref: string - path: string -}) { - const url = new URL(options.url) - const [owner, repo] = url.pathname.split("/").slice(2) - return `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${options.ref}/${options.path}` -} diff --git a/dataset/plugins/github/resource/convert/toGithub.ts b/dataset/plugins/github/resource/convert/toGithub.ts deleted file mode 100644 index c03fb8c0..00000000 --- a/dataset/plugins/github/resource/convert/toGithub.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import type { GithubResource } from "../Resource.ts" - -export function convertResourceToGithub( - resource: Resource, -): Partial { - if (!resource.path && !resource.name) { - return {} - } - - const githubResource: Partial = { - path: resource.name, - } - - return githubResource -} diff --git a/dataset/plugins/github/resource/index.ts b/dataset/plugins/github/resource/index.ts deleted file mode 100644 index c46bca58..00000000 --- a/dataset/plugins/github/resource/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type { GithubResource } from "./Resource.ts" -export { convertResourceFromGithub } from "./convert/fromGithub.ts" -export { convertResourceToGithub } from "./convert/toGithub.ts" diff --git a/dataset/plugins/zenodo/index.ts b/dataset/plugins/zenodo/index.ts deleted file mode 100644 index ae6326f6..00000000 --- a/dataset/plugins/zenodo/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./package/index.ts" -export * from "./resource/index.ts" -export * from "./plugin.ts" diff --git a/dataset/plugins/zenodo/package/Creator.ts b/dataset/plugins/zenodo/package/Creator.ts deleted file mode 100644 index 3da5a58b..00000000 --- a/dataset/plugins/zenodo/package/Creator.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Zenodo Creator interface - */ -export interface ZenodoCreator { - /** - * Creator name (format: Family name, Given names) - */ - name: string - - /** - * Creator affiliation - */ - affiliation?: string - - /** - * Creator identifiers (e.g., ORCID) - */ - identifiers?: Array<{ - identifier: string - scheme: string - }> -} diff --git a/dataset/plugins/zenodo/package/Package.ts b/dataset/plugins/zenodo/package/Package.ts deleted file mode 100644 index 955dd23b..00000000 --- a/dataset/plugins/zenodo/package/Package.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { ZenodoResource } from "../resource/index.ts" -import type { ZenodoCreator } from "./Creator.ts" - -/** - * Zenodo Deposit interface - */ -export interface ZenodoPackage { - /** - * Deposit identifier - */ - id: number - - /** - * Deposit URL - */ - links: { - self: string - html: string - files: string - bucket: string - publish?: string - discard?: string - edit?: string - } - - /** - * Deposit metadata - */ - metadata: { - /** - * Title of the deposit - */ - title: string - - /** - * Description of the deposit - */ - description: string - - /** - * Upload type, e.g., "dataset" - */ - upload_type: string - - /** - * Publication date in ISO format (YYYY-MM-DD) - */ - publication_date?: string - - /** - * Creators of the deposit - */ - creators: ZenodoCreator[] - - /** - * Access right, e.g., "open", "embargoed", "restricted", "closed" - */ - access_right?: string - - /** - * License identifier - */ - license?: string - - /** - * DOI of the deposit - */ - doi?: string - - /** - * Keywords/tags - */ - keywords?: string[] - - /** - * Related identifiers (e.g., DOIs of related works) - */ - related_identifiers?: Array<{ - identifier: string - relation: string - scheme: string - }> - - /** - * Communities the deposit belongs to - */ - communities?: Array<{ - identifier: string - }> - - /** - * Version of the deposit - */ - version?: string - } - - /** - * Files associated with the deposit - */ - files: ZenodoResource[] - - /** - * State of the deposit - */ - state: "unsubmitted" | "inprogress" | "done" - - /** - * Submitted flag - */ - submitted: boolean -} diff --git a/dataset/plugins/zenodo/package/convert/fromZenodo.ts b/dataset/plugins/zenodo/package/convert/fromZenodo.ts deleted file mode 100644 index 68549359..00000000 --- a/dataset/plugins/zenodo/package/convert/fromZenodo.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { Contributor, License, Package } from "@dpkit/metadata" -import { convertResourceFromZenodo } from "../../resource/index.ts" -import type { ZenodoPackage } from "../Package.ts" - -export function convertPackageFromZenodo( - zenodoPackage: ZenodoPackage, -): Package { - const datapackage: Package = { - name: `record-${zenodoPackage.id}`, - resources: [], - } - - const metadata = zenodoPackage.metadata - - datapackage.title = metadata.title - datapackage.description = metadata.description - - if (metadata.version) { - datapackage.version = metadata.version - } - - if (zenodoPackage.files && zenodoPackage.files.length > 0) { - datapackage.resources = zenodoPackage.files.map(zenodoResource => - convertResourceFromZenodo(zenodoResource), - ) - } - - if (metadata.license) { - const license: License = { - name: metadata.license, - } - datapackage.licenses = [license] - } - - if (metadata.creators && metadata.creators.length > 0) { - const contributors: Contributor[] = metadata.creators.map(creator => { - const contributor: Contributor = { - title: creator.name, - role: "author", - } - - if (creator.affiliation) { - contributor.path = creator.affiliation - } - - return contributor - }) - - datapackage.contributors = contributors - } - - if (metadata.keywords && metadata.keywords.length > 0) { - datapackage.keywords = metadata.keywords - } - - if (metadata.publication_date) { - datapackage.created = metadata.publication_date - } - - return datapackage -} diff --git a/dataset/plugins/zenodo/package/convert/toZenodo.ts b/dataset/plugins/zenodo/package/convert/toZenodo.ts deleted file mode 100644 index 805d0b74..00000000 --- a/dataset/plugins/zenodo/package/convert/toZenodo.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import type { ZenodoCreator } from "../Creator.ts" -import type { ZenodoPackage } from "../Package.ts" - -export function convertPackageToZenodo( - dataPackage: Package, -): Partial { - const metadata: Partial = { - upload_type: "dataset", - } - - if (dataPackage.title) { - metadata.title = dataPackage.title - } - - if (dataPackage.description) { - metadata.description = dataPackage.description - } else if (dataPackage.title) { - metadata.description = dataPackage.title - } else { - metadata.description = "Dataset created with @dpkit/zenodo" - } - - if (dataPackage.version) { - metadata.version = dataPackage.version - } - - if (dataPackage.licenses && dataPackage.licenses.length > 0) { - const license = dataPackage.licenses[0] - if (license?.name) { - metadata.license = license.name - } - } - - if (dataPackage.contributors && dataPackage.contributors.length > 0) { - const creators: ZenodoCreator[] = [] - - const authors = dataPackage.contributors.filter(c => c.role === "author") - if (authors.length > 0) { - authors.forEach(author => { - const creator: ZenodoCreator = { - name: author.title, - } - - if (author.path) { - creator.affiliation = author.path - } - - creators.push(creator) - }) - } else { - const firstContributor = dataPackage.contributors[0] - if (firstContributor) { - const creator: ZenodoCreator = { - name: firstContributor.title, - } - - if (firstContributor.path) { - creator.affiliation = firstContributor.path - } - - creators.push(creator) - } - } - - if (creators.length > 0) { - metadata.creators = creators - } else { - metadata.creators = [ - { - name: "Unknown Author", - affiliation: "Unknown Affiliation", - }, - ] - } - } else { - metadata.creators = [ - { - name: "Unknown Author", - affiliation: "Unknown Affiliation", - }, - ] - } - - if (dataPackage.keywords && dataPackage.keywords.length > 0) { - metadata.keywords = dataPackage.keywords - } - - return { - metadata: metadata as ZenodoPackage["metadata"], - } -} diff --git a/dataset/plugins/zenodo/package/fixtures/data.csv b/dataset/plugins/zenodo/package/fixtures/data.csv deleted file mode 100644 index d888cb89..00000000 --- a/dataset/plugins/zenodo/package/fixtures/data.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name,value -1,test,100 -2,example,200 diff --git a/dataset/plugins/zenodo/package/fixtures/generated/load.spec.ts.snap b/dataset/plugins/zenodo/package/fixtures/generated/load.spec.ts.snap deleted file mode 100644 index 6ca24442..00000000 --- a/dataset/plugins/zenodo/package/fixtures/generated/load.spec.ts.snap +++ /dev/null @@ -1,728 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`loadPackageFromZenodo > should load a package 1`] = ` -{ - "contributors": [ - { - "role": "author", - "title": "Sukhija, Vinay Jagdish", - }, - ], - "created": "2025-05-27", - "description": "

A brief project to extract PM 10 concentration from Japan. Data set has been created using the OpenAQ data through a python script

", - "keywords": [ - "PM10 concentrations", - "Air Quality", - "Japan", - "OpenAQ", - ], - "licenses": [ - { - "name": { - "id": "cc-by-4.0", - }, - }, - ], - "name": "record-15525711", - "resources": [ - { - "bytes": 272, - "format": "csv", - "hash": "md5:5ac1b92a57ec809c546ea90334c845c9", - "name": "openaq-measurements-p-m10-japan-20250527-095520", - "path": "https://zenodo.org/records/15525711/files/openaq_measurements_PM10_Japan_20250527_095520.csv", - "zenodo:key": undefined, - "zenodo:url": undefined, - }, - ], - "title": "PM10 Concentration in Japan", -} -`; - -exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ` -{ - "$schema": undefined, - "contributors": [ - { - "path": "Vogelwerkgroep Assen", - "role": "author", - "title": "Dijkstra, Bert", - }, - { - "path": "Vogelwerkgroep Assen", - "role": "author", - "title": "Dillerop, Rinus", - }, - { - "path": "Dutch Centre for Field Ornithology", - "role": "author", - "title": "Oosterbeek, Kees", - }, - { - "path": "University of Amsterdam", - "role": "author", - "title": "Bouten, Willem", - }, - { - "path": "Research Institute for Nature and Forest", - "role": "author", - "title": "Desmet, Peter", - }, - { - "path": "Dutch Bryological and Lichenological Society", - "role": "author", - "title": "van der Kolk, Henk-Jan", - }, - { - "path": "Dutch Centre for Field Ornithology", - "role": "author", - "title": "Ens, Bruno J.", - }, - ], - "created": "2023-10-30", - "description": "

O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands) is a bird tracking dataset published by the Vogelwerkgroep Assen, Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University, the University of Amsterdam and the Research Institute for Nature and Forest (INBO). It contains animal tracking data collected for the study O_ASSEN using trackers developed by the University of Amsterdam Bird Tracking System (UvA-BiTS, http://www.uva-bits.nl). The study was operational from 2018 to 2019. In total 6 individuals of Eurasian oystercatchers (Haematopus ostralegus) have been tagged as a breeding bird in the city of Assen (the Netherlands), mainly to study space use of oystercatchers breeding in urban areas. Data are uploaded from the UvA-BiTS database to Movebank and from there archived on Zenodo (see https://github.com/inbo/bird-tracking). No new data are expected.

-

See van der Kolk et al. (2022, https://doi.org/10.3897/zookeys.1123.90623) for a more detailed description of this dataset.

-

Files

-

Data in this package are exported from Movebank study 1605797471. Fields in the data follow the Movebank Attribute Dictionary and are described in datapackage.json. Files are structured as a Frictionless Data Package. You can access all data in R via https://zenodo.org/records/10053903/files/datapackage.json using frictionless.

-
    -
  • datapackage.json: technical description of the data files.
    • -
  • O_ASSEN-reference-data.csv: reference data about the animals, tags and deployments.
    • -
  • O_ASSEN-gps-yyyy.csv.gz: GPS data recorded by the tags, grouped by year.
    • -
  • O_ASSEN-acceleration-yyyy.csv.gz: acceleration data recorded by the tags, grouped by year.
    • -
-

Acknowledgements

-

These data were collected by Bert Dijkstra and Rinus Dillerop from Vogelwerkgroep Assen, in collaboration with the Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University and the University of Amsterdam (UvA). Funding was provided by the Prins Bernard Cultuurfonds Drenthe, municipality of Assen, IJsvogelfonds (from Birdlife Netherlands and Nationale Postcodeloterij) and the Waterleiding Maatschappij Drenthe. The dataset was published with funding from Stichting NLBIF - Netherlands Biodiversity Information Facility.

", - "id": "https://doi.org/10.5281/zenodo.10053903", - "keywords": [ - "animal movement", - "animal tracking", - "gps tracking", - "accelerometer", - "altitude", - "temperature", - "biologging", - "birds", - "UvA-BiTS", - "Movebank", - "frictionlessdata", - ], - "licenses": [ - { - "name": { - "id": "cc-zero", - }, - }, - ], - "name": "record-10053903", - "profile": "tabular-data-package", - "resources": [ - { - "$schema": undefined, - "encoding": "UTF-8", - "format": "csv", - "mediatype": "text/csv", - "name": "reference-data", - "path": "https://zenodo.org/records/10053903/files/O_ASSEN-reference-data.csv", - "profile": "tabular-data-resource", - "schema": { - "$schema": undefined, - "fields": [ - { - "description": "A unique identifier for the tag, provided by the data owner. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", - "format": "default", - "name": "tag-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000181/2/", - "title": "tag ID", - "type": "string", - }, - { - "description": "An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: 'TUSC_CV5'; Units: none; Entity described: individual", - "format": "default", - "name": "animal-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000016/3/", - "title": "animal ID", - "type": "string", - }, - { - "description": "The scientific name of the taxon on which the tag was deployed, as defined by the Integrated Taxonomic Information System www.itis.gov. If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. The values 'test' and 'calibration' identify events relevant to animal tracking studies that should not be associated with a taxon. Format: controlled list; Entity described: individual", - "format": "default", - "name": "animal-taxon", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000024/4/", - "title": "animal taxon", - "type": "string", - }, - { - "description": "The timestamp when the tag deployment started. Data records recorded before this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Example: '2008-08-30 18:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment", - "format": "%Y-%m-%d %H:%M:%S.%f", - "name": "deploy-on-date", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000081/3/", - "title": "deploy on timestamp", - "type": "datetime", - }, - { - "description": "The timestamp when the tag deployment ended. Data records recorded after this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Further information can be provided in 'deployment end type' and 'deployment end comments'. Example: '2009-10-01 12:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment", - "format": "%Y-%m-%d %H:%M:%S.%f", - "name": "deploy-off-date", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000077/4/", - "title": "deploy off timestamp", - "type": "datetime", - }, - { - "description": "A name or unique identifier for a project associated with the deployment, for example a monitoring program or another data platform. Best practice is to include the name of the related database or organization followed by the project identifier. Example: 'MOTUS145'; Units: none; Entity described: deployment", - "format": "default", - "name": "alt-project-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000244/2/", - "title": "alt project ID", - "type": "string", - }, - { - "description": "Additional information about the animal. Example: 'first to fledge from nest'; Units: none; Entity described: individual", - "format": "default", - "name": "animal-comments", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000012/3/", - "title": "animal comments", - "type": "string", - }, - { - "description": "The age class or life stage of the animal at the beginning of the deployment. Can be years or months of age or terms such as 'adult', 'subadult' and 'juvenile'. Best practice is to define units in the values if needed (e.g. '2 years'). Example: 'juvenile, adult'; Units: none; Entity described: deployment", - "format": "default", - "name": "animal-life-stage", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000018/3/", - "title": "animal life stage", - "type": "string", - }, - { - "description": "The mass of the animal, typically at the beginning of the deployment. Example: '500'; Units: grams; Entity described: deployment", - "format": "default", - "name": "animal-mass", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000019/2/", - "title": "animal mass", - "type": "number", - }, - { - "description": "An alternate identifier for the animal. Used as the display name for animals shown in the Animal Tracker App. Example: 'Ali'; Units: none; Entity described: individual", - "format": "default", - "name": "animal-nickname", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000020/2/", - "title": "animal nickname", - "type": "string", - }, - { - "description": "A number or color scheme for a band or ring attached to the animal. Color bands and other markings can be stored in 'animal marker ID'. Example: '26225'; Units: none; Entity described: individual", - "format": "default", - "name": "animal-ring-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000022/3/", - "title": "animal ring ID", - "type": "string", - }, - { - "description": "The sex of the animal. Allowed values are m = male; f = female; u = unknown. Format: controlled list; Entity described: individual", - "format": "default", - "name": "animal-sex", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000023/3/", - "title": "animal sex", - "type": "string", - }, - { - "description": "The way a tag is attached to an animal. Details can be provided in 'attachment comments'. Values are chosen from a controlled list: backpack-harness = The tag is attached to the animal using a backpack-style harness; collar = The tag is attached by a collar around the animal's neck; ear-tag = The tag is attached to the animal's ear; fin mount = The tag is attached to the animal's fin; glue = The tag is attached to the animal using glue; harness = The tag is attached to the animal using a harness; implant = The tag is placed under the skin of the animal; leg-band = The tag is attached as a leg band or ring; leg-loop-harness = The tag is attached to the animal using a leg-loop-style harness; none = No tag was attached, e.g., for observations using natural markings; other = The tag is attached using another method; subcutaneous-anchor = The tag is attached using one or more anchors attached underneath the animal's skin; suction-cup = The tag is attached using one or more suction cups; sutures = The tag is attached by one or more sutures; tape = The tag is attached to the animal using tape. Format: controlled list; Entity described: deployment", - "format": "default", - "name": "attachment-type", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000052/5/", - "title": "attachment type", - "type": "string", - }, - { - "description": "The geographic latitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '27.3516'; Units: decimal degrees, WGS84 reference system; Entity described: deployment", - "format": "default", - "name": "deploy-on-latitude", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000078/3/", - "title": "deploy on latitude", - "type": "number", - }, - { - "description": "The geographic longitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '-97.3321'; Units: decimal degrees, WGS84 reference system; Entity described: deployment", - "format": "default", - "name": "deploy-on-longitude", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000079/3/", - "title": "deploy on longitude", - "type": "number", - }, - { - "description": "A list of additional measurements taken during capture of the animal at the start of the deployment. Recommended best practice is to define units and use a key:value encoding schema for a data interchange format such as JSON. Example: "{tarsusLengthInMillimeters:17.3, wingChordInMillimeters:125}"; Units: not defined; Entity described: deployment", - "format": "default", - "name": "deploy-on-measurements", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000356/2/", - "title": "deploy on measurements", - "type": "string", - }, - { - "description": "Additional information about the tag deployment that is not described by other reference data terms. Example: 'body length 154 cm; condition good'; Units: none; Entity described: deployment", - "format": "default", - "name": "deployment-comments", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000082/2/", - "title": "deployment comments", - "type": "string", - }, - { - "description": "A categorical classification describing the end of the tag deployment on the animal. Best practice is to clarify how the 'deploy-off timestamp', if present, was chosen. Values are chosen from a controlled list: analysis-end = the end time represents the end of the period of interest; captured = The tag remained on the animal but the animal was captured or confined; dead = The deployment ended with the death of the animal that was carrying the tag; dead/fall-off = The tag stopped moving, and it is not possible to determine whether it is due to death of the animal or unscheduled tag detachment; equipment-failure = The tag stopped working; fall-off = The attachment of the tag to the animal failed, and it fell of accidentally; other = other; released = The tag remained on the animal but the animal was released from captivity or confinement; removal = The tag was purposefully removed from the animal; scheduled-detachment = The tag was programmed to detach from the animal; transmission-end = The tag stopped transmitting usable data; unknown = The cause of the end of data availability or transmission is unknown. Format: controlled list; Entity described: deployment", - "format": "default", - "name": "deployment-end-type", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000084/5/", - "title": "deployment end type", - "type": "string", - }, - { - "description": "A unique identifier for the deployment of a tag on animal, provided by the data owner. If the data owner does not provide a Deployment ID, an internal Movebank deployment identifier may sometimes be shown. Example: 'Jane_42818'; Units: none; Entity described: deployment", - "format": "default", - "name": "deployment-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000085/3/", - "title": "deployment ID", - "type": "string", - }, - { - "description": "Comments about the location accuracy. This can further describe values provided in 'location error text', 'location error numerical', 'vertical error numerical', 'lat lower', 'lat upper', 'long lower' and/or 'long upper'. The percentile uncertainty can be provided using 'location error percentile'. Example: '1 standard deviation errors, assuming normal distribution, provided by the GPS unit'; Units: none; Entity described: deployment", - "format": "default", - "name": "location-accuracy-comments", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000141/3/", - "title": "location accuracy comments", - "type": "string", - }, - { - "description": "The way in which the animal was manipulated during the deployment. Additional information can be provided using 'manipulation comments'. Changes in manipulation status during deployment can be identified using 'manipulation status'. Values are chosen from a controlled list: confined = The animal's movement was restricted to within a defined area; domesticated = The animal is domesticated, for example, is a house pet or part of a managed herd; manipulated-other = The animal was manipulated in some other way, such as a physiological manipulation; none = The animal received no treatment other than tag attachment and related measurements and sampling (if applicable); reintroduction = The animal has been reintroduced as part of wildlife conservation or management efforts; relocated = The animal was released from a site other than the one at which it was captured. Format: controlled list; Entity described: deployment", - "format": "default", - "name": "manipulation-type", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000155/6/", - "title": "manipulation type", - "type": "string", - }, - { - "description": "A location such as the deployment site, study site, or colony name. Example: 'Pickerel Island North'; Units: none; Entity described: deployment", - "format": "default", - "name": "study-site", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000175/3/", - "title": "study site", - "type": "string", - }, - { - "description": "The tag firmware and version used during the deployment. If needed, identify the relevant sensors on the tag. Units: none; Entity described: deployment", - "format": "default", - "name": "tag-firmware", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000380/1/", - "title": "tag firmware", - "type": "string", - }, - { - "description": "The company or person that produced the tag. Example: 'Holohil'; Units: none; Entity described: tag", - "format": "default", - "name": "tag-manufacturer-name", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000183/3/", - "title": "tag manufacturer name", - "type": "string", - }, - { - "description": "The mass of the tag. Can be used with 'tag mass total' to define the mass of the tag separately from that of the tag with additional hardware. Example: '24'; Units: grams; Entity described: tag", - "format": "default", - "name": "tag-mass", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000184/4/", - "title": "tag mass", - "type": "number", - }, - { - "description": "The way the data are received from the tag. Values are chosen from a controlled list: ISS = Data are transferred via the International Space Station; LPWAN = Data are transferred through a low-power wide-area network, such as LoRa or Sigfox; multiple = Data are acquired using multiple methods; none = Data are obtained without use of an animal-borne tag, such as by observing a unique marking; other-wireless = Data are transferred via another form of wireless data transfer, such as a VHF transmitter/receiver; phone-network = Data are transferred via a phone network, such as GSM or AMPS; satellite = Data are transferred via satellite; tag-retrieval = The tag must be physically retrieved in order to obtain the data; telemetry-network = Data are obtained through a radio or acoustic telemetry network; Wi-Fi/Bluetooth = Data are transferred via a local Wi-Fi or Bluetooth system. Format: controlled list; Entity described: deployment", - "format": "default", - "name": "tag-readout-method", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000188/4/", - "title": "tag readout method", - "type": "string", - }, - { - "description": "The serial number of the tag. Example: 'MN93-33243'; Units: none; Entity described: tag", - "format": "default", - "name": "tag-serial-no", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000189/3/", - "title": "tag serial no", - "type": "string", - }, - ], - "primaryKey": [ - "animal-id", - "tag-id", - ], - }, - "zenodo:key": undefined, - "zenodo:url": undefined, - }, - { - "$schema": undefined, - "encoding": "UTF-8", - "format": "csv", - "mediatype": "text/csv", - "name": "gps", - "path": [ - "https://zenodo.org/records/10053903/files/O_ASSEN-gps-2018.csv.gz", - "https://zenodo.org/records/10053903/files/O_ASSEN-gps-2019.csv.gz", - ], - "profile": "tabular-data-resource", - "schema": { - "$schema": undefined, - "fields": [ - { - "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", - "format": "default", - "name": "event-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/", - "title": "event ID", - "type": "integer", - }, - { - "description": "Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event", - "format": "default", - "name": "visible", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/", - "title": "visible", - "type": "boolean", - }, - { - "description": "The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event", - "format": "%Y-%m-%d %H:%M:%S.%f", - "name": "timestamp", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/", - "title": "timestamp", - "type": "datetime", - }, - { - "description": "The geographic longitude of the location as estimated by the sensor. Positive values are east of the Greenwich Meridian, negative values are west of it. Example: '-121.1761111'; Units: decimal degrees, WGS84 reference system; Entity described: event", - "format": "default", - "name": "location-long", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000146/2/", - "title": "location long", - "type": "number", - }, - { - "description": "The geographic latitude of the location as estimated by the sensor. Example: '-41.0982423'; Units: decimal degrees, WGS84 reference system; Entity described: event", - "format": "default", - "name": "location-lat", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000145/4/", - "title": "location lat", - "type": "number", - }, - { - "description": "The barometric air or water pressure. Example: '32536.0'; Units: mbar (hPa); Entity described: event", - "format": "default", - "name": "bar:barometric-pressure", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000055/3/", - "title": "barometric pressure", - "type": "number", - }, - { - "description": "The temperature measured by the tag (different from ambient temperature or internal body temperature of the animal). Example: '32.1'; Units: degrees Celsius; Entity described: event", - "format": "default", - "name": "external-temperature", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000104/2/", - "title": "external temperature", - "type": "number", - }, - { - "description": "Dilution of precision provided by the GPS. Example: '1.8'; Units: unitless; Entity described: event", - "format": "default", - "name": "gps:dop", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000115/2/", - "title": "GPS DOP", - "type": "number", - }, - { - "description": "The number of GPS satellites used to estimate the location. Example: '8'; Units: count; Entity described: event", - "format": "default", - "name": "gps:satellite-count", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000120/3/", - "title": "GPS satellite count", - "type": "integer", - }, - { - "description": "The time required to obtain the GPS location fix. Example: '36'; Units: seconds; Entity described: event", - "format": "default", - "name": "gps-time-to-fix", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000121/3/", - "title": "GPS time to fix", - "type": "number", - }, - { - "description": "The estimated ground speed provided by the sensor or calculated between consecutive locations. Example: '7.22'; Units: m/s; Entity described: event", - "format": "default", - "name": "ground-speed", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000124/2/", - "title": "ground speed", - "type": "number", - }, - { - "description": "The direction in which the tag is moving, in decimal degrees clockwise from north, as provided by the sensor or calculated between consecutive locations. Values range from 0-360: 0 = north, 90 = east, 180 = south, 270 = west. Example: '315.88'; Units: degrees clockwise from north; Entity described: event", - "format": "default", - "name": "heading", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000129/2/", - "title": "heading", - "type": "number", - }, - { - "description": "The estimated height of the tag above mean sea level, typically estimated by the tag. If altitudes are calculated as height above an ellipsoid, use 'height above ellipsoid'. Example: '34'; Units: meters; Entity described: event", - "format": "default", - "name": "height-above-msl", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000131/3/", - "title": "height above mean sea level", - "type": "number", - }, - { - "description": "Identifies events as outliers. Outliers have the value TRUE. Typically used to import a record of outliers that were identified by the data provider or owner with automated methods outside of Movebank. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event", - "format": "default", - "name": "import-marked-outlier", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000133/3/", - "title": "import marked outlier", - "type": "boolean", - }, - { - "description": "An estimate of the horizontal error of the location including only numbers. (If the error estimates include non-numerical characters such as '>' use 'location error text'.) These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '50'; Units: meters; Entity described: event", - "format": "default", - "name": "location-error-numerical", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000142/3/", - "title": "location error numerical", - "type": "number", - }, - { - "description": "Identifies events flagged manually as outliers, typically using the Event Editor in Movebank, and may also include outliers identified using other methods. Outliers have the value TRUE. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event", - "format": "default", - "name": "manually-marked-outlier", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000156/3/", - "title": "manually marked outlier", - "type": "boolean", - }, - { - "description": "An estimate of the vertical error of the location. These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '12'; Units: meters; Entity described: event", - "format": "default", - "name": "vertical-error-numerical", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000208/3/", - "title": "vertical error numerical", - "type": "number", - }, - { - "description": "The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event", - "format": "default", - "name": "sensor-type", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/", - "title": "sensor type", - "type": "string", - }, - { - "description": "This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual", - "format": "default", - "name": "individual-taxon-canonical-name", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/", - "title": "individual taxon canonical name", - "type": "string", - }, - { - "description": "This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", - "format": "default", - "name": "tag-local-identifier", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/", - "title": "tag local identifier", - "type": "string", - }, - { - "description": "This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual", - "format": "default", - "name": "individual-local-identifier", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/", - "title": "individual local identifier", - "type": "string", - }, - { - "description": "The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study", - "format": "default", - "name": "study-name", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/", - "title": "study name", - "type": "string", - }, - ], - "foreignKeys": [ - { - "fields": [ - "individual-local-identifier", - "tag-local-identifier", - ], - "reference": { - "fields": [ - "animal-id", - "tag-id", - ], - "resource": "reference-data", - }, - }, - ], - "primaryKey": [ - "event-id", - ], - }, - "zenodo:key": undefined, - "zenodo:url": undefined, - }, - { - "$schema": undefined, - "encoding": "UTF-8", - "format": "csv", - "mediatype": "text/csv", - "name": "acceleration", - "path": [ - "https://zenodo.org/records/10053903/files/O_ASSEN-acceleration-2018.csv.gz", - "https://zenodo.org/records/10053903/files/O_ASSEN-acceleration-2019.csv.gz", - ], - "profile": "tabular-data-resource", - "schema": { - "$schema": undefined, - "fields": [ - { - "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", - "format": "default", - "name": "event-id", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/", - "title": "event ID", - "type": "integer", - }, - { - "description": "Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event", - "format": "default", - "name": "visible", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/", - "title": "visible", - "type": "boolean", - }, - { - "description": "The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event", - "format": "%Y-%m-%d %H:%M:%S.%f", - "name": "timestamp", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/", - "title": "timestamp", - "type": "datetime", - }, - { - "description": "Raw acceleration values provided by the tag for the X axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.556641'; Units: not defined; Entity described: event", - "format": "default", - "name": "acceleration-raw-x", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000002/2/", - "title": "acceleration raw x", - "type": "number", - }, - { - "description": "Raw acceleration values provided by the tag for the Y axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.09375'; Units: not defined; Entity described: event", - "format": "default", - "name": "acceleration-raw-y", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000003/2/", - "title": "acceleration raw y", - "type": "number", - }, - { - "description": "Raw acceleration values provided by the tag for the Z axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '-0.84375'; Units: not defined; Entity described: event", - "format": "default", - "name": "acceleration-raw-z", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000004/2/", - "title": "acceleration raw z", - "type": "number", - }, - { - "description": "The date and time when the sampling interval or burst began. Example: '2011-01-03 13:45:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event", - "format": "%Y-%m-%d %H:%M:%S.%f", - "name": "start-timestamp", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000171/2/", - "title": "start timestamp", - "type": "datetime", - }, - { - "description": "Tilt provided by the accelerometer for the X axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event", - "format": "default", - "name": "tilt-x", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000197/2/", - "title": "tilt x", - "type": "number", - }, - { - "description": "Tilt provided by the accelerometer for the Y axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event", - "format": "default", - "name": "tilt-y", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000198/2/", - "title": "tilt y", - "type": "number", - }, - { - "description": "Tilt provided by the accelerometer for the Z axis. Example: '1'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event", - "format": "default", - "name": "tilt-z", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000199/2/", - "title": "tilt z", - "type": "number", - }, - { - "description": "The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event", - "format": "default", - "name": "sensor-type", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/", - "title": "sensor type", - "type": "string", - }, - { - "description": "This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual", - "format": "default", - "name": "individual-taxon-canonical-name", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/", - "title": "individual taxon canonical name", - "type": "string", - }, - { - "description": "This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", - "format": "default", - "name": "tag-local-identifier", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/", - "title": "tag local identifier", - "type": "string", - }, - { - "description": "This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual", - "format": "default", - "name": "individual-local-identifier", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/", - "title": "individual local identifier", - "type": "string", - }, - { - "description": "The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study", - "format": "default", - "name": "study-name", - "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/", - "title": "study name", - "type": "string", - }, - ], - "foreignKeys": [ - { - "fields": [ - "individual-local-identifier", - "tag-local-identifier", - ], - "reference": { - "fields": [ - "animal-id", - "tag-id", - ], - "resource": "reference-data", - }, - }, - ], - "primaryKey": [ - "event-id", - ], - }, - "zenodo:key": undefined, - "zenodo:url": undefined, - }, - ], - "title": "O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands)", -} -`; diff --git a/dataset/plugins/zenodo/package/fixtures/generated/loadPackageFromZenodo-should-load-a-package_3167400519/recording.har b/dataset/plugins/zenodo/package/fixtures/generated/loadPackageFromZenodo-should-load-a-package_3167400519/recording.har deleted file mode 100644 index 4f519237..00000000 --- a/dataset/plugins/zenodo/package/fixtures/generated/loadPackageFromZenodo-should-load-a-package_3167400519/recording.har +++ /dev/null @@ -1,161 +0,0 @@ -{ - "log": { - "_recordingName": "loadPackageFromZenodo-should load a package", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "b91d0d032e9ab808b928e484fd3e9df0", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 58, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://zenodo.org/api/records/15525711" - }, - "response": { - "bodySize": 3794, - "content": { - "mimeType": "application/json", - "size": 3794, - "text": "{\"created\": \"2025-05-27T10:20:07.936985+00:00\", \"modified\": \"2025-05-27T10:20:08.174083+00:00\", \"id\": 15525711, \"conceptrecid\": \"15525710\", \"doi\": \"10.5281/zenodo.15525711\", \"conceptdoi\": \"10.5281/zenodo.15525710\", \"doi_url\": \"https://doi.org/10.5281/zenodo.15525711\", \"metadata\": {\"title\": \"PM10 Concentration in Japan\", \"doi\": \"10.5281/zenodo.15525711\", \"publication_date\": \"2025-05-27\", \"description\": \"

A brief project to extract PM 10 concentration from Japan. Data set has been created using the OpenAQ data through a python script

\", \"access_right\": \"open\", \"creators\": [{\"name\": \"Sukhija, Vinay Jagdish\", \"affiliation\": null, \"orcid\": \"0000-0003-2155-6940\"}], \"keywords\": [\"PM10 concentrations\", \"Air Quality\", \"Japan\", \"OpenAQ\"], \"custom\": {\"code:programmingLanguage\": [{\"id\": \"python\", \"title\": {\"en\": \"Python\"}}]}, \"resource_type\": {\"title\": \"Dataset\", \"type\": \"dataset\"}, \"meeting\": {\"title\": \"Open Data and Open Science Conference\", \"dates\": \"27 May 2025\", \"place\": \"Padova\"}, \"license\": {\"id\": \"cc-by-4.0\"}, \"relations\": {\"version\": [{\"index\": 0, \"is_last\": true, \"parent\": {\"pid_type\": \"recid\", \"pid_value\": \"15525710\"}}]}}, \"title\": \"PM10 Concentration in Japan\", \"links\": {\"self\": \"https://zenodo.org/api/records/15525711\", \"self_html\": \"https://zenodo.org/records/15525711\", \"preview_html\": \"https://zenodo.org/records/15525711?preview=1\", \"doi\": \"https://doi.org/10.5281/zenodo.15525711\", \"self_doi\": \"https://doi.org/10.5281/zenodo.15525711\", \"self_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.15525711\", \"reserve_doi\": \"https://zenodo.org/api/records/15525711/draft/pids/doi\", \"parent\": \"https://zenodo.org/api/records/15525710\", \"parent_html\": \"https://zenodo.org/records/15525710\", \"parent_doi\": \"https://doi.org/10.5281/zenodo.15525710\", \"parent_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.15525710\", \"self_iiif_manifest\": \"https://zenodo.org/api/iiif/record:15525711/manifest\", \"self_iiif_sequence\": \"https://zenodo.org/api/iiif/record:15525711/sequence/default\", \"files\": \"https://zenodo.org/api/records/15525711/files\", \"media_files\": \"https://zenodo.org/api/records/15525711/media-files\", \"archive\": \"https://zenodo.org/api/records/15525711/files-archive\", \"archive_media\": \"https://zenodo.org/api/records/15525711/media-files-archive\", \"latest\": \"https://zenodo.org/api/records/15525711/versions/latest\", \"latest_html\": \"https://zenodo.org/records/15525711/latest\", \"versions\": \"https://zenodo.org/api/records/15525711/versions\", \"draft\": \"https://zenodo.org/api/records/15525711/draft\", \"access_links\": \"https://zenodo.org/api/records/15525711/access/links\", \"access_grants\": \"https://zenodo.org/api/records/15525711/access/grants\", \"access_users\": \"https://zenodo.org/api/records/15525711/access/users\", \"access_request\": \"https://zenodo.org/api/records/15525711/access/request\", \"access\": \"https://zenodo.org/api/records/15525711/access\", \"communities\": \"https://zenodo.org/api/records/15525711/communities\", \"communities-suggestions\": \"https://zenodo.org/api/records/15525711/communities-suggestions\", \"requests\": \"https://zenodo.org/api/records/15525711/requests\"}, \"updated\": \"2025-05-27T10:20:08.174083+00:00\", \"recid\": \"15525711\", \"revision\": 4, \"files\": [{\"id\": \"c50a6efe-4946-4054-a3bb-797a35da03d0\", \"key\": \"openaq_measurements_PM10_Japan_20250527_095520.csv\", \"size\": 272, \"checksum\": \"md5:5ac1b92a57ec809c546ea90334c845c9\", \"links\": {\"self\": \"https://zenodo.org/api/records/15525711/files/openaq_measurements_PM10_Japan_20250527_095520.csv/content\"}}], \"swh\": {}, \"owners\": [{\"id\": \"1328821\"}], \"status\": \"published\", \"stats\": {\"downloads\": 4, \"unique_downloads\": 4, \"views\": 6, \"unique_views\": 6, \"version_downloads\": 4, \"version_unique_downloads\": 4, \"version_unique_views\": 6, \"version_views\": 6}, \"state\": \"done\", \"submitted\": true}" - }, - "cookies": [ - { - "httpOnly": true, - "name": "5569e5a730cade8ff2b54f1e815f3670", - "path": "/", - "sameSite": "None", - "secure": true, - "value": "ee05b1c4fc826a86b832f3008890c77d" - } - ], - "headers": [ - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "access-control-expose-headers", - "value": "Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset" - }, - { - "name": "cache-control", - "value": "private" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" - }, - { - "name": "content-type", - "value": "application/json" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:25 GMT" - }, - { - "name": "etag", - "value": "W/\"4\"" - }, - { - "name": "link", - "value": " ; rel=\"linkset\" ; type=\"application/linkset+json\"" - }, - { - "name": "permissions-policy", - "value": "interest-cohort=()" - }, - { - "name": "referrer-policy", - "value": "strict-origin-when-cross-origin" - }, - { - "name": "retry-after", - "value": "60" - }, - { - "name": "server", - "value": "nginx" - }, - { - "name": "set-cookie", - "value": "5569e5a730cade8ff2b54f1e815f3670=ee05b1c4fc826a86b832f3008890c77d; path=/; HttpOnly; Secure; SameSite=None" - }, - { - "name": "strict-transport-security", - "value": "max-age=31556926; includeSubDomains, max-age=15768000" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "vary", - "value": "Accept-Encoding" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-frame-options", - "value": "sameorigin" - }, - { - "name": "x-ratelimit-limit", - "value": "133" - }, - { - "name": "x-ratelimit-remaining", - "value": "132" - }, - { - "name": "x-ratelimit-reset", - "value": "1754553026" - }, - { - "name": "x-request-id", - "value": "14f848af2c654cb8534b2f735f0a5686" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 1249, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:25.328Z", - "time": 560, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 560 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/dataset/plugins/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har b/dataset/plugins/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har deleted file mode 100644 index 2dd8b10d..00000000 --- a/dataset/plugins/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har +++ /dev/null @@ -1,450 +0,0 @@ -{ - "log": { - "_recordingName": "loadPackageFromZenodo-shoule merge datapackage.json if present", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "4c12784dd2adb5911ff7bf14f8b3d9b7", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 58, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://zenodo.org/api/records/10053903" - }, - "response": { - "bodySize": 10502, - "content": { - "mimeType": "application/json", - "size": 10502, - "text": "{\"created\": \"2023-10-30T11:31:16.569518+00:00\", \"modified\": \"2025-06-18T17:17:34.806869+00:00\", \"id\": 10053903, \"conceptrecid\": \"5653310\", \"doi\": \"10.5281/zenodo.10053903\", \"conceptdoi\": \"10.5281/zenodo.5653310\", \"doi_url\": \"https://doi.org/10.5281/zenodo.10053903\", \"metadata\": {\"title\": \"O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands)\", \"doi\": \"10.5281/zenodo.10053903\", \"publication_date\": \"2023-10-30\", \"description\": \"

O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands) is a bird tracking dataset published by the Vogelwerkgroep Assen, Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University, the University of Amsterdam and the Research Institute for Nature and Forest (INBO). It contains animal tracking data collected for the study O_ASSEN using trackers developed by the University of Amsterdam Bird Tracking System (UvA-BiTS, http://www.uva-bits.nl). The study was operational from 2018 to 2019. In total 6 individuals of Eurasian oystercatchers (Haematopus ostralegus) have been tagged as a breeding bird in the city of Assen (the Netherlands), mainly to study space use of oystercatchers breeding in urban areas. Data are uploaded from the UvA-BiTS database to Movebank and from there archived on Zenodo (see https://github.com/inbo/bird-tracking). No new data are expected.

\\n

See van der Kolk et al. (2022, https://doi.org/10.3897/zookeys.1123.90623) for a more detailed description of this dataset.

\\n

Files

\\n

Data in this package are exported from Movebank study 1605797471. Fields in the data follow the Movebank Attribute Dictionary and are described in datapackage.json. Files are structured as a Frictionless Data Package. You can access all data in R via https://zenodo.org/records/10053903/files/datapackage.json using frictionless.

\\n
    \\n
  • datapackage.json: technical description of the data files.
    • \\n
  • O_ASSEN-reference-data.csv: reference data about the animals, tags and deployments.
    • \\n
  • O_ASSEN-gps-yyyy.csv.gz: GPS data recorded by the tags, grouped by year.
    • \\n
  • O_ASSEN-acceleration-yyyy.csv.gz: acceleration data recorded by the tags, grouped by year.
    • \\n
\\n

Acknowledgements

\\n

These data were collected by Bert Dijkstra and Rinus Dillerop from Vogelwerkgroep Assen, in collaboration with the Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University and the University of Amsterdam (UvA). Funding was provided by the Prins Bernard Cultuurfonds Drenthe, municipality of Assen, IJsvogelfonds (from Birdlife Netherlands and Nationale Postcodeloterij) and the Waterleiding Maatschappij Drenthe. The dataset was published with funding from Stichting NLBIF - Netherlands Biodiversity Information Facility.

\", \"access_right\": \"open\", \"creators\": [{\"name\": \"Dijkstra, Bert\", \"affiliation\": \"Vogelwerkgroep Assen\"}, {\"name\": \"Dillerop, Rinus\", \"affiliation\": \"Vogelwerkgroep Assen\"}, {\"name\": \"Oosterbeek, Kees\", \"affiliation\": \"Dutch Centre for Field Ornithology\"}, {\"name\": \"Bouten, Willem\", \"affiliation\": \"University of Amsterdam\", \"orcid\": \"0000-0002-5250-8872\"}, {\"name\": \"Desmet, Peter\", \"affiliation\": \"Research Institute for Nature and Forest\", \"orcid\": \"0000-0002-8442-8025\"}, {\"name\": \"van der Kolk, Henk-Jan\", \"affiliation\": \"Dutch Bryological and Lichenological Society\", \"orcid\": \"0000-0002-8023-379X\"}, {\"name\": \"Ens, Bruno J.\", \"affiliation\": \"Dutch Centre for Field Ornithology\", \"orcid\": \"0000-0002-4659-4807\"}], \"contributors\": [{\"name\": \"Desmet, Peter\", \"affiliation\": \"Research Institute for Nature and Forest\", \"type\": \"DataCurator\", \"orcid\": \"0000-0002-8442-8025\"}], \"keywords\": [\"animal movement\", \"animal tracking\", \"gps tracking\", \"accelerometer\", \"altitude\", \"temperature\", \"biologging\", \"birds\", \"UvA-BiTS\", \"Movebank\", \"frictionlessdata\"], \"related_identifiers\": [{\"identifier\": \"https://www.movebank.org/cms/webapp?gwt_fragment=page=studies,path=study1605797471\", \"relation\": \"isDerivedFrom\", \"resource_type\": \"dataset\", \"scheme\": \"url\"}, {\"identifier\": \"https://natuurtijdschriften.nl/pub/717840/DV2018032001001005.pdf\", \"relation\": \"isSourceOf\", \"resource_type\": \"publication-article\", \"scheme\": \"url\"}, {\"identifier\": \"https://natuurtijdschriften.nl/pub/1018631/DV2019032001001005.pdf\", \"relation\": \"isSourceOf\", \"resource_type\": \"publication-article\", \"scheme\": \"url\"}, {\"identifier\": \"https://www.gbif.org/dataset/226421f2-1d29-4950-901c-aba9d0e8f2bc\", \"relation\": \"isSourceOf\", \"resource_type\": \"dataset\", \"scheme\": \"url\"}, {\"identifier\": \"https://obis.org/dataset/550b4cc1-c40d-4070-a0cb-26e010eca9d4\", \"relation\": \"isSourceOf\", \"resource_type\": \"dataset\", \"scheme\": \"url\"}, {\"identifier\": \"10.3897/zookeys.1123.90623\", \"relation\": \"isDescribedBy\", \"resource_type\": \"publication-article\", \"scheme\": \"doi\"}, {\"identifier\": \"https://inbo.github.io/movepub/\", \"relation\": \"isCompiledBy\", \"resource_type\": \"software\", \"scheme\": \"url\"}], \"language\": \"eng\", \"resource_type\": {\"title\": \"Dataset\", \"type\": \"dataset\"}, \"license\": {\"id\": \"cc-zero\"}, \"grants\": [{\"code\": \"NLBIF\", \"internal_id\": \"0566bfb96::NLBIF\", \"funder\": {\"name\": \"Naturalis Biodiversity Center\"}, \"title\": \"Netherlands Biodiversity Information Facility\"}], \"communities\": [{\"id\": \"inbo\"}, {\"id\": \"oscibio\"}, {\"id\": \"netherlandsbiodiversityinformationfacility\"}], \"relations\": {\"version\": [{\"index\": 2, \"is_last\": true, \"parent\": {\"pid_type\": \"recid\", \"pid_value\": \"5653310\"}}]}, \"notes\": \"

Changelog

\\n
    \\n
  • Add alt-project-id to the reference-data.
    • \\n
  • Reference the latest Movebank Attribute Dictionary.
    • \\n
\"}, \"title\": \"O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands)\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903\", \"self_html\": \"https://zenodo.org/records/10053903\", \"preview_html\": \"https://zenodo.org/records/10053903?preview=1\", \"doi\": \"https://doi.org/10.5281/zenodo.10053903\", \"self_doi\": \"https://doi.org/10.5281/zenodo.10053903\", \"self_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.10053903\", \"reserve_doi\": \"https://zenodo.org/api/records/10053903/draft/pids/doi\", \"parent\": \"https://zenodo.org/api/records/5653310\", \"parent_html\": \"https://zenodo.org/records/5653310\", \"parent_doi\": \"https://doi.org/10.5281/zenodo.5653310\", \"parent_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.5653310\", \"self_iiif_manifest\": \"https://zenodo.org/api/iiif/record:10053903/manifest\", \"self_iiif_sequence\": \"https://zenodo.org/api/iiif/record:10053903/sequence/default\", \"files\": \"https://zenodo.org/api/records/10053903/files\", \"media_files\": \"https://zenodo.org/api/records/10053903/media-files\", \"archive\": \"https://zenodo.org/api/records/10053903/files-archive\", \"archive_media\": \"https://zenodo.org/api/records/10053903/media-files-archive\", \"latest\": \"https://zenodo.org/api/records/10053903/versions/latest\", \"latest_html\": \"https://zenodo.org/records/10053903/latest\", \"versions\": \"https://zenodo.org/api/records/10053903/versions\", \"draft\": \"https://zenodo.org/api/records/10053903/draft\", \"access_links\": \"https://zenodo.org/api/records/10053903/access/links\", \"access_grants\": \"https://zenodo.org/api/records/10053903/access/grants\", \"access_users\": \"https://zenodo.org/api/records/10053903/access/users\", \"access_request\": \"https://zenodo.org/api/records/10053903/access/request\", \"access\": \"https://zenodo.org/api/records/10053903/access\", \"communities\": \"https://zenodo.org/api/records/10053903/communities\", \"communities-suggestions\": \"https://zenodo.org/api/records/10053903/communities-suggestions\", \"requests\": \"https://zenodo.org/api/records/10053903/requests\"}, \"updated\": \"2025-06-18T17:17:34.806869+00:00\", \"recid\": \"10053903\", \"revision\": 18, \"files\": [{\"id\": \"5ffb4935-0a78-4fd5-af2a-f0f5113ca7c0\", \"key\": \"datapackage.json\", \"size\": 42936, \"checksum\": \"md5:5ea86b7a222ac843e3833a2e50a2056b\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/datapackage.json/content\"}}, {\"id\": \"f09bbed1-50e3-46b3-a1d9-10e8adeec337\", \"key\": \"O_ASSEN-acceleration-2018.csv.gz\", \"size\": 4067780, \"checksum\": \"md5:40b771d9bd53ee5cfae0266209364d9c\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-acceleration-2018.csv.gz/content\"}}, {\"id\": \"73a7a2a5-cdfc-48a7-be65-641144ef9c38\", \"key\": \"O_ASSEN-acceleration-2019.csv.gz\", \"size\": 3696729, \"checksum\": \"md5:9d2cbbaece00ef8cf8b4fff87cf07079\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-acceleration-2019.csv.gz/content\"}}, {\"id\": \"93e83695-a619-4f09-aa93-edb2fadcd586\", \"key\": \"O_ASSEN-gps-2018.csv.gz\", \"size\": 751448, \"checksum\": \"md5:f5e041ea66ebe8708a46fe57f383f549\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-gps-2018.csv.gz/content\"}}, {\"id\": \"e64508be-1860-415f-bbbd-fc4353f33278\", \"key\": \"O_ASSEN-gps-2019.csv.gz\", \"size\": 263439, \"checksum\": \"md5:cec4550a769d4f053e92565b44d0409c\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-gps-2019.csv.gz/content\"}}, {\"id\": \"05581f20-937c-4180-824c-c3068b641d34\", \"key\": \"O_ASSEN-reference-data.csv\", \"size\": 2082, \"checksum\": \"md5:5a5da155f404d5a3b4e50c350ad089d1\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-reference-data.csv/content\"}}], \"swh\": {}, \"owners\": [{\"id\": \"6828\"}], \"status\": \"published\", \"stats\": {\"downloads\": 56783, \"unique_downloads\": 20314, \"views\": 1436, \"unique_views\": 1327, \"version_downloads\": 46734, \"version_unique_downloads\": 18284, \"version_unique_views\": 434, \"version_views\": 475}, \"state\": \"done\", \"submitted\": true}" - }, - "cookies": [ - { - "httpOnly": true, - "name": "5569e5a730cade8ff2b54f1e815f3670", - "path": "/", - "sameSite": "None", - "secure": true, - "value": "ee05b1c4fc826a86b832f3008890c77d" - } - ], - "headers": [ - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "access-control-expose-headers", - "value": "Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset" - }, - { - "name": "cache-control", - "value": "private" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" - }, - { - "name": "content-type", - "value": "application/json" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "etag", - "value": "W/\"18\"" - }, - { - "name": "link", - "value": " ; rel=\"linkset\" ; type=\"application/linkset+json\"" - }, - { - "name": "permissions-policy", - "value": "interest-cohort=()" - }, - { - "name": "referrer-policy", - "value": "strict-origin-when-cross-origin" - }, - { - "name": "retry-after", - "value": "59" - }, - { - "name": "server", - "value": "nginx" - }, - { - "name": "set-cookie", - "value": "5569e5a730cade8ff2b54f1e815f3670=ee05b1c4fc826a86b832f3008890c77d; path=/; HttpOnly; Secure; SameSite=None" - }, - { - "name": "strict-transport-security", - "value": "max-age=31556926; includeSubDomains, max-age=15768000" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "vary", - "value": "Accept-Encoding" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-frame-options", - "value": "sameorigin" - }, - { - "name": "x-ratelimit-limit", - "value": "133" - }, - { - "name": "x-ratelimit-remaining", - "value": "131" - }, - { - "name": "x-ratelimit-reset", - "value": "1754553026" - }, - { - "name": "x-request-id", - "value": "70ed876bc027711078c800c1e7030c13" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 1250, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:25.894Z", - "time": 426, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 426 - } - }, - { - "_id": "e41c8f62376401a90fe9fd1f22af5e8b", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 77, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://zenodo.org/records/10053903/files/datapackage.json" - }, - "response": { - "bodySize": 42936, - "content": { - "mimeType": "text/plain; charset=utf-8, text/plain; charset=utf-8", - "size": 42936, - "text": "{\n \"id\": \"https://doi.org/10.5281/zenodo.10053903\",\n \"profile\": \"tabular-data-package\",\n \"resources\": [\n {\n \"name\": \"reference-data\",\n \"path\": \"O_ASSEN-reference-data.csv\",\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"UTF-8\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"tag-id\",\n \"title\": \"tag ID\",\n \"description\": \"A unique identifier for the tag, provided by the data owner. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000181/2/\"\n },\n {\n \"name\": \"animal-id\",\n \"title\": \"animal ID\",\n \"description\": \"An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: 'TUSC_CV5'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000016/3/\"\n },\n {\n \"name\": \"animal-taxon\",\n \"title\": \"animal taxon\",\n \"description\": \"The scientific name of the taxon on which the tag was deployed, as defined by the Integrated Taxonomic Information System www.itis.gov. If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. The values 'test' and 'calibration' identify events relevant to animal tracking studies that should not be associated with a taxon. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000024/4/\"\n },\n {\n \"name\": \"deploy-on-date\",\n \"title\": \"deploy on timestamp\",\n \"description\": \"The timestamp when the tag deployment started. Data records recorded before this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Example: '2008-08-30 18:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000081/3/\"\n },\n {\n \"name\": \"deploy-off-date\",\n \"title\": \"deploy off timestamp\",\n \"description\": \"The timestamp when the tag deployment ended. Data records recorded after this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Further information can be provided in 'deployment end type' and 'deployment end comments'. Example: '2009-10-01 12:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000077/4/\"\n },\n {\n \"name\": \"alt-project-id\",\n \"title\": \"alt project ID\",\n \"description\": \"A name or unique identifier for a project associated with the deployment, for example a monitoring program or another data platform. Best practice is to include the name of the related database or organization followed by the project identifier. Example: 'MOTUS145'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000244/2/\"\n },\n {\n \"name\": \"animal-comments\",\n \"title\": \"animal comments\",\n \"description\": \"Additional information about the animal. Example: 'first to fledge from nest'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000012/3/\"\n },\n {\n \"name\": \"animal-life-stage\",\n \"title\": \"animal life stage\",\n \"description\": \"The age class or life stage of the animal at the beginning of the deployment. Can be years or months of age or terms such as 'adult', 'subadult' and 'juvenile'. Best practice is to define units in the values if needed (e.g. '2 years'). Example: 'juvenile, adult'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000018/3/\"\n },\n {\n \"name\": \"animal-mass\",\n \"title\": \"animal mass\",\n \"description\": \"The mass of the animal, typically at the beginning of the deployment. Example: '500'; Units: grams; Entity described: deployment\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000019/2/\"\n },\n {\n \"name\": \"animal-nickname\",\n \"title\": \"animal nickname\",\n \"description\": \"An alternate identifier for the animal. Used as the display name for animals shown in the Animal Tracker App. Example: 'Ali'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000020/2/\"\n },\n {\n \"name\": \"animal-ring-id\",\n \"title\": \"animal ring ID\",\n \"description\": \"A number or color scheme for a band or ring attached to the animal. Color bands and other markings can be stored in 'animal marker ID'. Example: '26225'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000022/3/\"\n },\n {\n \"name\": \"animal-sex\",\n \"title\": \"animal sex\",\n \"description\": \"The sex of the animal. Allowed values are m = male; f = female; u = unknown. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000023/3/\"\n },\n {\n \"name\": \"attachment-type\",\n \"title\": \"attachment type\",\n \"description\": \"The way a tag is attached to an animal. Details can be provided in 'attachment comments'. Values are chosen from a controlled list: backpack-harness = The tag is attached to the animal using a backpack-style harness; collar = The tag is attached by a collar around the animal's neck; ear-tag = The tag is attached to the animal's ear; fin mount = The tag is attached to the animal's fin; glue = The tag is attached to the animal using glue; harness = The tag is attached to the animal using a harness; implant = The tag is placed under the skin of the animal; leg-band = The tag is attached as a leg band or ring; leg-loop-harness = The tag is attached to the animal using a leg-loop-style harness; none = No tag was attached, e.g., for observations using natural markings; other = The tag is attached using another method; subcutaneous-anchor = The tag is attached using one or more anchors attached underneath the animal's skin; suction-cup = The tag is attached using one or more suction cups; sutures = The tag is attached by one or more sutures; tape = The tag is attached to the animal using tape. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000052/5/\"\n },\n {\n \"name\": \"deploy-on-latitude\",\n \"title\": \"deploy on latitude\",\n \"description\": \"The geographic latitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '27.3516'; Units: decimal degrees, WGS84 reference system; Entity described: deployment\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000078/3/\"\n },\n {\n \"name\": \"deploy-on-longitude\",\n \"title\": \"deploy on longitude\",\n \"description\": \"The geographic longitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '-97.3321'; Units: decimal degrees, WGS84 reference system; Entity described: deployment\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000079/3/\"\n },\n {\n \"name\": \"deploy-on-measurements\",\n \"title\": \"deploy on measurements\",\n \"description\": \"A list of additional measurements taken during capture of the animal at the start of the deployment. Recommended best practice is to define units and use a key:value encoding schema for a data interchange format such as JSON. Example: \\\"{tarsusLengthInMillimeters:17.3, wingChordInMillimeters:125}\\\"; Units: not defined; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000356/2/\"\n },\n {\n \"name\": \"deployment-comments\",\n \"title\": \"deployment comments\",\n \"description\": \"Additional information about the tag deployment that is not described by other reference data terms. Example: 'body length 154 cm; condition good'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000082/2/\"\n },\n {\n \"name\": \"deployment-end-type\",\n \"title\": \"deployment end type\",\n \"description\": \"A categorical classification describing the end of the tag deployment on the animal. Best practice is to clarify how the 'deploy-off timestamp', if present, was chosen. Values are chosen from a controlled list: analysis-end = the end time represents the end of the period of interest; captured = The tag remained on the animal but the animal was captured or confined; dead = The deployment ended with the death of the animal that was carrying the tag; dead/fall-off = The tag stopped moving, and it is not possible to determine whether it is due to death of the animal or unscheduled tag detachment; equipment-failure = The tag stopped working; fall-off = The attachment of the tag to the animal failed, and it fell of accidentally; other = other; released = The tag remained on the animal but the animal was released from captivity or confinement; removal = The tag was purposefully removed from the animal; scheduled-detachment = The tag was programmed to detach from the animal; transmission-end = The tag stopped transmitting usable data; unknown = The cause of the end of data availability or transmission is unknown. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000084/5/\"\n },\n {\n \"name\": \"deployment-id\",\n \"title\": \"deployment ID\",\n \"description\": \"A unique identifier for the deployment of a tag on animal, provided by the data owner. If the data owner does not provide a Deployment ID, an internal Movebank deployment identifier may sometimes be shown. Example: 'Jane_42818'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000085/3/\"\n },\n {\n \"name\": \"location-accuracy-comments\",\n \"title\": \"location accuracy comments\",\n \"description\": \"Comments about the location accuracy. This can further describe values provided in 'location error text', 'location error numerical', 'vertical error numerical', 'lat lower', 'lat upper', 'long lower' and/or 'long upper'. The percentile uncertainty can be provided using 'location error percentile'. Example: '1 standard deviation errors, assuming normal distribution, provided by the GPS unit'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000141/3/\"\n },\n {\n \"name\": \"manipulation-type\",\n \"title\": \"manipulation type\",\n \"description\": \"The way in which the animal was manipulated during the deployment. Additional information can be provided using 'manipulation comments'. Changes in manipulation status during deployment can be identified using 'manipulation status'. Values are chosen from a controlled list: confined = The animal's movement was restricted to within a defined area; domesticated = The animal is domesticated, for example, is a house pet or part of a managed herd; manipulated-other = The animal was manipulated in some other way, such as a physiological manipulation; none = The animal received no treatment other than tag attachment and related measurements and sampling (if applicable); reintroduction = The animal has been reintroduced as part of wildlife conservation or management efforts; relocated = The animal was released from a site other than the one at which it was captured. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000155/6/\"\n },\n {\n \"name\": \"study-site\",\n \"title\": \"study site\",\n \"description\": \"A location such as the deployment site, study site, or colony name. Example: 'Pickerel Island North'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000175/3/\"\n },\n {\n \"name\": \"tag-firmware\",\n \"title\": \"tag firmware\",\n \"description\": \"The tag firmware and version used during the deployment. If needed, identify the relevant sensors on the tag. Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000380/1/\"\n },\n {\n \"name\": \"tag-manufacturer-name\",\n \"title\": \"tag manufacturer name\",\n \"description\": \"The company or person that produced the tag. Example: 'Holohil'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000183/3/\"\n },\n {\n \"name\": \"tag-mass\",\n \"title\": \"tag mass\",\n \"description\": \"The mass of the tag. Can be used with 'tag mass total' to define the mass of the tag separately from that of the tag with additional hardware. Example: '24'; Units: grams; Entity described: tag\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000184/4/\"\n },\n {\n \"name\": \"tag-readout-method\",\n \"title\": \"tag readout method\",\n \"description\": \"The way the data are received from the tag. Values are chosen from a controlled list: ISS = Data are transferred via the International Space Station; LPWAN = Data are transferred through a low-power wide-area network, such as LoRa or Sigfox; multiple = Data are acquired using multiple methods; none = Data are obtained without use of an animal-borne tag, such as by observing a unique marking; other-wireless = Data are transferred via another form of wireless data transfer, such as a VHF transmitter/receiver; phone-network = Data are transferred via a phone network, such as GSM or AMPS; satellite = Data are transferred via satellite; tag-retrieval = The tag must be physically retrieved in order to obtain the data; telemetry-network = Data are obtained through a radio or acoustic telemetry network; Wi-Fi/Bluetooth = Data are transferred via a local Wi-Fi or Bluetooth system. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000188/4/\"\n },\n {\n \"name\": \"tag-serial-no\",\n \"title\": \"tag serial no\",\n \"description\": \"The serial number of the tag. Example: 'MN93-33243'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000189/3/\"\n }\n ],\n \"primaryKey\": [\"animal-id\", \"tag-id\"]\n }\n },\n {\n \"name\": \"gps\",\n \"path\": [\"O_ASSEN-gps-2018.csv.gz\", \"O_ASSEN-gps-2019.csv.gz\"],\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"UTF-8\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"event-id\",\n \"title\": \"event ID\",\n \"description\": \"An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event\",\n \"type\": \"integer\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/\"\n },\n {\n \"name\": \"visible\",\n \"title\": \"visible\",\n \"description\": \"Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/\"\n },\n {\n \"name\": \"timestamp\",\n \"title\": \"timestamp\",\n \"description\": \"The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/\"\n },\n {\n \"name\": \"location-long\",\n \"title\": \"location long\",\n \"description\": \"The geographic longitude of the location as estimated by the sensor. Positive values are east of the Greenwich Meridian, negative values are west of it. Example: '-121.1761111'; Units: decimal degrees, WGS84 reference system; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000146/2/\"\n },\n {\n \"name\": \"location-lat\",\n \"title\": \"location lat\",\n \"description\": \"The geographic latitude of the location as estimated by the sensor. Example: '-41.0982423'; Units: decimal degrees, WGS84 reference system; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000145/4/\"\n },\n {\n \"name\": \"bar:barometric-pressure\",\n \"title\": \"barometric pressure\",\n \"description\": \"The barometric air or water pressure. Example: '32536.0'; Units: mbar (hPa); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000055/3/\"\n },\n {\n \"name\": \"external-temperature\",\n \"title\": \"external temperature\",\n \"description\": \"The temperature measured by the tag (different from ambient temperature or internal body temperature of the animal). Example: '32.1'; Units: degrees Celsius; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000104/2/\"\n },\n {\n \"name\": \"gps:dop\",\n \"title\": \"GPS DOP\",\n \"description\": \"Dilution of precision provided by the GPS. Example: '1.8'; Units: unitless; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000115/2/\"\n },\n {\n \"name\": \"gps:satellite-count\",\n \"title\": \"GPS satellite count\",\n \"description\": \"The number of GPS satellites used to estimate the location. Example: '8'; Units: count; Entity described: event\",\n \"type\": \"integer\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000120/3/\"\n },\n {\n \"name\": \"gps-time-to-fix\",\n \"title\": \"GPS time to fix\",\n \"description\": \"The time required to obtain the GPS location fix. Example: '36'; Units: seconds; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000121/3/\"\n },\n {\n \"name\": \"ground-speed\",\n \"title\": \"ground speed\",\n \"description\": \"The estimated ground speed provided by the sensor or calculated between consecutive locations. Example: '7.22'; Units: m/s; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000124/2/\"\n },\n {\n \"name\": \"heading\",\n \"title\": \"heading\",\n \"description\": \"The direction in which the tag is moving, in decimal degrees clockwise from north, as provided by the sensor or calculated between consecutive locations. Values range from 0-360: 0 = north, 90 = east, 180 = south, 270 = west. Example: '315.88'; Units: degrees clockwise from north; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000129/2/\"\n },\n {\n \"name\": \"height-above-msl\",\n \"title\": \"height above mean sea level\",\n \"description\": \"The estimated height of the tag above mean sea level, typically estimated by the tag. If altitudes are calculated as height above an ellipsoid, use 'height above ellipsoid'. Example: '34'; Units: meters; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000131/3/\"\n },\n {\n \"name\": \"import-marked-outlier\",\n \"title\": \"import marked outlier\",\n \"description\": \"Identifies events as outliers. Outliers have the value TRUE. Typically used to import a record of outliers that were identified by the data provider or owner with automated methods outside of Movebank. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000133/3/\"\n },\n {\n \"name\": \"location-error-numerical\",\n \"title\": \"location error numerical\",\n \"description\": \"An estimate of the horizontal error of the location including only numbers. (If the error estimates include non-numerical characters such as '>' use 'location error text'.) These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '50'; Units: meters; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000142/3/\"\n },\n {\n \"name\": \"manually-marked-outlier\",\n \"title\": \"manually marked outlier\",\n \"description\": \"Identifies events flagged manually as outliers, typically using the Event Editor in Movebank, and may also include outliers identified using other methods. Outliers have the value TRUE. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000156/3/\"\n },\n {\n \"name\": \"vertical-error-numerical\",\n \"title\": \"vertical error numerical\",\n \"description\": \"An estimate of the vertical error of the location. These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '12'; Units: meters; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000208/3/\"\n },\n {\n \"name\": \"sensor-type\",\n \"title\": \"sensor type\",\n \"description\": \"The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/\"\n },\n {\n \"name\": \"individual-taxon-canonical-name\",\n \"title\": \"individual taxon canonical name\",\n \"description\": \"This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/\"\n },\n {\n \"name\": \"tag-local-identifier\",\n \"title\": \"tag local identifier\",\n \"description\": \"This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/\"\n },\n {\n \"name\": \"individual-local-identifier\",\n \"title\": \"individual local identifier\",\n \"description\": \"This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/\"\n },\n {\n \"name\": \"study-name\",\n \"title\": \"study name\",\n \"description\": \"The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/\"\n }\n ],\n \"primaryKey\": \"event-id\",\n \"foreignKeys\": [\n {\n \"fields\": [\"individual-local-identifier\", \"tag-local-identifier\"],\n \"reference\": {\n \"resource\": \"reference-data\",\n \"fields\": [\"animal-id\", \"tag-id\"]\n }\n }\n ]\n }\n },\n {\n \"name\": \"acceleration\",\n \"path\": [\"O_ASSEN-acceleration-2018.csv.gz\", \"O_ASSEN-acceleration-2019.csv.gz\"],\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"UTF-8\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"event-id\",\n \"title\": \"event ID\",\n \"description\": \"An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event\",\n \"type\": \"integer\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/\"\n },\n {\n \"name\": \"visible\",\n \"title\": \"visible\",\n \"description\": \"Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/\"\n },\n {\n \"name\": \"timestamp\",\n \"title\": \"timestamp\",\n \"description\": \"The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/\"\n },\n {\n \"name\": \"acceleration-raw-x\",\n \"title\": \"acceleration raw x\",\n \"description\": \"Raw acceleration values provided by the tag for the X axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.556641'; Units: not defined; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000002/2/\"\n },\n {\n \"name\": \"acceleration-raw-y\",\n \"title\": \"acceleration raw y\",\n \"description\": \"Raw acceleration values provided by the tag for the Y axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.09375'; Units: not defined; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000003/2/\"\n },\n {\n \"name\": \"acceleration-raw-z\",\n \"title\": \"acceleration raw z\",\n \"description\": \"Raw acceleration values provided by the tag for the Z axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '-0.84375'; Units: not defined; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000004/2/\"\n },\n {\n \"name\": \"start-timestamp\",\n \"title\": \"start timestamp\",\n \"description\": \"The date and time when the sampling interval or burst began. Example: '2011-01-03 13:45:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000171/2/\"\n },\n {\n \"name\": \"tilt-x\",\n \"title\": \"tilt x\",\n \"description\": \"Tilt provided by the accelerometer for the X axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000197/2/\"\n },\n {\n \"name\": \"tilt-y\",\n \"title\": \"tilt y\",\n \"description\": \"Tilt provided by the accelerometer for the Y axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000198/2/\"\n },\n {\n \"name\": \"tilt-z\",\n \"title\": \"tilt z\",\n \"description\": \"Tilt provided by the accelerometer for the Z axis. Example: '1'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000199/2/\"\n },\n {\n \"name\": \"sensor-type\",\n \"title\": \"sensor type\",\n \"description\": \"The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/\"\n },\n {\n \"name\": \"individual-taxon-canonical-name\",\n \"title\": \"individual taxon canonical name\",\n \"description\": \"This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/\"\n },\n {\n \"name\": \"tag-local-identifier\",\n \"title\": \"tag local identifier\",\n \"description\": \"This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/\"\n },\n {\n \"name\": \"individual-local-identifier\",\n \"title\": \"individual local identifier\",\n \"description\": \"This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/\"\n },\n {\n \"name\": \"study-name\",\n \"title\": \"study name\",\n \"description\": \"The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/\"\n }\n ],\n \"primaryKey\": \"event-id\",\n \"foreignKeys\": [\n {\n \"fields\": [\"individual-local-identifier\", \"tag-local-identifier\"],\n \"reference\": {\n \"resource\": \"reference-data\",\n \"fields\": [\"animal-id\", \"tag-id\"]\n }\n }\n ]\n }\n }\n ]\n}\n" - }, - "cookies": [ - { - "httpOnly": true, - "name": "5569e5a730cade8ff2b54f1e815f3670", - "path": "/", - "sameSite": "None", - "secure": true, - "value": "e26c51e5d1379a16d287881670fdf19c" - } - ], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "content-disposition", - "value": "inline" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8, text/plain; charset=utf-8" - }, - { - "name": "date", - "value": "Thu, 07 Aug 2025 07:49:26 GMT" - }, - { - "name": "oc-checksum", - "value": "MD5:5ea86b7a222ac843e3833a2e50a2056b" - }, - { - "name": "referrer-policy", - "value": "strict-origin-when-cross-origin" - }, - { - "name": "retry-after", - "value": "59" - }, - { - "name": "server", - "value": "nginx" - }, - { - "name": "set-cookie", - "value": "5569e5a730cade8ff2b54f1e815f3670=e26c51e5d1379a16d287881670fdf19c; path=/; HttpOnly; Secure; SameSite=None" - }, - { - "name": "strict-transport-security", - "value": "max-age=31556926; includeSubDomains" - }, - { - "name": "transfer-encoding", - "value": "chunked" - }, - { - "name": "vary", - "value": "Accept-Encoding" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-download-options", - "value": "noopen" - }, - { - "name": "x-frame-options", - "value": "sameorigin" - }, - { - "name": "x-permitted-cross-domain-policies", - "value": "none" - }, - { - "name": "x-ratelimit-limit", - "value": "133" - }, - { - "name": "x-ratelimit-remaining", - "value": "130" - }, - { - "name": "x-ratelimit-reset", - "value": "1754553026" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 1064, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-08-07T07:49:26.324Z", - "time": 273, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 273 - } - }, - { - "_id": "bd54488f667a8e2a2e0c71f5914bee02", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 78, - "httpVersion": "HTTP/1.1", - "method": "HEAD", - "queryString": [], - "url": "https://zenodo.org/records/10053903/files/datapackage.json" - }, - "response": { - "bodySize": 0, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 0 - }, - "cookies": [ - { - "httpOnly": true, - "name": "5569e5a730cade8ff2b54f1e815f3670", - "path": "/", - "sameSite": "None", - "secure": true, - "value": "3c2fa475c5dd7f217e458bf5ddeda192" - } - ], - "headers": [ - { - "name": "cache-control", - "value": "private" - }, - { - "name": "connection", - "value": "close" - }, - { - "name": "content-disposition", - "value": "inline" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-security-policy", - "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "date", - "value": "Fri, 31 Oct 2025 07:31:16 GMT" - }, - { - "name": "last-modified", - "value": "Wed, 23 Apr 2025 15:34:34 GMT" - }, - { - "name": "link", - "value": " ; rel=\"collection\" ; type=\"text/html\" , ; rel=\"linkset\" ; type=\"application/linkset+json\"" - }, - { - "name": "permissions-policy", - "value": "interest-cohort=()" - }, - { - "name": "referrer-policy", - "value": "strict-origin-when-cross-origin" - }, - { - "name": "retry-after", - "value": "60" - }, - { - "name": "server", - "value": "nginx" - }, - { - "name": "set-cookie", - "value": "5569e5a730cade8ff2b54f1e815f3670=3c2fa475c5dd7f217e458bf5ddeda192; path=/; HttpOnly; Secure; SameSite=None" - }, - { - "name": "strict-transport-security", - "value": "max-age=31556926; includeSubDomains, max-age=15768000" - }, - { - "name": "vary", - "value": "Accept-Encoding" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-download-options", - "value": "noopen" - }, - { - "name": "x-frame-options", - "value": "sameorigin" - }, - { - "name": "x-permitted-cross-domain-policies", - "value": "none" - }, - { - "name": "x-ratelimit-limit", - "value": "133" - }, - { - "name": "x-ratelimit-remaining", - "value": "132" - }, - { - "name": "x-ratelimit-reset", - "value": "1761895937" - }, - { - "name": "x-request-id", - "value": "dcb3f4cab21b0dbb46c56b858ff2a8b4" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 1308, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-31T07:31:16.093Z", - "time": 777, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 777 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/dataset/plugins/zenodo/package/index.ts b/dataset/plugins/zenodo/package/index.ts deleted file mode 100644 index 23021679..00000000 --- a/dataset/plugins/zenodo/package/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type { ZenodoPackage } from "./Package.ts" -export type { ZenodoCreator } from "./Creator.ts" -export { loadPackageFromZenodo } from "./load.ts" -export { savePackageToZenodo } from "./save.ts" -export { convertPackageFromZenodo } from "./convert/fromZenodo.ts" -export { convertPackageToZenodo } from "./convert/toZenodo.ts" diff --git a/dataset/plugins/zenodo/package/load.spec.ts b/dataset/plugins/zenodo/package/load.spec.ts deleted file mode 100644 index 0412fc0c..00000000 --- a/dataset/plugins/zenodo/package/load.spec.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadPackageFromZenodo } from "./load.ts" - -useRecording() - -describe("loadPackageFromZenodo", () => { - it("should load a package", async () => { - const datapackage = await loadPackageFromZenodo( - "https://zenodo.org/records/15525711", - ) - - expect(datapackage).toMatchSnapshot() - }) - - it("shoule merge datapackage.json if present", async () => { - const datapackage = await loadPackageFromZenodo( - "https://zenodo.org/records/10053903", - ) - - expect(datapackage).toMatchSnapshot() - }) -}) diff --git a/dataset/plugins/zenodo/package/load.ts b/dataset/plugins/zenodo/package/load.ts deleted file mode 100644 index 27a51f94..00000000 --- a/dataset/plugins/zenodo/package/load.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { mergePackages } from "../../../package/index.ts" -import { makeZenodoApiRequest } from "../zenodo/index.ts" -import type { ZenodoPackage } from "./Package.ts" -import { convertPackageFromZenodo } from "./convert/fromZenodo.ts" - -/** - * Load a package from a Zenodo deposit - * @param props Object containing the URL to the Zenodo deposit - * @returns Package object - */ -export async function loadPackageFromZenodo( - datasetUrl: string, - options?: { - apiKey?: string - }, -) { - const { apiKey } = options ?? {} - const sandbox = new URL(datasetUrl).host === "sandbox.zenodo.org" - - const recordId = extractRecordId(datasetUrl) - if (!recordId) { - throw new Error(`Failed to extract record ID from URL: ${datasetUrl}`) - } - - const zenodoPackage = await makeZenodoApiRequest({ - endpoint: `/records/${recordId}`, - apiKey, - sandbox, - }) - - const systemPackage = convertPackageFromZenodo(zenodoPackage) - const userPackagePath = systemPackage.resources - .filter(resource => resource["zenodo:key"] === "datapackage.json") - .map(resource => resource["zenodo:url"]) - .at(0) - - const datapackage = await mergePackages({ systemPackage, userPackagePath }) - datapackage.resources = datapackage.resources.map(resource => { - // TODO: remove these keys completely - return { ...resource, "zenodo:key": undefined, "zenodo:url": undefined } - }) - - return datapackage -} - -/** - * Extract deposit ID from URL - * - * Examples: - * - https://zenodo.org/records/1234567 - * - https://sandbox.zenodo.org/records/1234567 - */ -function extractRecordId(datasetUrl: string): string | undefined { - const url = new URL(datasetUrl) - const pathParts = url.pathname.split("/").filter(Boolean) - return pathParts.at(-1) -} diff --git a/dataset/plugins/zenodo/package/save.spec.ts b/dataset/plugins/zenodo/package/save.spec.ts deleted file mode 100644 index 94824008..00000000 --- a/dataset/plugins/zenodo/package/save.spec.ts +++ /dev/null @@ -1,632 +0,0 @@ -import { relative } from "node:path" -import type { Package } from "@dpkit/metadata" -import { loadPackageDescriptor } from "@dpkit/metadata" -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" -import { savePackageToZenodo } from "./save.ts" - -describe("savePackageToZenodo", () => { - const getFixturePath = (name: string) => - relative(process.cwd(), `${import.meta.dirname}/fixtures/${name}`) - - const mockPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - path: getFixturePath("data.csv"), - format: "csv", - bytes: 100, - }, - ], - } - - const mockOptions = { - apiKey: "test-api-key", - sandbox: true, - } - - const originalFetch = globalThis.fetch - let fetchMock: ReturnType - - beforeEach(() => { - fetchMock = vi.fn() - // @ts-ignore - globalThis.fetch = fetchMock - }) - - afterEach(() => { - globalThis.fetch = originalFetch - vi.resetAllMocks() - }) - - it.skip("should save a package", async () => { - const dataPackage = await loadPackageDescriptor( - "core/package/fixtures/package.json", - ) - - const result = await savePackageToZenodo(dataPackage, { - apiKey: "", - sandbox: true, - }) - - expect(result).toBeDefined() - }) - - it("creates a deposition in Zenodo with correct API calls", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "data.csv", - filesize: 100, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-2", - filename: "datapackage.json", - filesize: 500, - }), - }) - - const result = await savePackageToZenodo(mockPackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(3) - - const depositionCreateCall = fetchMock.mock.calls[0] - expect(depositionCreateCall).toBeDefined() - if (!depositionCreateCall) return - - expect(depositionCreateCall[0]).toContain( - "https://sandbox.zenodo.org/api/deposit/depositions", - ) - expect(depositionCreateCall[0]).toContain("access_token=test-api-key") - expect(depositionCreateCall[1]).toMatchObject({ - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }) - - const depositionPayload = JSON.parse(depositionCreateCall[1].body) - expect(depositionPayload.metadata.title).toEqual("Test Package") - expect(depositionPayload.metadata.description).toEqual("A test package") - - expect(result).toEqual({ - path: "https://sandbox.zenodo.org/records/12345/files/datapackage.json", - datasetUrl: "https://sandbox.zenodo.org/uploads/12345", - }) - }) - - it("uploads resource files to deposition", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "data.csv", - filesize: 100, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-2", - filename: "datapackage.json", - filesize: 500, - }), - }) - - await savePackageToZenodo(mockPackage, mockOptions) - - const fileUploadCall = fetchMock.mock.calls[1] - expect(fileUploadCall).toBeDefined() - if (!fileUploadCall) return - - expect(fileUploadCall[0]).toContain( - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - ) - expect(fileUploadCall[0]).toContain("access_token=test-api-key") - expect(fileUploadCall[1]).toMatchObject({ - method: "POST", - }) - - const formData = fileUploadCall[1].body - expect(formData).toBeInstanceOf(FormData) - }) - - it("uploads datapackage.json metadata file", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "data.csv", - filesize: 100, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-2", - filename: "datapackage.json", - filesize: 500, - }), - }) - - await savePackageToZenodo(mockPackage, mockOptions) - - const datapackageUploadCall = fetchMock.mock.calls[2] - expect(datapackageUploadCall).toBeDefined() - if (!datapackageUploadCall) return - - expect(datapackageUploadCall[0]).toContain( - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - ) - - const formData = datapackageUploadCall[1].body - expect(formData).toBeInstanceOf(FormData) - - const fileBlob = formData.get("file") - expect(fileBlob).toBeInstanceOf(Blob) - }) - - it("uses production API when sandbox is false", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://zenodo.org/api/deposit/depositions/12345", - html: "https://zenodo.org/deposit/12345", - files: "https://zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "data.csv", - }), - }) - - await savePackageToZenodo(mockPackage, { - apiKey: "test-api-key", - sandbox: false, - }) - - const depositionCreateCall = fetchMock.mock.calls[0] - expect(depositionCreateCall).toBeDefined() - if (!depositionCreateCall) return - - expect(depositionCreateCall[0]).toContain("https://zenodo.org/api") - expect(depositionCreateCall[0]).not.toContain("sandbox") - }) - - it("passes API key as access_token query parameter", async () => { - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - await savePackageToZenodo(mockPackage, { - apiKey: "custom-api-key", - sandbox: true, - }) - - const depositionCreateCall = fetchMock.mock.calls[0] - expect(depositionCreateCall).toBeDefined() - if (!depositionCreateCall) return - - expect(depositionCreateCall[0]).toContain("access_token=custom-api-key") - }) - - it("handles API errors from deposition creation", async () => { - fetchMock.mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: "Bad Request", - text: () => Promise.resolve("Invalid deposition data"), - }) - - await expect(savePackageToZenodo(mockPackage, mockOptions)).rejects.toThrow( - "Zenodo API error: 400 Bad Request", - ) - }) - - it("handles API errors from file upload", async () => { - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: false, - status: 500, - statusText: "Internal Server Error", - text: () => Promise.resolve("Failed to upload file"), - }) - - await expect(savePackageToZenodo(mockPackage, mockOptions)).rejects.toThrow( - "Zenodo API error: 500 Internal Server Error", - ) - }) - - it("handles packages with multiple resources", async () => { - const multiResourcePackage: Package = { - ...mockPackage, - resources: [ - { - name: "resource-1", - path: getFixturePath("data.csv"), - format: "csv", - }, - { - name: "resource-2", - path: getFixturePath("data.csv"), - format: "json", - }, - ], - } - - fetchMock.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - await savePackageToZenodo(multiResourcePackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(4) - - const secondFileUploadCall = fetchMock.mock.calls[2] - expect(secondFileUploadCall).toBeDefined() - if (!secondFileUploadCall) return - - expect(secondFileUploadCall[0]).toContain("/files") - }) - - it("handles packages with no resources", async () => { - const emptyPackage: Package = { - ...mockPackage, - resources: [], - } - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "datapackage.json", - filesize: 500, - }), - }) - - const result = await savePackageToZenodo(emptyPackage, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(result.datasetUrl).toEqual( - "https://sandbox.zenodo.org/uploads/12345", - ) - }) - - it("skips resources without path", async () => { - const packageWithoutPath: Package = { - ...mockPackage, - resources: [ - { - name: "resource-without-path", - format: "csv", - }, - ], - } - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "datapackage.json", - filesize: 500, - }), - }) - - await savePackageToZenodo(packageWithoutPath, mockOptions) - - expect(fetchMock).toHaveBeenCalledTimes(2) - }) - - it("includes contributors with author role as creators in metadata", async () => { - const packageWithContributors: Package = { - ...mockPackage, - contributors: [ - { - title: "Alice Smith", - role: "author", - path: "University of Example", - }, - { - title: "Bob Jones", - role: "author", - path: "Institute of Testing", - }, - { - title: "Charlie Brown", - role: "contributor", - }, - ], - } - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: 12345, - links: { - self: "https://sandbox.zenodo.org/api/deposit/depositions/12345", - html: "https://sandbox.zenodo.org/deposit/12345", - files: - "https://sandbox.zenodo.org/api/deposit/depositions/12345/files", - bucket: "https://sandbox.zenodo.org/api/files/bucket-id", - }, - metadata: { - title: "Test Package", - description: "A test package", - upload_type: "dataset", - creators: [ - { - name: "Alice Smith", - affiliation: "University of Example", - }, - { - name: "Bob Jones", - affiliation: "Institute of Testing", - }, - ], - }, - state: "unsubmitted", - submitted: false, - files: [], - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-1", - filename: "data.csv", - filesize: 100, - }), - }) - - fetchMock.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - id: "file-2", - filename: "datapackage.json", - filesize: 500, - }), - }) - - await savePackageToZenodo(packageWithContributors, mockOptions) - - const depositionCreateCall = fetchMock.mock.calls[0] - expect(depositionCreateCall).toBeDefined() - if (!depositionCreateCall) return - - const depositionPayload = JSON.parse(depositionCreateCall[1].body) - expect(depositionPayload.metadata.creators).toEqual([ - { - name: "Alice Smith", - affiliation: "University of Example", - }, - { - name: "Bob Jones", - affiliation: "Institute of Testing", - }, - ]) - }) -}) diff --git a/dataset/plugins/zenodo/package/save.ts b/dataset/plugins/zenodo/package/save.ts deleted file mode 100644 index 3f22c1f7..00000000 --- a/dataset/plugins/zenodo/package/save.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { blob } from "node:stream/consumers" -import type { Descriptor, Package } from "@dpkit/metadata" -import { stringifyDescriptor } from "@dpkit/metadata" -import { convertPackageToDescriptor } from "@dpkit/metadata" -import { getPackageBasepath } from "../../../package/index.ts" -import { saveResourceFiles } from "../../../resource/index.ts" -import { loadFileStream } from "../../../stream/index.ts" -import { makeZenodoApiRequest } from "../zenodo/index.ts" -import type { ZenodoPackage } from "./Package.ts" -import { convertPackageToZenodo } from "./convert/toZenodo.ts" - -/** - * Save a package to Zenodo - * @param options Object containing the package to save and Zenodo API details - * @returns Object with the deposit URL and DOI - */ -export async function savePackageToZenodo( - dataPackage: Package, - options: { - sandbox?: boolean - apiKey: string - }, -) { - const { apiKey, sandbox = false } = options - const basepath = getPackageBasepath(dataPackage) - - const newZenodoPackage = convertPackageToZenodo(dataPackage) - const zenodoPackage = (await makeZenodoApiRequest({ - payload: newZenodoPackage, - endpoint: "/deposit/depositions", - method: "POST", - apiKey, - sandbox, - })) as ZenodoPackage - - const resourceDescriptors: Descriptor[] = [] - for (const resource of dataPackage.resources) { - if (!resource.path) continue - - resourceDescriptors.push( - await saveResourceFiles(resource, { - basepath, - withRemote: false, - withoutFolders: true, - saveFile: async options => { - const upload = { - name: options.denormalizedPath, - data: await blob(await loadFileStream(options.normalizedPath)), - } - - // It seems that record and deposition files have different metadata - // structure, e.g. size vs filesize etc - await makeZenodoApiRequest({ - endpoint: `/deposit/depositions/${zenodoPackage.id}/files`, - method: "POST", - upload, - apiKey, - sandbox, - }) - - return options.denormalizedPath - }, - }), - ) - } - - const descriptor = { - ...convertPackageToDescriptor(dataPackage, { basepath }), - resources: resourceDescriptors, - } - - for (const denormalizedPath of ["datapackage.json"]) { - const upload = { - name: denormalizedPath, - data: new Blob([stringifyDescriptor(descriptor)]), - } - - await makeZenodoApiRequest({ - endpoint: `/deposit/depositions/${zenodoPackage.id}/files`, - method: "POST", - upload, - apiKey, - sandbox, - }) - } - - const url = new URL(zenodoPackage.links.html) - return { - path: `${url.origin}/records/${zenodoPackage.id}/files/datapackage.json`, - datasetUrl: `${url.origin}/uploads/${zenodoPackage.id}`, - } -} diff --git a/dataset/plugins/zenodo/plugin.spec.ts b/dataset/plugins/zenodo/plugin.spec.ts deleted file mode 100644 index 06023847..00000000 --- a/dataset/plugins/zenodo/plugin.spec.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/load.ts" -import { ZenodoPlugin } from "./plugin.ts" - -vi.mock("./package/load.ts", () => ({ - loadPackageFromZenodo: vi.fn(), -})) - -describe("ZenodoPlugin", () => { - let plugin: ZenodoPlugin - let mockLoadPackageFromZenodo: ReturnType - - beforeEach(() => { - plugin = new ZenodoPlugin() - mockLoadPackageFromZenodo = vi.mocked(packageModule.loadPackageFromZenodo) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from zenodo.org url", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromZenodo.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("https://zenodo.org/record/123") - - expect(mockLoadPackageFromZenodo).toHaveBeenCalledWith( - "https://zenodo.org/record/123", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for non-zenodo urls", async () => { - const result = await plugin.loadPackage("https://example.com/data") - - expect(mockLoadPackageFromZenodo).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for local paths", async () => { - const result = await plugin.loadPackage("./data") - - expect(mockLoadPackageFromZenodo).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for github urls", async () => { - const result = await plugin.loadPackage("https://github.com/owner/repo") - - expect(mockLoadPackageFromZenodo).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle sandbox.zenodo.org urls", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromZenodo.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://sandbox.zenodo.org/record/456", - ) - - expect(mockLoadPackageFromZenodo).toHaveBeenCalledWith( - "https://sandbox.zenodo.org/record/456", - ) - expect(result).toEqual(mockPackage) - }) - - it("should handle zenodo urls with paths", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromZenodo.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://zenodo.org/record/123/files/data.zip", - ) - - expect(mockLoadPackageFromZenodo).toHaveBeenCalledWith( - "https://zenodo.org/record/123/files/data.zip", - ) - expect(result).toEqual(mockPackage) - }) - - it("should handle zenodo urls with query parameters", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test", data: [] }], - } - mockLoadPackageFromZenodo.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage( - "https://zenodo.org/record/123?preview=1", - ) - - expect(mockLoadPackageFromZenodo).toHaveBeenCalledWith( - "https://zenodo.org/record/123?preview=1", - ) - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for http non-zenodo urls", async () => { - const result = await plugin.loadPackage("http://example.com/data") - - expect(mockLoadPackageFromZenodo).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/zenodo/plugin.ts b/dataset/plugins/zenodo/plugin.ts deleted file mode 100644 index 3ef62a64..00000000 --- a/dataset/plugins/zenodo/plugin.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { isRemotePath } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" -import { loadPackageFromZenodo } from "./package/load.ts" - -export class ZenodoPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isZenodo = getIsZenodo(source) - if (!isZenodo) return undefined - - const dataPackage = await loadPackageFromZenodo(source) - return dataPackage - } -} - -function getIsZenodo(path: string) { - const isRemote = isRemotePath(path) - if (!isRemote) return false - - return new URL(path).hostname.endsWith("zenodo.org") -} diff --git a/dataset/plugins/zenodo/resource/Resource.ts b/dataset/plugins/zenodo/resource/Resource.ts deleted file mode 100644 index a88b7bb9..00000000 --- a/dataset/plugins/zenodo/resource/Resource.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Zenodo File interface - */ -export interface ZenodoResource { - /** - * File identifier - */ - id: string - - /** - * File key - */ - key: string - - /** - * File size in bytes - */ - size: number - - /** - * File checksum - */ - checksum: string - - /** - * Links related to the file - */ - links: { - self: string - } -} diff --git a/dataset/plugins/zenodo/resource/convert/fromZenodo.ts b/dataset/plugins/zenodo/resource/convert/fromZenodo.ts deleted file mode 100644 index 077812fe..00000000 --- a/dataset/plugins/zenodo/resource/convert/fromZenodo.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { getFormat, getName } from "@dpkit/metadata" -import type { ZenodoResource } from "../Resource.ts" - -export function convertResourceFromZenodo(zenodoResource: ZenodoResource) { - const path = convertPath(zenodoResource.links.self) - - const resource = { - path, - name: getName(zenodoResource.key) ?? zenodoResource.id, - format: getFormat(zenodoResource.key), - bytes: zenodoResource.size, - hash: zenodoResource.checksum, - "zenodo:key": zenodoResource.key, - "zenodo:url": path, - } - - return resource -} - -function convertPath(link: string) { - return link.replace("/api/", "/").replace(/\/content$/, "") -} diff --git a/dataset/plugins/zenodo/resource/convert/toZenodo.ts b/dataset/plugins/zenodo/resource/convert/toZenodo.ts deleted file mode 100644 index f67f0cc7..00000000 --- a/dataset/plugins/zenodo/resource/convert/toZenodo.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import type { ZenodoResource } from "../Resource.ts" - -export function convertResourceToZenodo(resource: Resource) { - const zenodoResource: Partial = { - key: resource.name, - } - - if (resource.bytes) { - zenodoResource.size = resource.bytes - } - - if (resource.hash) { - zenodoResource.checksum = resource.hash - } - - return zenodoResource -} diff --git a/dataset/plugins/zenodo/resource/index.ts b/dataset/plugins/zenodo/resource/index.ts deleted file mode 100644 index aa490781..00000000 --- a/dataset/plugins/zenodo/resource/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type { ZenodoResource } from "./Resource.ts" -export { convertResourceFromZenodo } from "./convert/fromZenodo.ts" -export { convertResourceToZenodo } from "./convert/toZenodo.ts" diff --git a/dataset/plugins/zenodo/zenodo/index.ts b/dataset/plugins/zenodo/zenodo/index.ts deleted file mode 100644 index 6887883b..00000000 --- a/dataset/plugins/zenodo/zenodo/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { makeZenodoApiRequest } from "./request.ts" diff --git a/dataset/plugins/zenodo/zenodo/request.ts b/dataset/plugins/zenodo/zenodo/request.ts deleted file mode 100644 index eb9a85c8..00000000 --- a/dataset/plugins/zenodo/zenodo/request.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { Descriptor } from "@dpkit/metadata" - -export async function makeZenodoApiRequest(options: { - endpoint: string - method?: "GET" | "POST" | "PUT" | "DELETE" - payload?: Descriptor - upload?: { name: string; data: Blob } - apiKey?: string - sandbox?: boolean -}) { - const { - endpoint, - method = "GET", - payload, - upload, - apiKey, - sandbox = false, - } = options - - let body: string | FormData | undefined - const headers: Record = {} - - const baseUrl = sandbox - ? "https://sandbox.zenodo.org/api" - : "https://zenodo.org/api" - const url = new URL(`${baseUrl}${endpoint}`) - - if (apiKey) { - url.searchParams.append("access_token", apiKey) - } - - if (upload) { - body = new FormData() - body.append("file", upload.data, upload.name) - - if (payload) { - for (const [key, value] of Object.entries(payload)) { - body.append(key, JSON.stringify(value)) - } - } - } else if (payload) { - body = JSON.stringify(payload) - headers["Content-Type"] = "application/json" - } - - const response = await fetch(url.toString(), { - method, - headers, - body, - }) - - if (!response.ok) { - const errorText = await response.text() - throw new Error( - `Zenodo API error: ${response.status} ${response.statusText}\n${errorText}`, - ) - } - - return (await response.json()) as T -} diff --git a/dataset/plugins/zip/index.ts b/dataset/plugins/zip/index.ts deleted file mode 100644 index 8e03d380..00000000 --- a/dataset/plugins/zip/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./package/index.ts" -export * from "./plugin.ts" diff --git a/dataset/plugins/zip/package/index.ts b/dataset/plugins/zip/package/index.ts deleted file mode 100644 index 33c3172e..00000000 --- a/dataset/plugins/zip/package/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadPackageFromZip } from "./load.ts" -export { savePackageToZip } from "./save.ts" diff --git a/dataset/plugins/zip/package/load.spec.ts b/dataset/plugins/zip/package/load.spec.ts deleted file mode 100644 index cc4c253b..00000000 --- a/dataset/plugins/zip/package/load.spec.ts +++ /dev/null @@ -1,203 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it } from "vitest" -import { getTempFilePath, writeTempFile } from "../../../file/index.ts" -import { loadPackageFromZip } from "./load.ts" -import { savePackageToZip } from "./save.ts" - -describe("loadPackageFromZip", () => { - let tempZipPath: string - - beforeEach(() => { - tempZipPath = getTempFilePath() - }) - - it("should load a basic package from zip", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "empty-resource", - data: [], - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.name).toBe("test-package") - expect(loadedPackage.resources).toHaveLength(1) - }) - - it("should load package with metadata", async () => { - const originalPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test data package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage.name).toBe("test-package") - expect(loadedPackage.title).toBe("Test Package") - expect(loadedPackage.description).toBe("A test data package") - expect(loadedPackage.version).toBe("1.0.0") - }) - - it("should load package with inline data resources", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ], - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.resources).toHaveLength(1) - expect(loadedPackage.resources[0]?.name).toBe("test-resource") - expect(loadedPackage.resources[0]?.data).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should load package with file resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.resources).toHaveLength(1) - expect(loadedPackage.resources[0]?.name).toBe("test-resource") - expect(loadedPackage.resources[0]?.format).toBe("csv") - }) - - it("should load package with schema", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage.resources[0]?.schema).toBeDefined() - const schema = loadedPackage.resources[0]?.schema - expect(typeof schema === "object" && "fields" in schema).toBe(true) - if (typeof schema === "object" && "fields" in schema) { - expect(schema.fields).toHaveLength(2) - } - }) - - it("should load package with multiple resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "resource-1", - path: csvPath, - format: "csv", - }, - { - name: "resource-2", - data: [{ id: 1, value: 100 }], - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage).toBeDefined() - expect(loadedPackage.name).toBe("test-package") - expect(loadedPackage.resources).toHaveLength(2) - expect(loadedPackage.resources[0]?.name).toBe("resource-1") - expect(loadedPackage.resources[1]?.name).toBe("resource-2") - }) - - it("should load package with dialect", async () => { - const csvContent = "id;name\n1;alice\n2;bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - dialect: { - delimiter: ";", - }, - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const loadedPackage = await loadPackageFromZip(tempZipPath) - - expect(loadedPackage.resources[0]?.dialect).toBeDefined() - const dialect = loadedPackage.resources[0]?.dialect - expect(typeof dialect === "object" && "delimiter" in dialect).toBe(true) - if (typeof dialect === "object" && "delimiter" in dialect) { - expect(dialect.delimiter).toBe(";") - } - }) - - it("should throw error for non-existent zip file", async () => { - const nonExistentPath = "/non/existent/path.zip" - - await expect(loadPackageFromZip(nonExistentPath)).rejects.toThrow() - }) - - it("should throw error for invalid zip file", async () => { - const invalidZipPath = await writeTempFile("not a zip file") - - await expect(loadPackageFromZip(invalidZipPath)).rejects.toThrow() - }) -}) diff --git a/dataset/plugins/zip/package/load.ts b/dataset/plugins/zip/package/load.ts deleted file mode 100644 index dc42ea9d..00000000 --- a/dataset/plugins/zip/package/load.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { readFile, writeFile } from "node:fs/promises" -import { mkdir } from "node:fs/promises" -import { dirname, join } from "node:path" -import { loadPackageDescriptor } from "@dpkit/metadata" -import { getTempFolderPath } from "../../../folder/index.ts" -import { unzip } from "fflate" - -export async function loadPackageFromZip(archivePath: string) { - const basepath = getTempFolderPath() - const zipData = await readFile(archivePath) - - const entries = await new Promise>( - (resolve, reject) => { - unzip(zipData, (err, unzipped) => { - if (err) reject(err) - else resolve(unzipped) - }) - }, - ) - - for (const [filename, data] of Object.entries(entries)) { - const path = join(basepath, filename) - - if (filename.endsWith("/")) { - await mkdir(path, { recursive: true }) - continue - } - - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, data) - } - - const dataPackage = await loadPackageDescriptor( - join(basepath, "datapackage.json"), - ) - - return dataPackage -} diff --git a/dataset/plugins/zip/package/save.spec.ts b/dataset/plugins/zip/package/save.spec.ts deleted file mode 100644 index 81fddd68..00000000 --- a/dataset/plugins/zip/package/save.spec.ts +++ /dev/null @@ -1,338 +0,0 @@ -import { readFile } from "node:fs/promises" -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it } from "vitest" -import { getTempFilePath, writeTempFile } from "../../../file/index.ts" -import { loadPackageFromZip } from "./load.ts" -import { savePackageToZip } from "./save.ts" - -describe("savePackageToZip", () => { - let tempZipPath: string - - beforeEach(() => { - tempZipPath = getTempFilePath() - }) - - it("should save a basic package to zip", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save package with metadata", async () => { - const dataPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save package with inline data resources", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ], - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save package with file resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save package with multiple resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "resource-1", - path: csvPath, - format: "csv", - }, - { - name: "resource-2", - data: [{ id: 1, value: 100 }], - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save package with schema", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save package with dialect", async () => { - const csvContent = "id;name\n1;alice\n2;bob" - const csvPath = await writeTempFile(csvContent) - - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - dialect: { - delimiter: ";", - }, - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - - const fileBuffer = await readFile(tempZipPath) - expect(fileBuffer.length).toBeGreaterThan(0) - }) - - it("should save and reload package with same structure", async () => { - const originalPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const reloadedPackage = await loadPackageFromZip(tempZipPath) - - expect(reloadedPackage).toBeDefined() - expect(reloadedPackage.name).toBe("test-package") - expect(reloadedPackage.title).toBe("Test Package") - expect(reloadedPackage.description).toBe("A test package") - expect(reloadedPackage.resources).toHaveLength(1) - expect(reloadedPackage.resources[0]?.name).toBe("test-resource") - }) - - it("should save and reload package preserving metadata", async () => { - const originalPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - version: "1.0.0", - keywords: ["test", "package"], - resources: [ - { - name: "test-resource", - data: [{ id: 1 }], - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const reloadedPackage = await loadPackageFromZip(tempZipPath) - - expect(reloadedPackage.name).toBe("test-package") - expect(reloadedPackage.title).toBe("Test Package") - expect(reloadedPackage.version).toBe("1.0.0") - expect(reloadedPackage.keywords).toEqual(["test", "package"]) - }) - - it("should save and reload package with schema", async () => { - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1, name: "alice" }], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const reloadedPackage = await loadPackageFromZip(tempZipPath) - - const schema = reloadedPackage.resources[0]?.schema - expect(schema).toBeDefined() - expect(typeof schema === "object" && "fields" in schema).toBe(true) - if (typeof schema === "object" && "fields" in schema) { - expect(schema.fields).toHaveLength(2) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") - } - }) - - it("should save and reload package with file resources", async () => { - const csvContent = "id,name\n1,alice\n2,bob" - const csvPath = await writeTempFile(csvContent) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - path: csvPath, - format: "csv", - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const reloadedPackage = await loadPackageFromZip(tempZipPath) - - expect(reloadedPackage.resources).toHaveLength(1) - expect(reloadedPackage.resources[0]?.name).toBe("test-resource") - expect(reloadedPackage.resources[0]?.format).toBe("csv") - }) - - it("should throw error when saving to existing file", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [], - }, - ], - } - - await writeTempFile("existing content", { persist: true }) - const existingPath = await writeTempFile("existing content") - - await expect( - savePackageToZip(dataPackage, { archivePath: existingPath }), - ).rejects.toThrow() - }) - - it("should create valid zip file structure", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [ - { - name: "test-resource", - data: [{ id: 1 }], - }, - ], - } - - await savePackageToZip(dataPackage, { archivePath: tempZipPath }) - const reloadedPackage = await loadPackageFromZip(tempZipPath) - - expect(reloadedPackage).toMatchObject({ - name: "test-package", - resources: [ - { - name: "test-resource", - }, - ], - }) - }) - - it("should save package with multiple file resources", async () => { - const csv1Content = "id,name\n1,alice" - const csv2Content = "id,value\n1,100" - const csv1Path = await writeTempFile(csv1Content) - const csv2Path = await writeTempFile(csv2Content) - - const originalPackage: Package = { - name: "test-package", - resources: [ - { - name: "resource-1", - path: csv1Path, - format: "csv", - }, - { - name: "resource-2", - path: csv2Path, - format: "csv", - }, - ], - } - - await savePackageToZip(originalPackage, { archivePath: tempZipPath }) - const reloadedPackage = await loadPackageFromZip(tempZipPath) - - expect(reloadedPackage.resources).toHaveLength(2) - expect(reloadedPackage.resources[0]?.name).toBe("resource-1") - expect(reloadedPackage.resources[1]?.name).toBe("resource-2") - }) -}) diff --git a/dataset/plugins/zip/package/save.ts b/dataset/plugins/zip/package/save.ts deleted file mode 100644 index 0a85d38c..00000000 --- a/dataset/plugins/zip/package/save.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Buffer } from "node:buffer" -import { writeFile } from "node:fs/promises" -import type { Readable } from "node:stream" -import type { Descriptor, Package } from "@dpkit/metadata" -import { stringifyDescriptor } from "@dpkit/metadata" -import { convertPackageToDescriptor } from "@dpkit/metadata" -import { zip } from "fflate" -import { assertLocalPathVacant } from "../../../file/index.ts" -import { getPackageBasepath } from "../../../package/index.ts" -import { saveResourceFiles } from "../../../resource/index.ts" -import { loadFileStream } from "../../../stream/index.ts" - -export async function savePackageToZip( - dataPackage: Package, - options: { - archivePath: string - withRemote?: boolean - }, -) { - const { archivePath, withRemote } = options - const basepath = getPackageBasepath(dataPackage) - - await assertLocalPathVacant(archivePath) - const files: Record = {} - - const resourceDescriptors: Descriptor[] = [] - for (const resource of dataPackage.resources) { - resourceDescriptors.push( - await saveResourceFiles(resource, { - basepath, - withRemote, - saveFile: async options => { - const stream = await loadFileStream(options.normalizedPath) - const buffer = await streamToBuffer(stream) - files[options.denormalizedPath] = buffer - - return options.denormalizedPath - }, - }), - ) - } - - const descriptor = { - ...convertPackageToDescriptor(dataPackage, { basepath }), - resources: resourceDescriptors, - } - - files["datapackage.json"] = Buffer.from(stringifyDescriptor(descriptor)) - - const zipData = await new Promise((resolve, reject) => { - zip(files, (err, data) => { - if (err) reject(err) - else resolve(data) - }) - }) - - await writeFile(archivePath, zipData) -} - -async function streamToBuffer(stream: Readable) { - const chunks: Uint8Array[] = [] - for await (const chunk of stream) { - chunks.push(chunk) - } - return Buffer.concat(chunks) -} diff --git a/dataset/plugins/zip/plugin.spec.ts b/dataset/plugins/zip/plugin.spec.ts deleted file mode 100644 index 16b293d3..00000000 --- a/dataset/plugins/zip/plugin.spec.ts +++ /dev/null @@ -1,195 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import { beforeEach, describe, expect, it, vi } from "vitest" -import * as packageModule from "./package/index.ts" -import { ZipPlugin } from "./plugin.ts" - -vi.mock("./package/index.ts", () => ({ - loadPackageFromZip: vi.fn(), - savePackageToZip: vi.fn(), -})) - -describe("ZipPlugin", () => { - let plugin: ZipPlugin - let mockLoadPackageFromZip: ReturnType - let mockSavePackageToZip: ReturnType - - beforeEach(() => { - plugin = new ZipPlugin() - mockLoadPackageFromZip = vi.mocked(packageModule.loadPackageFromZip) - mockSavePackageToZip = vi.mocked(packageModule.savePackageToZip) - vi.clearAllMocks() - }) - - describe("loadPackage", () => { - it("should load package from zip file", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [{ name: "test-resource", data: [] }], - } - mockLoadPackageFromZip.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("test.zip") - - expect(mockLoadPackageFromZip).toHaveBeenCalledWith("test.zip") - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for non-zip files", async () => { - const result = await plugin.loadPackage("test.json") - - expect(mockLoadPackageFromZip).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should recognize .zip extension case-insensitively", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [], - } - mockLoadPackageFromZip.mockResolvedValue(mockPackage) - - await plugin.loadPackage("test.zip") - - expect(mockLoadPackageFromZip).toHaveBeenCalledWith("test.zip") - }) - - it("should handle paths with directories", async () => { - const mockPackage: Package = { - name: "test-package", - resources: [], - } - mockLoadPackageFromZip.mockResolvedValue(mockPackage) - - const result = await plugin.loadPackage("/path/to/file.zip") - - expect(mockLoadPackageFromZip).toHaveBeenCalledWith("/path/to/file.zip") - expect(result).toEqual(mockPackage) - }) - - it("should return undefined for files without extension", async () => { - const result = await plugin.loadPackage("test") - - expect(mockLoadPackageFromZip).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("savePackage", () => { - it("should save package to zip file", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [{ name: "test-resource", data: [] }], - } - mockSavePackageToZip.mockResolvedValue(undefined) - - const result = await plugin.savePackage(dataPackage, { - target: "output.zip", - }) - - expect(mockSavePackageToZip).toHaveBeenCalledWith(dataPackage, { - archivePath: "output.zip", - withRemote: false, - }) - expect(result).toEqual({ path: undefined }) - }) - - it("should return undefined for non-zip targets", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [], - } - - const result = await plugin.savePackage(dataPackage, { - target: "output.json", - }) - - expect(mockSavePackageToZip).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should pass withRemote option", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [], - } - mockSavePackageToZip.mockResolvedValue(undefined) - - await plugin.savePackage(dataPackage, { - target: "output.zip", - withRemote: true, - }) - - expect(mockSavePackageToZip).toHaveBeenCalledWith(dataPackage, { - archivePath: "output.zip", - withRemote: true, - }) - }) - - it("should handle withRemote as false when not provided", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [], - } - mockSavePackageToZip.mockResolvedValue(undefined) - - await plugin.savePackage(dataPackage, { - target: "output.zip", - }) - - expect(mockSavePackageToZip).toHaveBeenCalledWith(dataPackage, { - archivePath: "output.zip", - withRemote: false, - }) - }) - - it("should handle paths with directories", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [], - } - mockSavePackageToZip.mockResolvedValue(undefined) - - await plugin.savePackage(dataPackage, { - target: "/path/to/output.zip", - }) - - expect(mockSavePackageToZip).toHaveBeenCalledWith(dataPackage, { - archivePath: "/path/to/output.zip", - withRemote: false, - }) - }) - - it("should save package with metadata", async () => { - const dataPackage: Package = { - name: "test-package", - title: "Test Package", - description: "A test package", - resources: [], - } - mockSavePackageToZip.mockResolvedValue(undefined) - - await plugin.savePackage(dataPackage, { - target: "output.zip", - }) - - expect(mockSavePackageToZip).toHaveBeenCalledWith(dataPackage, { - archivePath: "output.zip", - withRemote: false, - }) - }) - - it("should return undefined for files without extension", async () => { - const dataPackage: Package = { - name: "test-package", - resources: [], - } - - const result = await plugin.savePackage(dataPackage, { - target: "output", - }) - - expect(mockSavePackageToZip).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/dataset/plugins/zip/plugin.ts b/dataset/plugins/zip/plugin.ts deleted file mode 100644 index 8ded7102..00000000 --- a/dataset/plugins/zip/plugin.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { Package } from "@dpkit/metadata" -import type { DatasetPlugin } from "../../plugin.ts" -import { loadPackageFromZip, savePackageToZip } from "./package/index.ts" - -export class ZipPlugin implements DatasetPlugin { - async loadPackage(source: string) { - const isZip = getIsZip(source) - if (!isZip) return undefined - - const dataPackage = await loadPackageFromZip(source) - return dataPackage - } - - async savePackage( - dataPackage: Package, - options: { target: string; withRemote?: boolean }, - ) { - const isZip = getIsZip(options.target) - if (!isZip) return undefined - - await savePackageToZip(dataPackage, { - archivePath: options.target, - withRemote: !!options?.withRemote, - }) - - return { path: undefined } - } -} - -function getIsZip(path: string) { - return path.endsWith(".zip") -} diff --git a/dataset/resource/index.ts b/dataset/resource/index.ts deleted file mode 100644 index c69d13f9..00000000 --- a/dataset/resource/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { saveResourceFiles } from "./save.ts" diff --git a/dataset/resource/save.spec.ts b/dataset/resource/save.spec.ts deleted file mode 100644 index a2a8c60f..00000000 --- a/dataset/resource/save.spec.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { describe, expect, it } from "vitest" -import { saveResourceFiles } from "./save.ts" - -describe("saveResourceFiles", () => { - it.each([ - { - description: "local path", - basepath: "data", - resource: { path: "data/table.csv" }, - descriptor: { path: "table.csv" }, - }, - { - description: "local paths", - basepath: "data", - resource: { path: ["data/table1.csv", "data/table2.csv"] }, - descriptor: { path: ["table1.csv", "table2.csv"] }, - }, - { - description: "local path and remote path", - basepath: "data", - resource: { - path: "data/table.csv", - schema: "https://example.com/schema.json", - }, - descriptor: { - path: "table.csv", - schema: "https://example.com/schema.json", - }, - }, - { - description: "local path and remote path using withRemote", - basepath: "data", - withRemote: true, - resource: { - type: "table", - path: "data/table.csv", - schema: "https://example.com/schema.json", - }, - descriptor: { - type: "table", - path: "table.csv", - schema: "schema.json", - }, - }, - { - description: "remote paths with the same filename using withRemote", - basepath: "data", - withRemote: true, - resource: { - path: [ - "http://example1.com/table.csv", - "http://example2.com/table.csv", - "http://example3.com/table.csv", - "http://example4.com/table.csv.zip", - "http://example5.com/table.csv.zip", - ], - }, - descriptor: { - path: [ - "table.csv", - "table-1.csv", - "table-2.csv", - "table.csv.zip", - "table-1.csv.zip", - ], - }, - }, - { - description: "local paths in different folders", - basepath: "data", - resource: { - type: "table", - path: "data/folder1/table.csv", - schema: "data/folder2/schema.json", - }, - descriptor: { - type: "table", - path: "folder1/table.csv", - schema: "folder2/schema.json", - }, - }, - { - description: "local paths in different folders using withoutFolders", - basepath: "data", - withoutFolders: true, - resource: { - type: "table", - path: "data/folder1/table.csv", - schema: "data/folder2/schema.json", - }, - descriptor: { - type: "table", - path: "folder1-table.csv", - schema: "folder2-schema.json", - }, - }, - ])( - "$description", - async ({ resource, basepath, withRemote, withoutFolders, descriptor }) => { - expect( - // @ts-ignore - await saveResourceFiles(resource, { - basepath, - withRemote, - withoutFolders, - saveFile: async props => props.denormalizedPath, - }), - ).toEqual(descriptor) - }, - ) -}) diff --git a/dataset/resource/save.ts b/dataset/resource/save.ts deleted file mode 100644 index 7c5fdbb0..00000000 --- a/dataset/resource/save.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { - convertResourceToDescriptor, - denormalizePath, - getFilename, - isRemotePath, -} from "@dpkit/metadata" -import invariant from "tiny-invariant" - -export type SaveFile = (options: { - propertyName: string - propertyIndex: number - normalizedPath: string - denormalizedPath: string -}) => Promise - -export async function saveResourceFiles( - resource: Resource, - options: { - saveFile: SaveFile - basepath?: string - withRemote?: boolean - withoutFolders?: boolean - }, -) { - const { basepath, withRemote, withoutFolders } = options - - const descriptor = convertResourceToDescriptor(resource, { basepath }) - const dedupIndexes = new Map() - - const saveFile = async (path: string, name: string, index: number) => { - const isRemote = isRemotePath(path) - - // Denormalized path always uses "/" as the path separator - let denormalizedPath = denormalizePath(path, { basepath }) - const normalizedPath = path - - if (isRemote) { - if (!withRemote) return path - const filename = getFilename(path) - if (!filename) return path - denormalizedPath = filename - } else if (withoutFolders) { - denormalizedPath = denormalizedPath.replaceAll("/", "-") - } - - const dedupIndex = dedupIndexes.get(denormalizedPath) ?? 0 - dedupIndexes.set(denormalizedPath, dedupIndex + 1) - - if (dedupIndex) { - denormalizedPath = denormalizedPath.replace( - /^(.*?)([^\/]+?)(\.[^\/]+(?:\.[^\/]+)*)$/, - `$1$2-${dedupIndex}$3`, - ) - } - - denormalizedPath = await options.saveFile({ - propertyName: name, - propertyIndex: index, - normalizedPath, - denormalizedPath, - }) - - return denormalizedPath - } - - if (typeof resource.path === "string") { - descriptor.path = await saveFile(resource.path, "path", 0) - } - - if (Array.isArray(resource.path)) { - invariant(Array.isArray(descriptor.path), "Multipart as resource.path") - for (const [index, path] of resource.path.entries()) { - descriptor.path[index] = await saveFile(path, "path", index) - } - } - - for (const name of ["dialect", "schema"] as const) { - const path = resource[name] - if (typeof path === "string") { - descriptor[name] = await saveFile(path, name, 0) - } - } - - return descriptor -} diff --git a/dataset/stream/concat.ts b/dataset/stream/concat.ts deleted file mode 100644 index 4ee77fa8..00000000 --- a/dataset/stream/concat.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { Readable } from "node:stream" -import { default as Multistream } from "multistream" - -export function concatFileStreams(streams: Readable[]) { - return new Multistream(streams) -} diff --git a/dataset/stream/index.ts b/dataset/stream/index.ts deleted file mode 100644 index 9bb335db..00000000 --- a/dataset/stream/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { concatFileStreams } from "./concat.ts" -export { loadFileStream } from "./load.ts" -export { saveFileStream } from "./save.ts" diff --git a/dataset/stream/load.ts b/dataset/stream/load.ts deleted file mode 100644 index 54cb43c1..00000000 --- a/dataset/stream/load.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { createReadStream } from "node:fs" -import { Readable, Transform } from "node:stream" -import { isRemotePath } from "@dpkit/metadata" - -export async function loadFileStream( - path: string | string[], - options?: { - index?: number - maxBytes?: number - }, -) { - const index = options?.index ?? 0 - - const paths = Array.isArray(path) ? path : [path] - const indexPath = paths[index] - - if (!indexPath) { - throw new Error(`Cannot stream resource ${indexPath} at index ${index}`) - } - - const isRemote = isRemotePath(indexPath) - const stream = isRemote - ? await loadRemoteFileStream(indexPath, options) - : await loadLocalFileStream(indexPath, options) - - return stream -} - -async function loadRemoteFileStream( - path: string, - options?: { maxBytes?: number }, -) { - const response = await fetch(path) - if (!response.body) { - throw new Error(`Cannot stream remote resource: ${path}`) - } - - let stream = Readable.fromWeb(response.body) - - if (options?.maxBytes) { - stream = limitBytesStream(stream, options.maxBytes) - } - - return stream -} - -async function loadLocalFileStream( - path: string, - options?: { maxBytes?: number }, -) { - const end = options?.maxBytes ? options.maxBytes - 1 : undefined - return createReadStream(path, { end }) -} - -function limitBytesStream(inputStream: Readable, maxBytes: number) { - let total = 0 - return inputStream.pipe( - new Transform({ - transform(chunk, _encoding, callback) { - if (total >= maxBytes) { - this.push(null) - callback() - return - } - - total += chunk.length - callback(null, chunk) - }, - }), - ) -} diff --git a/dataset/stream/save.ts b/dataset/stream/save.ts deleted file mode 100644 index 5c2ab1b0..00000000 --- a/dataset/stream/save.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { createWriteStream } from "node:fs" -import { mkdir } from "node:fs/promises" -import { dirname } from "node:path" -import type { Readable } from "node:stream" -import { pipeline } from "node:stream/promises" - -export async function saveFileStream( - stream: Readable, - options: { - path: string - overwrite?: boolean - }, -) { - const { path, overwrite } = options - - // It is an equivalent to ensureDir function that won't overwrite an existing directory - await mkdir(dirname(path), { recursive: true }) - - await pipeline( - stream, - // The "wx" flag ensures that the file won't overwrite an existing file - createWriteStream(path, { flags: overwrite ? "w" : "wx" }), - ) -} diff --git a/dataset/tsconfig.json b/dataset/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/dataset/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/dataset/typedoc.json b/dataset/typedoc.json deleted file mode 100644 index f8e49f3a..00000000 --- a/dataset/typedoc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "entryPoints": ["index.ts"], - "skipErrorChecking": true -} diff --git a/desktop/README.md b/desktop/README.md deleted file mode 100644 index 1ca42a2d..00000000 --- a/desktop/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/desktop - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/desktop/services/api.ts b/desktop/services/api.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/audio/index.ts b/desktop/services/engine.ts similarity index 100% rename from audio/index.ts rename to desktop/services/engine.ts diff --git a/document/README.md b/document/README.md deleted file mode 100644 index 2c700cc7..00000000 --- a/document/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/document - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/document/document/Document.ts b/document/document/Document.ts deleted file mode 100644 index 396dabb9..00000000 --- a/document/document/Document.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { JsonDocument } from "./types/Json.ts" - -export type Document = JsonDocument diff --git a/document/document/index.ts b/document/document/index.ts deleted file mode 100644 index a3f6d661..00000000 --- a/document/document/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type { Document } from "./Document.ts" -export { validateDocument } from "./validate.ts" -export * from "./types/Json.ts" diff --git a/document/document/types/Base.ts b/document/document/types/Base.ts deleted file mode 100644 index 2fdda134..00000000 --- a/document/document/types/Base.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface BaseDocument { - type: string -} diff --git a/document/document/types/Json.ts b/document/document/types/Json.ts deleted file mode 100644 index 224059a7..00000000 --- a/document/document/types/Json.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { BaseDocument } from "./Base.ts" - -export interface JsonDocument extends BaseDocument { - type: "json" - data: Record -} diff --git a/document/document/validate.spec.ts b/document/document/validate.spec.ts deleted file mode 100644 index 05b5b91a..00000000 --- a/document/document/validate.spec.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { describe, expect, it } from "vitest" -import { validateDocument } from "./validate.ts" - -describe("validateDocument", () => { - it("should return valid report when data matches jsonSchema", async () => { - const resource = { - name: "test-document", - data: { - name: "test-package", - version: "1.0.0", - }, - jsonSchema: { - type: "object", - required: ["name", "version"], - properties: { - name: { type: "string" }, - version: { type: "string" }, - }, - }, - } - - const report = await validateDocument(resource) - - expect(report.valid).toBe(true) - expect(report.errors).toEqual([]) - }) - - it("should return error when data is missing with jsonSchema", async () => { - const resource = { - name: "test-document", - jsonSchema: { - type: "object", - required: ["name"], - properties: { - name: { type: "string" }, - }, - }, - } - - const report = await validateDocument(resource) - - expect(report.valid).toBe(false) - expect(report.errors).toEqual([ - { - type: "data", - message: "missing test-document data", - }, - ]) - }) - - it("should return validation errors when data does not match jsonSchema", async () => { - const resource = { - name: "test-document", - data: { - name: "test-package", - version: 123, - }, - jsonSchema: { - type: "object", - required: ["name", "version"], - properties: { - name: { type: "string" }, - version: { type: "string" }, - }, - }, - } - - const report = await validateDocument(resource) - - expect(report.valid).toBe(false) - expect(report.errors).toEqual([ - { - type: "document/json", - pointer: "/version", - message: "must be string", - }, - ]) - }) -}) diff --git a/document/document/validate.ts b/document/document/validate.ts deleted file mode 100644 index 106ea74f..00000000 --- a/document/document/validate.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import type { DataError } from "@dpkit/metadata" -import type { JsonDocumentError } from "@dpkit/metadata" -import { createReport } from "@dpkit/metadata" -import { resolveJsonSchema } from "@dpkit/metadata" -import { inspectJsonValue } from "@dpkit/metadata" - -export async function validateDocument(resource: Partial) { - if (resource.jsonSchema) { - const jsonSchema = await resolveJsonSchema(resource.jsonSchema) - - if (!resource.data) { - return createReport([ - { - type: "data", - message: `missing ${resource.name} data`, - }, - ]) - } - - if (jsonSchema) { - const errors = await inspectJsonValue(resource.data, { jsonSchema }) - - return createReport( - errors.map(error => ({ - type: "document/json", - ...error, - })), - ) - } - } - - return createReport() -} diff --git a/document/index.ts b/document/index.ts deleted file mode 100644 index f4f31d7e..00000000 --- a/document/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type { Document } from "./document/index.ts" -export type { JsonDocument } from "./document/index.ts" - -export { convertSchemaToHtml } from "./schema/index.tsx" -export { convertSchemaToMarkdown } from "./schema/index.ts" -export { validateDocument } from "./document/index.ts" diff --git a/document/package.json b/document/package.json deleted file mode 100644 index 17eca4ab..00000000 --- a/document/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@dpkit/document", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "document", - "html" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "@dpkit/dataset": "workspace:*", - "htmlfy": "^1.0.0", - "react": "^19.2.0", - "react-dom": "^19.2.0", - "remark": "^15.0.1", - "remark-gfm": "^4.0.0" - }, - "devDependencies": { - "@types/mdast": "^4.0.0", - "@types/react": "^19.2.0", - "@types/react-dom": "^19.2.0" - } -} diff --git a/document/plugin.ts b/document/plugin.ts deleted file mode 100644 index dc071d3d..00000000 --- a/document/plugin.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: consider having a plugin here (e.g. plugin.convertSchema) diff --git a/document/schema/convert/toHtml.spec.tsx b/document/schema/convert/toHtml.spec.tsx deleted file mode 100644 index 1803fc23..00000000 --- a/document/schema/convert/toHtml.spec.tsx +++ /dev/null @@ -1,502 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import { describe, expect, it } from "vitest" -import { convertSchemaToHtml } from "./toHtml.tsx" - -describe("convertSchemaToHtml", () => { - it("converts a simple schema to html table", () => { - const schema: Schema = { - fields: [ - { - name: "id", - type: "integer", - title: "Identifier", - description: "Unique identifier", - }, - { - name: "name", - type: "string", - title: "Name", - description: "Person name", - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("

Fields

") - expect(result).toContain("") - expect(result).toContain("") - expect(result).toContain("") - expect(result).toContain("") - expect(result).toContain("id?") - expect(result).toContain("name?") - expect(result).toContain("

Unique identifier

") - expect(result).toContain("

Person name

") - expect(result).toContain("integer") - expect(result).toContain("string") - }) - - it("includes schema title and description", () => { - const schema: Schema = { - title: "Test Schema", - description: "A test schema for validation", - fields: [ - { - name: "field1", - type: "string", - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain('

Test Schema

') - expect(result).toContain("

A test schema for validation

") - }) - - it("handles field constraints", () => { - const schema: Schema = { - fields: [ - { - name: "age", - type: "integer", - constraints: { - required: true, - minimum: 0, - maximum: 120, - }, - }, - { - name: "email", - type: "string", - constraints: { - required: true, - pattern: "^[a-z]+@[a-z]+\\.[a-z]+$", - }, - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("Constraints") - expect(result).toContain("required:") - expect(result).toContain("true") - expect(result).toContain("minimum:") - expect(result).toContain("0") - expect(result).toContain("maximum:") - expect(result).toContain("120") - expect(result).toContain("pattern:") - }) - - it("handles required field indicator", () => { - const schema: Schema = { - fields: [ - { - name: "requiredField", - type: "string", - constraints: { - required: true, - }, - }, - { - name: "optionalField", - type: "string", - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("requiredField") - expect(result).not.toContain("requiredField?") - expect(result).toContain("optionalField?") - }) - - it("handles empty fields array", () => { - const schema: Schema = { - fields: [], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("

Fields

") - expect(result).toContain("
NameDefinitionType
") - expect(result).toContain("
") - }) - - it("escapes HTML special characters", () => { - const schema: Schema = { - title: "Test & ", - description: 'Description with "quotes" and ', - fields: [ - { - name: "field", - type: "string", - description: "Description with ", - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("Test & <Schema>") - expect(result).toContain( - "Description with "quotes" and <tags>", - ) - expect(result).toContain( - "Description with <script>alert('xss')</script>", - ) - }) - - it("handles fields with enum constraints", () => { - const schema: Schema = { - fields: [ - { - name: "status", - type: "string", - constraints: { - enum: ["active", "inactive", "pending"], - }, - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("enum:") - expect(result).toContain("active, inactive, pending") - }) - - it("handles multiple constraint types", () => { - const schema: Schema = { - fields: [ - { - name: "username", - type: "string", - constraints: { - required: true, - unique: true, - minLength: 3, - maxLength: 20, - pattern: "^[a-zA-Z0-9_]+$", - }, - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("required:") - expect(result).toContain("unique:") - expect(result).toContain("minLength:") - expect(result).toContain("3") - expect(result).toContain("maxLength:") - expect(result).toContain("20") - expect(result).toContain("pattern:") - }) - - it("handles field examples", () => { - const schema: Schema = { - fields: [ - { - name: "email", - type: "string", - examples: ["user@example.com", "admin@test.org"], - }, - { - name: "age", - type: "integer", - examples: [25, 30], - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("Examples") - expect(result).toContain("user@example.com") - expect(result).toContain("admin@test.org") - expect(result).toContain("25") - expect(result).toContain("30") - }) - - it("handles different field types", () => { - const schema: Schema = { - fields: [ - { name: "field1", type: "string" }, - { name: "field2", type: "integer" }, - { name: "field3", type: "number" }, - { name: "field4", type: "boolean" }, - { name: "field5", type: "datetime" }, - { name: "field6", type: "any" }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain("string") - expect(result).toContain("integer") - expect(result).toContain("number") - expect(result).toContain("boolean") - expect(result).toContain("datetime") - expect(result).toContain("any") - }) - - it("sanitizes IDs for anchors", () => { - const schema: Schema = { - title: "Test Schema & More!", - fields: [ - { - name: "field-with-dashes", - type: "string", - }, - { - name: "Field With Spaces", - type: "string", - }, - ], - } - - const result = convertSchemaToHtml(schema) - - expect(result).toContain('id="test-schema-more"') - expect(result).toContain('id="field-with-dashes"') - expect(result).toContain('id="field-with-spaces"') - }) - - it("does not include top-level html tags", () => { - const schema: Schema = { - title: "Test", - fields: [{ name: "field1", type: "string" }], - } - - const result = convertSchemaToHtml(schema) - - expect(result).not.toContain("") - expect(result).not.toContain("") - expect(result).not.toContain("") - expect(result).not.toContain(" diff --git a/portal/terminal/content.config.ts b/portal/terminal/content.config.ts deleted file mode 100644 index 4a2747a6..00000000 --- a/portal/terminal/content.config.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { defineCollection } from "astro:content" -import { docsLoader } from "@astrojs/starlight/loaders" -import { docsSchema } from "@astrojs/starlight/schema" -import { changelogsLoader } from "starlight-changelogs/loader" - -export const collections = { - docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), - changelogs: defineCollection({ - loader: changelogsLoader([ - { - base: "changelog", - provider: "github", - owner: "datisthq", - repo: "dpkit", - }, - ]), - }), -} diff --git a/portal/terminal/content/docs/guides/commands.md b/portal/terminal/content/docs/guides/commands.md deleted file mode 100644 index f47b6c5d..00000000 --- a/portal/terminal/content/docs/guides/commands.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Commands -sidebar: - order: 2 ---- - -After instalation, the dpkit's command-line tool is available in your terminal under name `dpkit`. Please read the [Getting Started](../../overview/getting-started) guide for more information. - -## Command categories - -The CLI commands are organized into categories named after the main objects they support: - -- package -- resource -- dialect -- schema -- table -- file - -Each category has its own commands for example the `table` category: - -- `dpkit table convert` -- `dpkit table describe` -- `dpkit table explore` -- `dpkit table script` -- `dpkit table validate` - -## Working with data packages - -Usually non-package command support the `-p/--package` and `-r/--resource` options to specify the datapackage file path and a resource name to access an object inside a data package. - -For example, we can explore a table using this command: - -```bash -dpkit table explore table.csv -``` - -Or this command using an interactive mode: - -```bash -dpkit table explore -p datapackage.json -# it will ask you to select a resource -``` - -Or this command using both the datapackage file path and the resource name making it non-interactive similarly to the plain path-based command: - -```bash -dpkit table explore -p datapackage.json -r table -``` - -> [!TIP] -> When you use the `-p` option, the CLI will open any supported Data Package source including Zenodo, Ckan, and others. diff --git a/portal/terminal/content/docs/guides/dialect.md b/portal/terminal/content/docs/guides/dialect.md deleted file mode 100644 index 10adf63c..00000000 --- a/portal/terminal/content/docs/guides/dialect.md +++ /dev/null @@ -1,316 +0,0 @@ ---- -title: Dialect -sidebar: - order: 5 ---- - -Table Dialect commands help you work with CSV dialects - metadata that describes how to parse CSV and similar tabular text files. These commands allow you to infer parsing parameters from files, validate dialect definitions, and explore dialect properties. - -## Available Commands - -### `dpkit dialect infer` - -Infer a table dialect from a table by analyzing its structure and determining the best parsing parameters such as delimiter, quote character, and header configuration. - -```bash -dpkit dialect infer -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode -- `--sample-bytes`: Number of bytes to sample for dialect inference - -**Examples:** -```bash -# Infer dialect from CSV file -dpkit dialect infer data.csv - -# Infer from remote file -dpkit dialect infer https://example.com/data.csv - -# Infer from resource in package -dpkit dialect infer --from-package datapackage.json --from-resource "users" - -# Export dialect as JSON -dpkit dialect infer data.csv --json > dialect.json - -# Use larger sample for complex files -dpkit dialect infer complex_data.csv --sample-bytes 8192 -``` - -### `dpkit dialect explore` - -Explore a table dialect from a local or remote path to view its parsing configuration in an interactive format. - -```bash -dpkit dialect explore -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Explore dialect descriptor -dpkit dialect explore dialect.json - -# Explore remote dialect -dpkit dialect explore https://example.com/dialect.json - -# Explore dialect from package resource -dpkit dialect explore --from-package datapackage.json --from-resource "users" - -# Export dialect structure as JSON -dpkit dialect explore dialect.json --json -``` - -### `dpkit dialect validate` - -Validate a table dialect from a local or remote path against the CSV Dialect specification. - -```bash -dpkit dialect validate -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output validation results as JSON -- `-d, --debug`: Enable debug mode -- `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) -- `-a, --all`: Skip selection prompts when all can be selected - -**Examples:** -```bash -# Validate dialect descriptor -dpkit dialect validate dialect.json - -# Validate remote dialect -dpkit dialect validate https://example.com/dialect.json - -# Validate dialect from package resource -dpkit dialect validate --from-package datapackage.json --from-resource "users" - -# Get validation results as JSON -dpkit dialect validate dialect.json --json - -# Interactive selection when no path provided -dpkit dialect validate --from-package datapackage.json -``` - -### `dpkit dialect script` - -Open an interactive scripting session with a loaded table dialect. This provides a REPL environment where you can programmatically interact with the dialect definition. - -```bash -dpkit dialect script -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Available Variables:** -- `dpkit`: The dpkit library object -- `dialect`: The loaded dialect object - -**Examples:** -```bash -# Start scripting session with dialect -dpkit dialect script dialect.json - -# Script dialect from package resource -dpkit dialect script --from-package datapackage.json --from-resource "users" - -# In the REPL session: -dpkit> dialect.delimiter -dpkit> dialect.quoteChar -dpkit> dialect.header -dpkit> dialect.skipInitialSpace -``` - -## Common Workflows - -### Creating Dialect Definitions - -1. **Infer from data file:** - ```bash - dpkit dialect infer data.csv --json > dialect.json - ``` - -2. **Validate the generated dialect:** - ```bash - dpkit dialect validate dialect.json - ``` - -3. **Explore the dialect configuration:** - ```bash - dpkit dialect explore dialect.json - ``` - -### Dialect Analysis for Complex Files - -```bash -# Infer dialect with larger sample for better accuracy -dpkit dialect infer complex_file.csv --sample-bytes 16384 - -# Validate and explore for verification -dpkit dialect validate dialect.json -dpkit dialect explore dialect.json - -# Script for custom dialect analysis -dpkit dialect script dialect.json -``` - -### Working with Package Dialects - -```bash -# Validate all dialects in a package interactively -dpkit dialect validate --from-package datapackage.json - -# Infer improved dialect for specific resource -dpkit dialect infer --from-package datapackage.json --from-resource "transactions" - -# Compare dialects using scripting -dpkit dialect script --from-package datapackage.json --from-resource "users" -``` - -### Remote Dialect Handling - -```bash -# Work with remote dialects -dpkit dialect explore https://example.com/dialect.json -dpkit dialect validate https://example.com/dialect.json -dpkit dialect infer https://example.com/data.csv -``` - -## Dialect Properties - -CSV Dialect specifications typically include: - -### Core Properties -- **delimiter**: Field separator character (e.g., `,`, `;`, `\t`) -- **quoteChar**: Character used to quote fields (e.g., `"`, `'`) -- **escapeChar**: Character used to escape quotes within fields -- **doubleQuote**: Whether quotes are escaped by doubling them - -### Header Configuration -- **header**: Whether the first row contains headers -- **headerRows**: Number of header rows -- **headerJoin**: Character used to join multi-row headers - -### Whitespace Handling -- **skipInitialSpace**: Whether to skip whitespace after delimiters -- **nullSequence**: Sequence representing null values - -### Comment Handling -- **commentRows**: Number of comment rows to skip -- **commentChar**: Character indicating comment lines - -## Common Dialect Patterns - -### Standard CSV -```json -{ - "delimiter": ",", - "quoteChar": "\"", - "doubleQuote": true, - "header": true -} -``` - -### European CSV (semicolon-separated) -```json -{ - "delimiter": ";", - "quoteChar": "\"", - "doubleQuote": true, - "header": true -} -``` - -### Tab-separated values -```json -{ - "delimiter": "\t", - "quoteChar": "\"", - "doubleQuote": true, - "header": true -} -``` - -### Custom formats with comments -```json -{ - "delimiter": "|", - "quoteChar": "'", - "header": true, - "commentRows": 3, - "commentChar": "#" -} -``` - -## Troubleshooting Dialect Inference - -### For files with unusual formatting: -```bash -# Use larger sample size -dpkit dialect infer unusual_file.csv --sample-bytes 32768 - -# Check inferred dialect -dpkit dialect explore dialect.json - -# Manually verify with table commands -dpkit table explore unusual_file.csv --dialect dialect.json -``` - -### For files with multiple header rows: -```bash -# The dialect inference will detect headerRows automatically -dpkit dialect infer multi_header.csv --json - -# Verify the header configuration -dpkit dialect script dialect.json -# Then in REPL: dialect.headerRows -``` - -## Output Formats - -All dialect commands support multiple output formats: - -- **Interactive Display**: Default rich terminal interface showing dialect properties -- **JSON**: Use `--json` flag for machine-readable output -- **Debug Mode**: Use `--debug` for detailed operation logs - -## Integration with Other Commands - -Dialect commands work seamlessly with other dpkit commands: - -```bash -# Create dialect, then use it for table operations -dpkit dialect infer data.csv --json > dialect.json -dpkit table validate data.csv --dialect dialect.json - -# Work within package context -dpkit package infer *.csv --json > datapackage.json -dpkit dialect validate --from-package datapackage.json --from-resource "data" - -# Use inferred dialect for schema inference -dpkit dialect infer data.csv --json > dialect.json -dpkit schema infer data.csv --delimiter ";" --header-rows 2 -``` - -## Best Practices - -1. **Sample Size**: Use larger `--sample-bytes` for files with complex or inconsistent formatting -2. **Validation**: Always validate inferred dialects before using them in production -3. **Testing**: Test dialect definitions with actual data using table commands -4. **Documentation**: Include dialect files alongside data files for reproducibility diff --git a/portal/terminal/content/docs/guides/file.md b/portal/terminal/content/docs/guides/file.md deleted file mode 100644 index f4792d81..00000000 --- a/portal/terminal/content/docs/guides/file.md +++ /dev/null @@ -1,304 +0,0 @@ ---- -title: File -sidebar: - order: 8 ---- - -File commands help you work with individual files, providing utilities for copying, describing, and validating files of various formats. These commands are useful for file-level operations and diagnostics. - -## Available Commands - -### `dpkit file copy` - -Copy a file from one location to another with support for local and remote sources and destinations. - -```bash -dpkit file copy -``` - -**Options:** -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Copy local file -dpkit file copy data.csv backup.csv - -# Copy remote file to local -dpkit file copy https://example.com/data.csv local_data.csv - -# Copy to different directory -dpkit file copy data.csv ./backup/data_backup.csv -``` - -### `dpkit file describe` - -Describe a file's properties including size, format, encoding, and basic metadata information. - -```bash -dpkit file describe -``` - -**Options:** -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Describe local file -dpkit file describe data.csv - -# Describe remote file -dpkit file describe https://example.com/data.csv - -# Get description as JSON -dpkit file describe data.csv --json - -# Describe various file types -dpkit file describe document.pdf -dpkit file describe image.png -dpkit file describe archive.zip -``` - -### `dpkit file validate` - -Validate a file's integrity, format compliance, and accessibility. - -```bash -dpkit file validate -``` - -**Options:** -- `--json`: Output validation results as JSON -- `-d, --debug`: Enable debug mode -- `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) -- `-a, --all`: Skip selection prompts when all can be selected - -**Examples:** -```bash -# Validate local file -dpkit file validate data.csv - -# Validate remote file -dpkit file validate https://example.com/data.csv - -# Get validation results as JSON -dpkit file validate data.csv --json - -# Validate multiple file types -dpkit file validate document.json -dpkit file validate image.jpg -dpkit file validate data.parquet -``` - -## Common Workflows - -### File Backup and Migration - -```bash -# Create backup copy -dpkit file copy important_data.csv backup/important_data_$(date +%Y%m%d).csv - -# Validate backup integrity -dpkit file validate backup/important_data_20240101.csv - -# Describe backup properties -dpkit file describe backup/important_data_20240101.csv -``` - -### Remote File Handling - -```bash -# Download and validate remote file -dpkit file copy https://example.com/dataset.csv local_dataset.csv -dpkit file validate local_dataset.csv - -# Describe remote file without downloading -dpkit file describe https://example.com/dataset.csv -``` - -### File Diagnostics - -```bash -# Check file properties -dpkit file describe suspicious_file.csv - -# Validate file integrity -dpkit file validate suspicious_file.csv - -# Get detailed diagnostics as JSON -dpkit file describe problematic_file.csv --json -dpkit file validate problematic_file.csv --json -``` - -### Batch File Operations - -```bash -# Describe multiple files -for file in *.csv; do - echo "Describing $file:" - dpkit file describe "$file" - echo "---" -done - -# Validate all files in directory -for file in data/*.json; do - dpkit file validate "$file" --json >> validation_report.json -done -``` - -## File Type Support - -File commands work with various file formats: - -### Data Formats -- **CSV/TSV**: Comma and tab-separated values -- **JSON**: JavaScript Object Notation -- **Excel**: .xlsx and .xls files -- **Parquet**: Apache Parquet files -- **Arrow**: Apache Arrow files -- **ODS**: OpenDocument Spreadsheet - -### Archive Formats -- **ZIP**: Compressed archives -- **TAR**: Tape archives -- **GZ**: Gzip compressed files - -### Document Formats -- **PDF**: Portable Document Format -- **XML**: Extensible Markup Language -- **YAML**: YAML Ain't Markup Language - -### Image Formats -- **PNG**: Portable Network Graphics -- **JPEG/JPG**: Joint Photographic Experts Group -- **SVG**: Scalable Vector Graphics - -## File Information Retrieved - -### Basic Properties -- **Size**: File size in bytes -- **Format**: Detected file format and MIME type -- **Encoding**: Text encoding (for text files) -- **Permissions**: File access permissions (local files) - -### Content Analysis -- **Structure**: Basic structure analysis for supported formats -- **Validity**: Format compliance checking -- **Metadata**: Embedded metadata extraction - -### Remote File Properties -- **Accessibility**: Whether the remote file is accessible -- **Headers**: HTTP headers for remote files -- **Redirects**: Information about URL redirections - -## Error Handling - -### Common Issues and Solutions - -#### File Not Found -```bash -dpkit file describe missing_file.csv -# Error: File not found -# Solution: Check file path and permissions -``` - -#### Network Issues (Remote Files) -```bash -dpkit file copy https://unreachable.com/data.csv local.csv -# Error: Network timeout -# Solution: Check URL and network connectivity -``` - -#### Format Recognition -```bash -dpkit file describe unknown_format.dat -# May show limited information for unknown formats -# Solution: Use --debug for more details -``` - -#### Permission Issues -```bash -dpkit file copy protected_file.csv backup.csv -# Error: Permission denied -# Solution: Check file permissions -``` - -## Advanced Usage - -### Scripting and Automation -```bash -#!/bin/bash -# File processing script - -FILES="*.csv" -for file in $FILES; do - echo "Processing $file" - - # Validate file - if dpkit file validate "$file" --json | jq -r '.valid' | grep -q "true"; then - echo "✓ $file is valid" - - # Create backup - dpkit file copy "$file" "backup/${file%.csv}_$(date +%Y%m%d).csv" - - # Get file info - dpkit file describe "$file" --json > "info/${file%.csv}_info.json" - else - echo "✗ $file is invalid" - fi -done -``` - -### Integration with Other Commands -```bash -# Validate file before processing with table commands -dpkit file validate data.csv && dpkit table explore data.csv - -# Describe file and then infer schema -dpkit file describe data.csv -dpkit schema infer data.csv --json > schema.json - -# Copy and then create package -dpkit file copy remote_data.csv local_data.csv -dpkit package infer local_data.csv --json > datapackage.json -``` - -### Monitoring and Logging -```bash -# Create validation log -dpkit file validate data.csv --json | jq '{file: "data.csv", valid: .valid, timestamp: now}' >> validation.log - -# Monitor file changes -while true; do - dpkit file describe changing_file.csv --json > current_state.json - if ! cmp -s current_state.json previous_state.json; then - echo "File changed at $(date)" - cp current_state.json previous_state.json - fi - sleep 60 -done -``` - -## Output Formats - -File commands support multiple output formats: - -- **Human-readable**: Default formatted output for terminal viewing -- **JSON**: Machine-readable structured output with `--json` flag -- **Debug**: Detailed diagnostic information with `--debug` flag - -## Best Practices - -1. **Validation First**: Always validate files before processing them with other commands -2. **Backup Important Files**: Use `copy` command to create backups before modifications -3. **Remote File Handling**: Describe remote files before downloading to check size and format -4. **Error Checking**: Use JSON output for programmatic error checking and handling -5. **Documentation**: Use `describe` to document file properties for reproducibility - -## Security Considerations - -- Remote file operations follow URL protocols and security restrictions -- Local file operations respect system file permissions -- Validation helps identify potentially corrupted or malicious files -- Debug mode may expose sensitive file system information diff --git a/portal/terminal/content/docs/guides/package.md b/portal/terminal/content/docs/guides/package.md deleted file mode 100644 index 6cdaa07f..00000000 --- a/portal/terminal/content/docs/guides/package.md +++ /dev/null @@ -1,265 +0,0 @@ ---- -title: Package -sidebar: - order: 3 ---- - -Data Package commands help you work with Data Packages - collections of data files along with their metadata. These commands allow you to create, validate, explore, and publish data packages across various platforms. - -## Available Commands - -### `dpkit package copy` - -Copy a local or remote Data Package to a local folder, a ZIP archive or a database. - -```bash -dpkit package copy --to-path -``` - -**Options:** -- `--to-path` (required): Target destination for the copy -- `--with-remote`: Include remote resources in the copy -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Copy package to local directory -dpkit package copy datapackage.json --to-path ./output - -# Copy package to ZIP archive -dpkit package copy datapackage.json --to-path package.zip - -# Copy remote package including remote resources -dpkit package copy https://example.com/datapackage.json --to-path ./local --with-remote -``` - -### `dpkit package infer` - -Infer a data package from local or remote file paths. This command analyzes data files and automatically generates metadata including schema information. - -```bash -dpkit package infer -``` - -**Table Dialect Options:** -- `--delimiter`: Field delimiter character -- `--header`: Whether files have headers -- `--header-rows`: Number of header rows -- `--header-join`: Join character for multi-row headers -- `--comment-rows`: Number of comment rows to skip -- `--comment-char`: Comment character -- `--quote-char`: Quote character for fields -- `--double-quote`: Whether quotes are doubled for escaping -- `--escape-char`: Escape character -- `--null-sequence`: Sequence representing null values -- `--skip-initial-space`: Skip initial whitespace -- `--property`: JSON property path for nested data -- `--item-type`: Type of items in arrays -- `--item-keys`: Keys for object items -- `--sheet-number`: Excel sheet number -- `--sheet-name`: Excel sheet name -- `--table`: Database table name -- `--sample-bytes`: Bytes to sample for inference - -**Table Schema Options:** -- `--field-names`: Override field names -- `--field-types`: Override field types -- `--missing-values`: Values to treat as missing -- `--string-format`: String format specification -- `--decimal-char`: Decimal separator character -- `--group-char`: Thousands separator character -- `--bare-number`: Allow bare numbers -- `--true-values`: Values to treat as true -- `--false-values`: Values to treat as false -- `--datetime-format`: DateTime format string -- `--date-format`: Date format string -- `--time-format`: Time format string -- `--array-type`: Type of array elements -- `--list-delimiter`: List item delimiter -- `--list-item-type`: Type of list items -- `--geopoint-format`: Geopoint format specification -- `--geojson-format`: GeoJSON format specification -- `--sample-rows`: Rows to sample for inference -- `--confidence`: Confidence threshold for type inference -- `--comma-decimal`: Use comma as decimal separator -- `--month-first`: Parse dates with month first -- `--keep-strings`: Keep string types when possible - -**Examples:** -```bash -# Infer package from CSV files -dpkit package infer data1.csv data2.csv - -# Infer with custom delimiter -dpkit package infer data.csv --delimiter ";" - -# Infer from remote files -dpkit package infer https://example.com/data.csv -``` - -### `dpkit package explore` - -Explore a Data Package descriptor to view its structure and metadata in an interactive format. - -```bash -dpkit package explore -``` - -**Options:** -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Explore local package -dpkit package explore datapackage.json - -# Explore remote package -dpkit package explore https://example.com/datapackage.json - -# Export structure as JSON -dpkit package explore datapackage.json --json -``` - -### `dpkit package validate` - -Validate a data package from a local or remote path against the Data Package specification. - -```bash -dpkit package validate -``` - -**Options:** -- `--json`: Output validation results as JSON -- `-d, --debug`: Enable debug mode -- `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) -- `-a, --all`: Skip selection prompts when all can be selected - -**Examples:** -```bash -# Validate local package -dpkit package validate datapackage.json - -# Validate remote package -dpkit package validate https://example.com/datapackage.json - -# Get validation results as JSON -dpkit package validate datapackage.json --json -``` - -### `dpkit package script` - -Open an interactive scripting session with a loaded Data Package. This provides a REPL environment where you can programmatically interact with the package data. - -```bash -dpkit package script -``` - -**Available Variables:** -- `dpkit`: The dpkit library object -- `dataPackage`: The loaded data package object - -**Examples:** -```bash -# Start scripting session -dpkit package script datapackage.json - -# In the REPL session: -dpkit> dataPackage.resources.length -dpkit> dataPackage.resources[0].schema.fields -``` - -### `dpkit package publish` - -Publish data packages to various platforms. This is a parent command with platform-specific subcommands. - -#### `dpkit package publish ckan` - -Publish a data package to a CKAN instance. - -```bash -dpkit package publish ckan -``` - -**CKAN Options:** -- `--to-ckan-api-key`: CKAN API key for authentication -- `--to-ckan-url`: CKAN instance URL -- `--to-ckan-owner-org`: Organization to publish under -- `--to-ckan-dataset-name`: Name for the dataset - -**Examples:** -```bash -# Publish to CKAN -dpkit package publish ckan datapackage.json \ - --to-ckan-url https://demo.ckan.org \ - --to-ckan-api-key your-api-key \ - --to-ckan-owner-org your-org -``` - -#### `dpkit package publish github` - -Publish a data package to GitHub as releases or repository files. - -```bash -dpkit package publish github -``` - -#### `dpkit package publish zenodo` - -Publish a data package to Zenodo for academic archiving. - -```bash -dpkit package publish zenodo -``` - -## Common Workflows - -### Creating a New Package - -1. **Infer from data files:** - ```bash - dpkit package infer *.csv --json > datapackage.json - ``` - -2. **Validate the generated package:** - ```bash - dpkit package validate datapackage.json - ``` - -3. **Explore the package structure:** - ```bash - dpkit package explore datapackage.json - ``` - -### Working with Remote Packages - -```bash -# Explore remote package -dpkit package explore https://example.com/datapackage.json - -# Copy remote package locally -dpkit package copy https://example.com/datapackage.json --to-path ./local-copy - -# Validate remote package -dpkit package validate https://example.com/datapackage.json -``` - -### Publishing Workflow - -```bash -# Validate before publishing -dpkit package validate datapackage.json - -# Publish to CKAN -dpkit package publish ckan datapackage.json \ - --to-ckan-url https://your-ckan-instance.org \ - --to-ckan-api-key $CKAN_API_KEY -``` - -## Output Formats - -Most commands support multiple output formats: - -- **Interactive Display**: Default rich terminal interface -- **JSON**: Use `--json` flag for machine-readable output -- **Debug Mode**: Use `--debug` for detailed operation logs diff --git a/portal/terminal/content/docs/guides/resource.md b/portal/terminal/content/docs/guides/resource.md deleted file mode 100644 index d3614b6c..00000000 --- a/portal/terminal/content/docs/guides/resource.md +++ /dev/null @@ -1,271 +0,0 @@ ---- -title: Resource -sidebar: - order: 4 ---- - -Data Resource commands help you work with individual data resources - the building blocks of Data Packages. These commands allow you to infer metadata from data files, validate resource structure, and explore resource properties. - -## Available Commands - -### `dpkit resource infer` - -Infer a data resource from a table by analyzing its structure and generating metadata including schema information. - -```bash -dpkit resource infer -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Table Dialect Options:** -- `--delimiter`: Field delimiter character -- `--header`: Whether files have headers -- `--header-rows`: Number of header rows -- `--header-join`: Join character for multi-row headers -- `--comment-rows`: Number of comment rows to skip -- `--comment-char`: Comment character -- `--quote-char`: Quote character for fields -- `--double-quote`: Whether quotes are doubled for escaping -- `--escape-char`: Escape character -- `--null-sequence`: Sequence representing null values -- `--skip-initial-space`: Skip initial whitespace -- `--property`: JSON property path for nested data -- `--item-type`: Type of items in arrays -- `--item-keys`: Keys for object items -- `--sheet-number`: Excel sheet number -- `--sheet-name`: Excel sheet name -- `--table`: Database table name -- `--sample-bytes`: Bytes to sample for inference - -**Table Schema Options:** -- `--field-names`: Override field names -- `--field-types`: Override field types -- `--missing-values`: Values to treat as missing -- `--string-format`: String format specification -- `--decimal-char`: Decimal separator character -- `--group-char`: Thousands separator character -- `--bare-number`: Allow bare numbers -- `--true-values`: Values to treat as true -- `--false-values`: Values to treat as false -- `--datetime-format`: DateTime format string -- `--date-format`: Date format string -- `--time-format`: Time format string -- `--array-type`: Type of array elements -- `--list-delimiter`: List item delimiter -- `--list-item-type`: Type of list items -- `--geopoint-format`: Geopoint format specification -- `--geojson-format`: GeoJSON format specification -- `--sample-rows`: Rows to sample for inference -- `--confidence`: Confidence threshold for type inference -- `--comma-decimal`: Use comma as decimal separator -- `--month-first`: Parse dates with month first -- `--keep-strings`: Keep string types when possible - -**Examples:** -```bash -# Infer resource from CSV file -dpkit resource infer data.csv - -# Infer with custom delimiter -dpkit resource infer data.csv --delimiter ";" - -# Infer from remote file -dpkit resource infer https://example.com/data.csv - -# Infer from resource in package -dpkit resource infer --from-package datapackage.json --from-resource "users" - -# Export as JSON -dpkit resource infer data.csv --json -``` - -### `dpkit resource explore` - -Explore a data resource from a local or remote path to view its structure and metadata in an interactive format. - -```bash -dpkit resource explore -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Explore resource descriptor -dpkit resource explore resource.json - -# Explore remote resource -dpkit resource explore https://example.com/resource.json - -# Explore resource from package -dpkit resource explore --from-package datapackage.json --from-resource "users" - -# Export structure as JSON -dpkit resource explore resource.json --json -``` - -### `dpkit resource validate` - -Validate a data resource from a local or remote path against the Data Resource specification. - -```bash -dpkit resource validate [descriptor-path] -``` - -**Options:** -- `--from-package`: Path to package containing the resource -- `--from-resource`: Name of resource within package -- `--json`: Output validation results as JSON -- `--debug`: Enable debug mode -- `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) -- `-a, --all`: Skip selection prompts when all can be selected - -**Examples:** -```bash -# Validate resource descriptor -dpkit resource validate resource.json - -# Validate remote resource -dpkit resource validate https://example.com/resource.json - -# Validate resource from package -dpkit resource validate --from-package datapackage.json --from-resource "users" - -# Get validation results as JSON -dpkit resource validate resource.json --json - -# Interactive selection when no path provided -dpkit resource validate --from-package datapackage.json -``` - -### `dpkit resource script` - -Open an interactive scripting session with a loaded data resource. This provides a REPL environment where you can programmatically interact with the resource metadata. - -```bash -dpkit resource script -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Available Variables:** -- `dpkit`: The dpkit library object -- `resource`: The loaded resource object - -**Examples:** -```bash -# Start scripting session with resource -dpkit resource script resource.json - -# Script resource from package -dpkit resource script --from-package datapackage.json --from-resource "users" - -# In the REPL session: -dpkit> resource.schema.fields.length -dpkit> resource.schema.fields[0].type -dpkit> resource.path -``` - -## Common Workflows - -### Creating Resource Metadata - -1. **Infer from data file:** - ```bash - dpkit resource infer data.csv --json > resource.json - ``` - -2. **Validate the generated resource:** - ```bash - dpkit resource validate resource.json - ``` - -3. **Explore the resource structure:** - ```bash - dpkit resource explore resource.json - ``` - -### Working with Package Resources - -```bash -# Explore all resources in a package interactively -dpkit resource validate --from-package datapackage.json - -# Infer metadata for specific resource -dpkit resource infer --from-package datapackage.json --from-resource "users" - -# Script specific resource from package -dpkit resource script --from-package datapackage.json --from-resource "transactions" -``` - -### Resource Analysis Workflow - -```bash -# Infer resource with custom options -dpkit resource infer data.csv \ - --delimiter ";" \ - --header-rows 2 \ - --sample-rows 1000 - -# Validate the inferred resource -dpkit resource validate resource.json - -# Explore interactively to verify structure -dpkit resource explore resource.json -``` - -### Remote Resource Handling - -```bash -# Work with remote resources -dpkit resource explore https://example.com/resource.json -dpkit resource validate https://example.com/resource.json -dpkit resource infer https://example.com/data.csv -``` - -## Resource Selection - -When working with resources from packages, you can either: - -1. **Specify explicitly:** - ```bash - dpkit resource explore --from-package datapackage.json --from-resource "users" - ``` - -2. **Interactive selection:** - ```bash - dpkit resource validate --from-package datapackage.json - # Will prompt to select from available resources - ``` - -## Output Formats - -All resource commands support multiple output formats: - -- **Interactive Display**: Default rich terminal interface showing resource structure -- **JSON**: Use `--json` flag for machine-readable output -- **Debug Mode**: Use `--debug` for detailed operation logs - -## Integration with Package Commands - -Resource commands work seamlessly with package commands: - -```bash -# Create package, then work with individual resources -dpkit package infer *.csv --json > datapackage.json -dpkit resource validate --from-package datapackage.json --from-resource "data" -dpkit resource explore --from-package datapackage.json --from-resource "users" -``` diff --git a/portal/terminal/content/docs/guides/schema.md b/portal/terminal/content/docs/guides/schema.md deleted file mode 100644 index 31df07e7..00000000 --- a/portal/terminal/content/docs/guides/schema.md +++ /dev/null @@ -1,374 +0,0 @@ ---- -title: Schema -sidebar: - order: 6 ---- - -Table Schema commands help you work with table schemas - metadata that describes the structure, types, and constraints of tabular data. These commands allow you to infer schema from data, validate schema definitions, and explore schema properties. - -## Available Commands - -### `dpkit schema infer` - -Infer a table schema from a table by analyzing its data and generating field definitions including types, constraints, and formats. - -```bash -dpkit schema infer -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Table Dialect Options:** -- `--delimiter`: Field delimiter character -- `--header`: Whether files have headers -- `--header-rows`: Number of header rows -- `--header-join`: Join character for multi-row headers -- `--comment-rows`: Number of comment rows to skip -- `--comment-char`: Comment character -- `--quote-char`: Quote character for fields -- `--double-quote`: Whether quotes are doubled for escaping -- `--escape-char`: Escape character -- `--null-sequence`: Sequence representing null values -- `--skip-initial-space`: Skip initial whitespace -- `--property`: JSON property path for nested data -- `--item-type`: Type of items in arrays -- `--item-keys`: Keys for object items -- `--sheet-number`: Excel sheet number -- `--sheet-name`: Excel sheet name -- `--table`: Database table name -- `--sample-bytes`: Bytes to sample for inference - -**Table Schema Options:** -- `--field-names`: Override field names -- `--field-types`: Override field types -- `--missing-values`: Values to treat as missing -- `--string-format`: String format specification -- `--decimal-char`: Decimal separator character -- `--group-char`: Thousands separator character -- `--bare-number`: Allow bare numbers -- `--true-values`: Values to treat as true -- `--false-values`: Values to treat as false -- `--datetime-format`: DateTime format string -- `--date-format`: Date format string -- `--time-format`: Time format string -- `--array-type`: Type of array elements -- `--list-delimiter`: List item delimiter -- `--list-item-type`: Type of list items -- `--geopoint-format`: Geopoint format specification -- `--geojson-format`: GeoJSON format specification -- `--sample-rows`: Rows to sample for inference -- `--confidence`: Confidence threshold for type inference -- `--comma-decimal`: Use comma as decimal separator -- `--month-first`: Parse dates with month first -- `--keep-strings`: Keep string types when possible - -**Examples:** -```bash -# Infer schema from CSV file -dpkit schema infer data.csv - -# Infer with custom delimiter and date format -dpkit schema infer data.csv --delimiter ";" --date-format "%d/%m/%Y" - -# Infer from remote file -dpkit schema infer https://example.com/data.csv - -# Infer from resource in package -dpkit schema infer --from-package datapackage.json --from-resource "users" - -# Export schema as JSON -dpkit schema infer data.csv --json > schema.json -``` - -### `dpkit schema convert` - -Convert table schemas between different formats, supporting bidirectional conversion between Table Schema and JSONSchema formats. - -```bash -dpkit schema convert -``` - -**Options:** -- `--format `: Source schema format (`schema`, `jsonschema`) -- `--to-format `: Target schema format (`schema`, `jsonschema`) -- `--to-path `: Output path for converted schema -- `-j, --json`: Output as JSON (automatically enabled when no --to-path) -- `-s, --silent`: Suppress all output except errors -- `-d, --debug`: Enable debug mode - -**Supported Formats:** -- **`schema`**: Data Package Table Schema format -- **`jsonschema`**: JSON Schema format - -**Examples:** -```bash -# Convert Table Schema to JSONSchema -dpkit schema convert schema.json --to-format jsonschema - -# Convert JSONSchema to Table Schema -dpkit schema convert schema.jsonschema.json --format jsonschema - -# Save converted schema to file -dpkit schema convert schema.json --to-format jsonschema --to-path converted.jsonschema.json - -# Convert from JSONSchema and save as Table Schema -dpkit schema convert input.jsonschema.json --format jsonschema --to-path output.schema.json -``` - -### `dpkit schema explore` - -Explore a table schema from a local or remote path to view its field definitions and constraints in an interactive format. - -```bash -dpkit schema explore -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Explore schema descriptor -dpkit schema explore schema.json - -# Explore remote schema -dpkit schema explore https://example.com/schema.json - -# Explore schema from package resource -dpkit schema explore --from-package datapackage.json --from-resource "users" - -# Export schema structure as JSON -dpkit schema explore schema.json --json -``` - -### `dpkit schema validate` - -Validate a table schema from a local or remote path against the Table Schema specification. - -```bash -dpkit schema validate -``` - -**Options:** -- `--from-package`: Path to package containing the resource -- `--from-resource`: Name of resource within package -- `--json`: Output validation results as JSON -- `--debug`: Enable debug mode -- `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) -- `-a, --all`: Skip selection prompts when all can be selected - -**Examples:** -```bash -# Validate schema descriptor -dpkit schema validate schema.json - -# Validate remote schema -dpkit schema validate https://example.com/schema.json - -# Validate schema from package resource -dpkit schema validate --from-package datapackage.json --from-resource "users" - -# Get validation results as JSON -dpkit schema validate schema.json --json - -# Interactive selection when no path provided -dpkit schema validate --from-package datapackage.json -``` - -### `dpkit schema script` - -Open an interactive scripting session with a loaded table schema. This provides a REPL environment where you can programmatically interact with the schema definition. - -```bash -dpkit schema script -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Available Variables:** -- `dpkit`: The dpkit library object -- `schema`: The loaded schema object - -**Examples:** -```bash -# Start scripting session with schema -dpkit schema script schema.json - -# Script schema from package resource -dpkit schema script --from-package datapackage.json --from-resource "users" - -# In the REPL session: -dpkit> schema.fields.length -dpkit> schema.fields[0].name -dpkit> schema.fields.filter(f => f.type === 'integer') -dpkit> schema.primaryKey -``` - -## Common Workflows - -### Creating Schema Definitions - -1. **Infer from data file:** - ```bash - dpkit schema infer data.csv --json > schema.json - ``` - -2. **Validate the generated schema:** - ```bash - dpkit schema validate schema.json - ``` - -3. **Explore the schema structure:** - ```bash - dpkit schema explore schema.json - ``` - -### Schema Format Conversion - -```bash -# Convert Table Schema to JSONSchema for JSON Schema validation tools -dpkit schema infer data.csv --json > table.schema.json -dpkit schema convert table.schema.json --to-format jsonschema --to-path api.jsonschema.json - -# Convert JSONSchema back to Table Schema for dpkit tools -dpkit schema convert api.jsonschema.json --format jsonschema --to-path converted.schema.json - -# Validate the round-trip conversion -dpkit schema validate converted.schema.json -``` - -### Schema Analysis and Refinement - -```bash -# Infer schema with high confidence threshold -dpkit schema infer data.csv --confidence 0.8 --sample-rows 10000 - -# Validate and explore for refinement -dpkit schema validate schema.json -dpkit schema explore schema.json - -# Script for custom analysis -dpkit schema script schema.json -``` - -### Working with Package Schemas - -```bash -# Validate all schemas in a package interactively -dpkit schema validate --from-package datapackage.json - -# Infer improved schema for specific resource -dpkit schema infer --from-package datapackage.json --from-resource "transactions" - -# Compare schemas using scripting -dpkit schema script --from-package datapackage.json --from-resource "users" -``` - -### Custom Type Inference - -```bash -# Configure specific data types and formats -dpkit schema infer data.csv \ - --datetime-format "%Y-%m-%d %H:%M:%S" \ - --true-values "Yes,True,1" \ - --false-values "No,False,0" \ - --decimal-char "," \ - --missing-values "NULL,N/A,," -``` - -### Remote Schema Handling - -```bash -# Work with remote schemas -dpkit schema explore https://example.com/schema.json -dpkit schema validate https://example.com/schema.json -dpkit schema infer https://example.com/data.csv -``` - -## Schema Field Types - -The schema inference supports various field types: - -- **Basic Types**: `string`, `integer`, `number`, `boolean` -- **Date/Time Types**: `date`, `datetime`, `time`, `year`, `yearmonth`, `duration` -- **Structured Types**: `array`, `object`, `list` -- **Geographic Types**: `geopoint`, `geojson` - -## Advanced Inference Options - -### Confidence Tuning -```bash -# High confidence for clean data -dpkit schema infer data.csv --confidence 0.9 - -# Lower confidence for messy data -dpkit schema infer data.csv --confidence 0.6 -``` - -### Sample Size Control -```bash -# Large sample for better inference -dpkit schema infer large_data.csv --sample-rows 50000 - -# Quick inference with small sample -dpkit schema infer data.csv --sample-rows 100 -``` - -### Format Specifications -```bash -# European date format -dpkit schema infer data.csv --date-format "%d.%m.%Y" - -# Custom boolean values -dpkit schema infer data.csv --true-values "Ja,Oui,Sí" --false-values "Nein,Non,No" -``` - -## Output Formats - -All schema commands support multiple output formats: - -- **Interactive Display**: Default rich terminal interface showing field definitions -- **JSON**: Use `--json` flag for machine-readable output -- **Debug Mode**: Use `--debug` for detailed operation logs - -## Schema Format Interoperability - -The `convert` command enables seamless integration with other schema ecosystems: - -```bash -# Use with JSON Schema validation libraries -dpkit schema infer data.csv --json > table.schema.json -dpkit schema convert table.schema.json --to-format jsonschema --to-path validation.jsonschema.json - -# Import existing JSONSchema into dpkit workflow -dpkit schema convert external.jsonschema.json --format jsonschema --to-path dpkit.schema.json -dpkit table validate data.csv --schema dpkit.schema.json - -# Cross-platform schema sharing -dpkit schema convert schema.json --to-format jsonschema --to-path api-spec.jsonschema.json -``` - -## Integration with Other Commands - -Schema commands work seamlessly with other dpkit commands: - -```bash -# Create schema, then use it for validation -dpkit schema infer data.csv --json > schema.json -dpkit table validate data.csv --schema schema.json - -# Work within package context -dpkit package infer *.csv --json > datapackage.json -dpkit schema validate --from-package datapackage.json --from-resource "data" -``` diff --git a/portal/terminal/content/docs/guides/table.md b/portal/terminal/content/docs/guides/table.md deleted file mode 100644 index 138bebec..00000000 --- a/portal/terminal/content/docs/guides/table.md +++ /dev/null @@ -1,360 +0,0 @@ ---- -title: Table -sidebar: - order: 7 ---- - -Table commands help you work directly with tabular data files. These commands allow you to explore, validate, convert, and analyze data tables with support for various formats including CSV, Excel, JSON, and more. - -## Available Commands - -### `dpkit table convert` - -Convert a table from one format to another with support for various input and output formats. - -```bash -dpkit table convert -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-s, --silent`: Suppress all output except errors -- `-d, --debug`: Enable debug mode - -**Supported Formats:** -- **Input**: CSV, TSV, Excel (.xlsx, .xls), JSON, Parquet, Arrow, ODS -- **Output**: CSV, TSV, Excel (.xlsx), JSON, Parquet, Arrow, ODS - -**Examples:** -```bash -# Convert CSV to Excel -dpkit table convert data.csv data.xlsx - -# Convert Excel to JSON -dpkit table convert data.xlsx data.json - -# Convert from package resource -dpkit table convert --from-package datapackage.json --from-resource "users" users.xlsx - -# Convert Parquet to CSV -dpkit table convert data.parquet data.csv -``` - -### `dpkit table describe` - -Generate statistical descriptions and summaries of table data including column statistics, data types, and quality metrics. - -```bash -dpkit table describe -``` - -**Options:** -- `-p, --from-package`: Path to package containing the resource -- `-r, --from-resource`: Name of resource within package -- `-j, --json`: Output as JSON -- `-d, --debug`: Enable debug mode - -**Examples:** -```bash -# Describe CSV file -dpkit table describe data.csv - -# Describe with JSON output -dpkit table describe data.csv --json - -# Describe resource from package -dpkit table describe --from-package datapackage.json --from-resource "sales" - -# Describe remote table -dpkit table describe https://example.com/data.csv -``` - -### `dpkit table explore` - -Explore a table interactively, viewing data samples, column information, and basic statistics in a rich terminal interface. - -```bash -dpkit table explore -``` - -**Options:** -- `--from-package`: Path to package containing the resource -- `--from-resource`: Name of resource within package -- `--schema`: Path to schema file for validation during exploration -- `--dialect`: Path to dialect file for parsing configuration -- `--json`: Output as JSON -- `--debug`: Enable debug mode - -**Examples:** -```bash -# Explore CSV file -dpkit table explore data.csv - -# Explore with schema validation -dpkit table explore data.csv --schema schema.json - -# Explore with custom dialect -dpkit table explore data.csv --dialect dialect.json - -# Explore resource from package -dpkit table explore --from-package datapackage.json --from-resource "users" - -# Explore remote table -dpkit table explore https://example.com/data.csv -``` - -### `dpkit table validate` - -Validate table data against a schema, checking data types, constraints, and data quality issues. - -```bash -dpkit table validate -``` - -**Options:** -- `--from-package`: Path to package containing the resource -- `--from-resource`: Name of resource within package -- `--schema`: Path to schema file for validation -- `--dialect`: Path to dialect file for parsing -- `--json`: Output validation results as JSON -- `--debug`: Enable debug mode -- `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) -- `-a, --all`: Skip selection prompts when all can be selected - -**Examples:** -```bash -# Validate with schema -dpkit table validate data.csv --schema schema.json - -# Validate with custom dialect and schema -dpkit table validate data.csv --dialect dialect.json --schema schema.json - -# Validate resource from package -dpkit table validate --from-package datapackage.json --from-resource "users" - -# Get validation results as JSON -dpkit table validate data.csv --schema schema.json --json - -# Validate remote table -dpkit table validate https://example.com/data.csv --schema https://example.com/schema.json -``` - -### `dpkit table script` - -Open an interactive scripting session with a loaded table. This provides a REPL environment where you can programmatically analyze and manipulate table data. - -```bash -dpkit table script -``` - -**Options:** -- `--from-package`: Path to package containing the resource -- `--from-resource`: Name of resource within package -- `--schema`: Path to schema file -- `--dialect`: Path to dialect file -- `--json`: Output as JSON -- `--debug`: Enable debug mode - -**Available Variables:** -- `dpkit`: The dpkit library object -- `table`: The loaded table object - -**Examples:** -```bash -# Start scripting session with table -dpkit table script data.csv - -# Script with schema and dialect -dpkit table script data.csv --schema schema.json --dialect dialect.json - -# Script resource from package -dpkit table script --from-package datapackage.json --from-resource "sales" - -# In the REPL session: -dpkit> table.rows.length -dpkit> table.columns -dpkit> table.rows[0] -dpkit> table.schema.fields.map(f => f.name) -``` - -## Common Workflows - -### Data Exploration and Analysis - -1. **Quick exploration:** - ```bash - dpkit table explore data.csv - ``` - -2. **Generate statistical summary:** - ```bash - dpkit table describe data.csv - ``` - -3. **Interactive analysis:** - ```bash - dpkit table script data.csv - ``` - -### Data Validation Workflow - -```bash -# Infer schema from table -dpkit schema infer data.csv --json > schema.json - -# Validate table against schema -dpkit table validate data.csv --schema schema.json - -# Explore validation issues -dpkit table explore data.csv --schema schema.json -``` - -### Format Conversion Pipeline - -```bash -# Convert Excel to CSV for processing -dpkit table convert input.xlsx temp.csv - -# Process and validate -dpkit table validate temp.csv --schema schema.json - -# Convert to final format -dpkit table convert temp.csv output.json -``` - -### Package Integration Workflow - -```bash -# Create package with tables -dpkit package infer *.csv --json > datapackage.json - -# Validate individual tables -dpkit table validate --from-package datapackage.json --from-resource "users" - -# Describe tables for documentation -dpkit table describe --from-package datapackage.json --from-resource "sales" --json -``` - -### Remote Table Processing - -```bash -# Explore remote table -dpkit table explore https://example.com/data.csv - -# Convert remote to local format -dpkit table convert https://example.com/data.csv local_data.xlsx - -# Validate remote table with local schema -dpkit table validate https://example.com/data.csv --schema local_schema.json -``` - -## Supported File Formats - -### Input Formats -- **CSV/TSV**: Comma/tab-separated values with dialect support -- **Excel**: .xlsx and .xls files with sheet selection -- **JSON**: Various JSON table formats -- **Parquet**: Apache Parquet columnar format -- **Arrow**: Apache Arrow format -- **ODS**: OpenDocument Spreadsheet format - -### Output Formats -- **CSV**: Standard comma-separated values -- **Excel**: .xlsx format -- **JSON**: Various JSON structures -- **Parquet**: Efficient columnar storage -- **Arrow**: High-performance analytics -- **ODS**: Cross-platform spreadsheets - -## Advanced Options - -### Schema-Aware Operations -```bash -# Validate with type checking -dpkit table validate data.csv --schema schema.json - -# Explore with schema hints -dpkit table explore data.csv --schema schema.json -``` - -### Custom Parsing -```bash -# Use custom dialect for parsing -dpkit table explore data.csv --dialect custom_dialect.json - -# Convert with parsing options -dpkit table convert complex_data.csv output.xlsx --dialect dialect.json -``` - -### Statistical Analysis -```bash -# Generate comprehensive statistics -dpkit table describe large_dataset.csv --json > stats.json - -# Interactive statistical exploration -dpkit table script data.csv -# In REPL: analyze column distributions, correlations, etc. -``` - -## Error Handling and Debugging - -### Validation Issues -```bash -# Get detailed validation report -dpkit table validate data.csv --schema schema.json --json - -# Interactive error exploration (don't quit on errors) -dpkit table validate data.csv --schema schema.json -``` - -### Parsing Problems -```bash -# Debug parsing issues -dpkit table explore problematic.csv --debug - -# Infer and test dialect -dpkit dialect infer problematic.csv --json > dialect.json -dpkit table explore problematic.csv --dialect dialect.json -``` - -### Performance Optimization -```bash -# For large files, use sampling -dpkit table describe huge_file.csv --sample-rows 10000 - -# Convert to efficient formats for repeated analysis -dpkit table convert large_data.csv data.parquet -``` - -## Output Formats - -Table commands support multiple output formats: - -- **Interactive Display**: Rich terminal interface with data previews -- **JSON**: Machine-readable structured output -- **Statistical Reports**: Comprehensive data summaries -- **Debug Mode**: Detailed operation logs and diagnostics - -## Integration Examples - -### With Package Commands -```bash -# Create and validate package -dpkit package infer *.csv --json > datapackage.json -dpkit table validate --from-package datapackage.json --from-resource "main" -``` - -### With Schema Commands -```bash -# Infer schema and validate table -dpkit schema infer data.csv --json > schema.json -dpkit table validate data.csv --schema schema.json -``` - -### With Dialect Commands -```bash -# Infer dialect and use for table operations -dpkit dialect infer data.csv --json > dialect.json -dpkit table explore data.csv --dialect dialect.json -``` diff --git a/portal/terminal/content/docs/index.md b/portal/terminal/content/docs/index.md deleted file mode 100644 index 158259af..00000000 --- a/portal/terminal/content/docs/index.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: dpkit Terminal -sidebar: - order: 1 - label: Getting Started ---- - -This guide will help you get started with dpkit in Terminal. If you are new to the core framework's tecnhologies, please take a look at the [Data Package standard](https://datapackage.org/) and [Polars DataFrames](https://pola.rs/) documentation. - -## Prerequisites - -Supported operating systems: - -- **Linux** (x64/arm64) -- **macOS** (x64/arm64) -- **Windows** (x64) - -## Installation - -dpkit is a command-line tool that can be downloaded as a binary executable or installed using Node.js and its package managers. - -### Binary - -You can download the latest binary from the [releases page](https://github.com/datisthq/dpkit/releases) or use the following command (for POSIX-compatible shells including Git for Windows): - -```sh -curl -fsSL https://dpkit.app/install.sh | sh -``` - -After downloading, you can verify the binary using the following command: - -```sh -./dpkit --version -``` - -We recommend adding the binary to your PATH environment variable to make it easier to use. - -### Node - -You can install the CLI using this command: - -```bash -npm install -g dpkit -``` - -After that you can use the CLI binary: - -```bash -dpkit --version -``` diff --git a/portal/terminal/content/docs/overview/contributing.md b/portal/terminal/content/docs/overview/contributing.md deleted file mode 100644 index 81fee01b..00000000 --- a/portal/terminal/content/docs/overview/contributing.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: Contributing -sidebar: - order: 2 ---- -Thank you for your interest in contributing to dpkit! This document provides guidelines and instructions for contributing to this project. - -## Project Overview - -Project is a monorepo with the following packages: - -- `@dpkit/metadata`: Core metadata functionality -- `@dpkit/dataset`: File-related functionality -- `@dpkit/table`: Table-related functionality -- `@dpkit/`: Domain-specific functionality -- `@dpkit/library`: All-in-one package that re-exports all functionality -- `@dpkit/terminal`: Terminal interface for running tasks -- `@dpkit/website`: Website-related functionality -- `dpkit`: Meta-package that re-exports the underlying functionality - -## Development Environment - -### Prerequisites - -> [!TIP] -> For CLI compilation and development, we recommend using [Bun](https://bun.sh/). For example, to run CLI in dev `bun cli/main.ts` - -- **Node.js**: v24.0.0 or higher -- **PNPM**: v10.0.0 or higher - -### Setup - -1. Clone the repository - ```bash - git clone https://github.com/yourusername/dpkit.git dpkit - cd dpkit - ``` - -2. Install dependencies - ```bash - pnpm install - ``` -## Development Workflow - -### Code Style and Quality - -We use Biome for linting and formatting, and TypeScript for type checking: - -- **Lint**: Check for code issues - ```bash - pnpm run lint - ``` - -- **Format**: Auto-fix formatting issues - ```bash - pnpm run format - ``` - -- **Type Check**: Verify TypeScript types - ```bash - pnpm run type - ``` - -- **Comprehensive Check**: Run lint and type checking - ```bash - pnpm run check - ``` - -### Testing - -Tests are located in `__spec__` directories and use Vitest: - -- **Run All Tests**: (includes linting and type checking) - ```bash - pnpm test - ``` - -- **Run Tests Only**: (without linting/type checking) - ```bash - pnpm run spec - ``` - -- **Run a Specific Test**: - ```bash - pnpm exec vitest run core/actions/__spec__/findTask.ts - ``` - -### Dependencies - -Update all dependencies to their latest versions: - -```bash -pnpm run bump -``` -## Code Style Guidelines - -- Use TypeScript with strict type checking -- Follow ES modules pattern (`import`/`export`) -- Tests should be placed in `__spec__` directories -- Use semicolons as needed (not required everywhere) -- Use arrow function parentheses as needed (omitted for single parameters) - -## Making Changes to the Meta-Package - -When adding new functionality: - -1. Add it to the appropriate package first (`core`, `cli`, or `cron`) -2. Ensure it's properly exported from that package -3. No additional work is needed for the meta-package as it automatically re-exports everything - -## Submitting Changes - -1. Create a feature branch (`git checkout -b feature/your-feature`) -2. Make your changes with appropriate tests -3. Ensure the code passes all checks: `pnpm test` -4. Commit your changes with a descriptive message -5. Submit a pull request - -## License - -By contributing to dpkit, you agree that your contributions will be licensed under the project's license. - -Thank you for your contribution! diff --git a/portal/terminal/content/docs/overview/funding.md b/portal/terminal/content/docs/overview/funding.md deleted file mode 100644 index fdea7321..00000000 --- a/portal/terminal/content/docs/overview/funding.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Funding ---- - -This project is funded through [NGI Zero Commons Fund](https://nlnet.nl/commonsfund/), a fund established by [NLnet](https://nlnet.nl) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu) program. Learn more at the [NLnet project page](https://nlnet.nl/project/DataPackage-TS/). - - diff --git a/portal/terminal/public/favicon.png b/portal/terminal/public/favicon.png deleted file mode 100644 index 384b92dd..00000000 Binary files a/portal/terminal/public/favicon.png and /dev/null differ diff --git a/portal/terminal/styles/custom.css b/portal/terminal/styles/custom.css deleted file mode 100644 index eed9a576..00000000 --- a/portal/terminal/styles/custom.css +++ /dev/null @@ -1,52 +0,0 @@ -:root { - --purple-hsl: 209, 60%, 60%; - --overlay-blurple: hsla(var(--purple-hsl), 0.2); - --scrollbar-color: #ddd; -} - -:root[data-theme="light"] { - --purple-hsl: 209, 85%, 65%; - --sl-color-text-accent: #228be6; - --sl-color-banner-bg: #228be6; -} - -:root[data-theme="dark"] { - --sl-color-text-accent: #228be6; - --sl-color-banner-bg: #228be6; -} - -/* logo */ - -.site-title { - gap: 5px; - font-size: 22px; - font-weight: bold; - text-decoration: underline; -} - -.site-title img { - width: 32px; - height: 32px; -} - -html[data-theme="light"] .site-title { - color: black; -} - -html[data-theme="dark"] .site-title { - color: white; -} - -/* scrollbar */ - -* { - scrollbar-color: var(--scrollbar-color) transparent; - scrollbar-gutter: stable; -} - -/* scroll-to-top */ - -#scroll-to-top-button { - accent-color: var(--sl-color-text-accent); - background-color: var(--sl-color-text-accent); -} diff --git a/portal/terminal/tsconfig.json b/portal/terminal/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/portal/terminal/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/portal/terminal/wrangler.jsonc b/portal/terminal/wrangler.jsonc deleted file mode 100644 index d7cc4a70..00000000 --- a/portal/terminal/wrangler.jsonc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "dpkit-terminal", - // "preview_urls": true, - "compatibility_date": "2025-08-15", - "assets": { - "directory": "./build" - } -} diff --git a/portal/tsconfig.json b/portal/tsconfig.json deleted file mode 100644 index 8ae05755..00000000 --- a/portal/tsconfig.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "extends": "astro/tsconfigs/strict", - "include": [".astro/types.d.ts", "**/*"], - "exclude": ["**/build/*"] -} diff --git a/portal/typescript/assets/dpkit-logo.svg b/portal/typescript/assets/dpkit-logo.svg deleted file mode 100644 index 10c9c736..00000000 --- a/portal/typescript/assets/dpkit-logo.svg +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - diff --git a/portal/typescript/assets/styles.css b/portal/typescript/assets/styles.css deleted file mode 100644 index eb698350..00000000 --- a/portal/typescript/assets/styles.css +++ /dev/null @@ -1,46 +0,0 @@ -:root { - --purple-hsl: 209, 60%, 60%; - --overlay-blurple: hsla(var(--purple-hsl), 0.2); - --scrollbar-color: #ddd; -} - -:root[data-theme="light"] { - --purple-hsl: 209, 85%, 65%; - --sl-color-text-accent: #228be6; - --sl-color-banner-bg: #228be6; - /* --sl-color-bg-inline-code: #228be6; */ -} - -:root[data-theme="dark"] { - --sl-color-text-accent: #228be6; - --sl-color-banner-bg: #228be6; - /* --sl-color-bg-inline-code: #228be6; */ -} - -/* Scrollbar */ - -* { - scrollbar-color: var(--scrollbar-color) transparent; - scrollbar-gutter: stable; -} - -/* Markdown heading links. */ - -.sl-markdown-content :is(h1, h2, h3, h4, h5, h6) > a { - color: var(--sl-color-white); - text-decoration: none; - &:hover { - text-decoration: underline; - } -} - -article.card { - border-radius: 15px; -} - -/* Scroll-to-top button */ - -#scroll-to-top-button { - accent-color: var(--sl-color-text-accent); - background-color: var(--sl-color-text-accent); -} diff --git a/portal/typescript/astro.config.ts b/portal/typescript/astro.config.ts deleted file mode 100644 index 7107a22a..00000000 --- a/portal/typescript/astro.config.ts +++ /dev/null @@ -1,131 +0,0 @@ -import starlight from "@astrojs/starlight" -import { defineConfig } from "astro/config" -import starlightChangelogs, { - makeChangelogsSidebarLinks, -} from "starlight-changelogs" -import starlightGitHubAlerts from "starlight-github-alerts" -import starlightScrollToTop from "starlight-scroll-to-top" -import starlightTypeDoc from "starlight-typedoc" - -const PACKAGES = { - dpkit: "../dpkit", - "@dpkit/audio": "../../audio", - "@dpkit/database": "../../database", - "@dpkit/dataset": "../../dataset", - "@dpkit/image": "../../image", - "@dpkit/library": "../../library", - "@dpkit/metadata": "../../metadata", - "@dpkit/table": "../../table", - "@dpkit/terminal": "../../terminal", - "@dpkit/video": "../../video", -} - -export default defineConfig({ - site: "https://typescript.dpkit.app", - srcDir: ".", - outDir: "build", - integrations: [ - starlight({ - title: "dpkit", - description: - "dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub", - customCss: ["/styles/custom.css"], - components: { - SocialIcons: "./components/SocialIcons.astro", - }, - logo: { - src: "/assets/dpkit-logo.svg", - alt: "dpkit Logo", - }, - social: [ - { - icon: "github", - label: "GitHub", - href: "https://github.com/datisthq/dpkit", - }, - ], - favicon: "favicon.png", - editLink: { - baseUrl: "https://github.com/datisthq/dpkit/edit/main/", - }, - lastUpdated: true, - tableOfContents: { minHeadingLevel: 2, maxHeadingLevel: 5 }, - expressiveCode: { - themes: ["starlight-dark", "starlight-light"], - }, - plugins: [ - starlightGitHubAlerts(), - starlightScrollToTop(), - starlightChangelogs(), - starlightTypeDoc({ - entryPoints: generatePackageEntrypoints(), - tsconfig: "../tsconfig.json", - typeDoc: { entryPointStrategy: "packages", router: "structure" }, - output: "reference", - sidebar: { - label: "API Reference", - collapsed: true, - }, - }), - ], - sidebar: [ - { - label: "Overview", - items: [ - { label: "Getting Started", slug: "index" }, - { label: "Contributing", slug: "overview/contributing" }, - { label: "Funding", slug: "overview/funding" }, - ], - }, - { label: "Guides", autogenerate: { directory: "guides" } }, - { - label: "API Reference", - collapsed: true, - items: generatePackageSidebars(), - }, - { - label: "Changelog", - collapsed: true, - items: makeChangelogsSidebarLinks([ - { - type: "recent", - base: "changelog", - count: 10, - }, - ]), - }, - ], - head: [ - { - tag: "script", - attrs: { - src: "https://plausible.io/js/script.js", - "data-domain": "dpkit.app", - defer: true, - }, - }, - ], - }), - ], -}) - -function generatePackageEntrypoints() { - return Object.values(PACKAGES) -} - -function generatePackageSidebars() { - return Object.entries(PACKAGES).map(([name, _path]) => - generatePackageSidebar({ name }), - ) -} - -function generatePackageSidebar(props: { name: string }) { - const name = props.name - const slug = name.replace("@", "_") - - return { - label: name, - collapsed: true, - autogenerate: { directory: `reference/${slug}` }, - } -} diff --git a/portal/typescript/components/SocialIcons.astro b/portal/typescript/components/SocialIcons.astro deleted file mode 100644 index 0d972681..00000000 --- a/portal/typescript/components/SocialIcons.astro +++ /dev/null @@ -1,35 +0,0 @@ ---- -import Default from "@astrojs/starlight/components/SocialIcons.astro" -import type { Props } from "@astrojs/starlight/props" ---- - - - Web - - - - Terminal - - - - -TypeScript - - - - - diff --git a/portal/typescript/content.config.ts b/portal/typescript/content.config.ts deleted file mode 100644 index 4a2747a6..00000000 --- a/portal/typescript/content.config.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { defineCollection } from "astro:content" -import { docsLoader } from "@astrojs/starlight/loaders" -import { docsSchema } from "@astrojs/starlight/schema" -import { changelogsLoader } from "starlight-changelogs/loader" - -export const collections = { - docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), - changelogs: defineCollection({ - loader: changelogsLoader([ - { - base: "changelog", - provider: "github", - owner: "datisthq", - repo: "dpkit", - }, - ]), - }), -} diff --git a/portal/typescript/content/docs/guides/arrow.md b/portal/typescript/content/docs/guides/arrow.md deleted file mode 100644 index 0f0b6515..00000000 --- a/portal/typescript/content/docs/guides/arrow.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Working with Arrow -sidebar: - label: Arrow - order: 3 ---- - -The `@dpkit/arrow` package provides efficient support for loading and saving data in Apache Arrow format. It uses Polars DataFrames for high-performance columnar data processing. - -## Installation - -```bash -npm install @dpkit/arrow -``` - -## Basic Usage - -> [!TIP] -> You can use `loadTable` and `saveTable` from `dpkit` instead of `@dpkit/arrow` to load and save ARROW files if the framework can infer that files are in the `arrow/feather` format. - -### Loading Data - -```typescript -import { loadArrowTable } from "@dpkit/arrow" - -// Load from local file -const table = await loadArrowTable({ path: "data.arrow" }) - -// Load multiple files (concatenated) -const table = await loadArrowTable({ - path: ["file1.arrow", "file2.arrow"] -}) -``` - -### Saving Data - -```typescript -import { saveArrowTable } from "@dpkit/arrow" - -// Save as Arrow format -await saveArrowTable(table, { path: "output.arrow" }) -``` diff --git a/portal/typescript/content/docs/guides/assets/jupyter.png b/portal/typescript/content/docs/guides/assets/jupyter.png deleted file mode 100644 index 5fd6bed4..00000000 Binary files a/portal/typescript/content/docs/guides/assets/jupyter.png and /dev/null differ diff --git a/portal/typescript/content/docs/guides/cloud.md b/portal/typescript/content/docs/guides/cloud.md deleted file mode 100644 index 659b6373..00000000 --- a/portal/typescript/content/docs/guides/cloud.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: Self-Hosting dpkit Cloud -sidebar: - label: dpkit Cloud - order: 11 ---- - -This document provides a brief overview of self-hosting [dpkit Cloud](https://cloud.dpkit.dev) on your own infrastructure. - -> [!WARNING] -> Currently, this setup below is blocked by [this CloudFlare issue](https://github.com/cloudflare/containers/issues/101). - -> [!TIP] -> dpkit Cloud is built on top of [Cloudflare Workers](https://workers.cloudflare.com/) but it is possible to use any other runtime that fits your needs e.g. Digital Ocean Apps or Docker. - -## Deployment - -1. Fork the [dpkit repository](https://github.com/datisthq/dpkit) to your GitHub account. -2. Clone the forked repository to your local machine. -3. Update the `wrangler.jsonc` with you application name e.g. `dpkit-cloud-custom`. -4. Setup a new CloudFlare worker with the following configuration: - - Name: e.g. `dpkit-cloud-custom` (should be the same as in `wrangler.jsonc`) - - Github repository: point to your forked repository - - Build command: `pnpm build && pnpm -F cloud build` - - Deploy command: `cd cloud && pnpm wrangler deploy` - - Branch deploy command: `cd cloud && pnpm wrangler versions upload` - - Variables ans secrets: `NODE_VERSION=24` -5. Push a new commit to your forked repository to trigger the build and deployment. - -## Usage - -Find a `workder.dev` subdomain in your CloudFlare dashboard and visit it in your browser or add a custom domain if it is desired. Now, you can use your custom dpkit Cloud the same way you use [dpkit Cloud](https://cloud.dpkit.dev). - -## Support - -We happy to help with a deployment of any complexity. Please [reach out to us to get a quote](https://www.linkedin.com/in/evgeny-karev/). diff --git a/portal/typescript/content/docs/guides/csv.md b/portal/typescript/content/docs/guides/csv.md deleted file mode 100644 index 0f85c4df..00000000 --- a/portal/typescript/content/docs/guides/csv.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -title: Working with CSV -sidebar: - label: CSV - order: 1 ---- -Comprehensive CSV and TSV file handling with automatic format detection, advanced header processing, and high-performance data operations. - -## Introduction - -> [!TIP] -> You can use `loadTable` and `saveTable` from `dpkit` instead of `@dpkit/csv` to load and save CSV files if the framework can infer that files are in the `csv/tsv` format. - -The CSV plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - -- `loadCsvTable` -- `saveCsvTable` -- `inferCsvDialect` - -For example: - -```typescript -import { loadCsvTable } from "@dpkit/csv" - -const table = await loadCsvTable({path: "table.csv"}) -// the field types will be automatically inferred -// or you can provide a Table Schema -``` - -## Basic Usage - -### Loading CSV Files - -```typescript -import { loadCsvTable } from "@dpkit/csv" - -// Load a simple CSV file -const table = await loadCsvTable({ path: "data.csv" }) - -// Load with custom dialect -const table = await loadCsvTable({ - path: "data.csv", - dialect: { - delimiter: ";", - header: true, - skipInitialSpace: true - } -}) - -// Load multiple CSV files (concatenated) -const table = await loadCsvTable({ - path: ["part1.csv", "part2.csv", "part3.csv"] -}) -``` - -### Saving CSV Files - -```typescript -import { saveCsvTable } from "@dpkit/csv" - -// Save with default options -await saveCsvTable(table, { path: "output.csv" }) - -// Save with custom dialect -await saveCsvTable(table, { - path: "output.csv", - dialect: { - delimiter: "\t", - quoteChar: "'" - } -}) -``` - -### Dialect Detection - -```typescript -import { inferCsvDialect } from "@dpkit/csv" - -// Automatically detect CSV format -const dialect = await inferCsvDialect({ path: "unknown-dialect.csv" }) -console.log(dialect) // { delimiter: ",", header: true, quoteChar: '"' } - -// Use detected dialect to load -const table = await loadCsvTable({ - path: "unknown-dialect.csv", - dialect -}) -``` - -## Advanced Features - -### Multi-Header Row Processing - -```typescript -// CSV with multiple header rows: -// Year,2023,2023,2024,2024 -// Quarter,Q1,Q2,Q1,Q2 -// Revenue,100,120,110,130 - -const table = await loadCsvTable({ - path: "multi-header.csv", - dialect: { - headerRows: [1, 2], - headerJoin: "_" - } -}) -// Resulting columns: ["Year_Quarter", "2023_Q1", "2023_Q2", "2024_Q1", "2024_Q2"] -``` - -### Comment Row Handling - -```typescript -// CSV with comment rows: -// # This is a comment -// # Generated on 2024-01-01 -// Name,Age,City -// John,25,NYC - -const table = await loadCsvTable({ - path: "with-comments.csv", - dialect: { - commentRows: [1, 2], - header: true - } -}) -``` - -### Remote File Loading - -```typescript -// Load from URL -const table = await loadCsvTable({ - path: "https://example.com/data.csv" -}) - -// Load multiple remote files -const table = await loadCsvTable({ - path: [ - "https://api.example.com/data-2023.csv", - "https://api.example.com/data-2024.csv" - ] -}) -``` diff --git a/portal/typescript/content/docs/guides/database.md b/portal/typescript/content/docs/guides/database.md deleted file mode 100644 index 4b5b5005..00000000 --- a/portal/typescript/content/docs/guides/database.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Working with Database -sidebar: - label: Database - order: 5 ---- -Database connectivity and operations with support for SQLite, PostgreSQL, and MySQL through Kysely query builder integration. - -## Introduction - -> [!TIP] -> You can use `loadTable` and `saveTable` from `dpkit` instead of `@dpkit/database` to load and save database tables if the framework can infer the database connection format. - -The Database plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - -- `loadDatabaseTable` -- `saveDatabaseTable` -- `inferDatabaseSchema` -- `loadPackageFromDatabase` - -For example: - -```typescript -import { loadDatabaseTable } from "@dpkit/database" - -const table = await loadDatabaseTable({ - path: "sqlite://database.db", - dialect: { table: "users" } -}) -// field types will be automatically inferred from database schema -// or you can provide a Table Schema -``` - -> [!TIP] -> The ouput of `loadDatabaseTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. - -## Supported Databases - -The plugin supports three database types: - -- **SQLite** - File-based database (`sqlite://path/to/file.db`) -- **PostgreSQL** - Network database (`postgresql://user:pass@host:port/db`) -- **MySQL** - Network database (`mysql://user:pass@host:port/db`) - -## Connection Formats - -Database connections are specified using standard connection strings: - -```typescript -// SQLite -const sqliteTable = await loadDatabaseTable({ - path: "sqlite://data.db", - dialect: { table: "products" } -}) - -// PostgreSQL -const pgTable = await loadDatabaseTable({ - path: "postgresql://user:password@localhost:5432/mydb", - dialect: { table: "orders" } -}) - -// MySQL -const mysqlTable = await loadDatabaseTable({ - path: "mysql://user:password@localhost:3306/mydb", - dialect: { table: "customers" } -}) -``` - -## Schema Inference - -The database adapter automatically infers Table Schema from database table definitions: - -```typescript -import { inferDatabaseSchema } from "@dpkit/database" - -const schema = await inferDatabaseSchema({ - path: "sqlite://shop.db", - dialect: { table: "products" } -}) -// Returns a Table Schema with field types matching database columns -``` - -## Package Loading - -Load entire Data Packages from databases: - -```typescript -import { loadPackageFromDatabase } from "@dpkit/database" - -const package = await loadPackageFromDatabase("sqlite://catalog.db") -// Loads all tables as package resources -``` - -## Database Adapters - -The plugin uses database-specific adapters built on Kysely: - -- **BaseAdapter** - Common functionality for all databases -- **SqliteAdapter** - SQLite-specific operations using better-sqlite3 -- **PostgreSQLAdapter** - PostgreSQL operations using pg driver -- **MySQLAdapter** - MySQL operations using mysql2 driver - -Each adapter handles: -- Type normalization between database and Table Schema types -- Connection management with LRU caching -- Database-specific SQL dialect handling diff --git a/portal/typescript/content/docs/guides/inline.md b/portal/typescript/content/docs/guides/inline.md deleted file mode 100644 index 30ffceb5..00000000 --- a/portal/typescript/content/docs/guides/inline.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: Working with inline data -sidebar: - label: Inline Data - order: 5 ---- - -Dpkit provides a package for reading inline data tables embedded directly in data package resources. - -## Array Format Data - -```typescript -import { readInlineTable } from "@dpkit/inline" - -const resource = { - name: "languages", - type: "table", - data: [ - ["id", "name"], - [1, "english"], - [2, "中文"] - ] -} - -const table = await readInlineTable(resource) -``` - -## Object Format Data - -```typescript -const resource = { - name: "languages", - type: "table", - data: [ - { id: 1, name: "english" }, - { id: 2, name: "中文" } - ] -} - -const table = await readInlineTable(resource) -``` - -## With Processing Based on Schema - -```typescript -const resource = { - name: "languages", - type: "table", - data: [ - ["id", "name"], - [1, "english"], - [2, "中文"] - ], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" } - ] - } -} - -const table = await readInlineTable(resource) -``` - -## Inline Resource Validation - -```typescript -import { validateInlineTable } from "@dpkit/inline" - -const resource = { - name: "languages", - type: "table", - data: [ - ["id", "name"], - [1, "english"], - [2, "中文"] - ], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "integer" } - ] - } -} - -const {valid, errors} = await validateInlineTable(resource) -//{ -// type: "cell/type", -// fieldName: "name", -// rowNumber: 1, -// cell: "english", -//} -//{ -// type: "cell/type", -// fieldName: "name", -// rowNumber: 2, -// cell: "中文", -//} -``` - diff --git a/portal/typescript/content/docs/guides/json.md b/portal/typescript/content/docs/guides/json.md deleted file mode 100644 index d47977e8..00000000 --- a/portal/typescript/content/docs/guides/json.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: Working with JSON -sidebar: - label: JSON - order: 2 ---- - -The `@dpkit/json` package provides comprehensive support for loading and saving data in JSON and JSONL (JSON Lines) formats. It leverages Polars DataFrames for efficient data processing and supports flexible data transformations through dialect configurations. - -## Installation - -The JSON package is part of dpkit's modular architecture: - -```bash -npm install @dpkit/json -``` - -## Basic Usage - -> [!TIP] -> You can use `loadTable` and `saveTable` from `dpkit` instead of `@dpkit/json` to load and save JSON files if the framework can infer that files are in the `json/jsonl` format. - -### Loading JSON Data - -```typescript -import { loadJsonTable } from "@dpkit/json" - -// Load from local file -const table = await loadJsonTable({ path: "data.json" }) - -// Load from remote URL -const table = await loadJsonTable({ - path: "https://example.com/data.json" -}) - -// Load multiple files (concatenated) -const table = await loadJsonTable({ - path: ["file1.json", "file2.json"] -}) -``` - -### Loading JSONL Data - -```typescript -import { loadJsonTable } from "@dpkit/json" - -// Load JSONL (JSON Lines) format -const table = await loadJsonTable({ path: "data.jsonl", format: 'jsonl' }) -``` - -### Saving Data - -```typescript -import { saveJsonTable } from "@dpkit/json" - -// Save as JSON -await saveJsonTable(table, { path: "output.json" }) - -// Save as JSONL -await saveJsonTable(table, { path: "output.jsonl", format: 'jsonl' }) -``` - -## Data Formats - -The package supports two main JSON formats: - -### JSON Format -Standard JSON arrays of objects: -```json -[ - {"id": 1, "name": "Alice"}, - {"id": 2, "name": "Bob"} -] -``` - -### JSONL Format -Newline-delimited JSON objects: -```jsonl -{"id": 1, "name": "Alice"} -{"id": 2, "name": "Bob"} -``` - -## Dialect Support - -Dialects provide flexible data transformation capabilities: - -### Property Extraction - -Extract data from nested objects using the `property` option: - -```typescript -// Input: {"users": [{"id": 1, "name": "Alice"}]} -const table = await loadJsonTable({ - path: "data.json", - dialect: { property: "users" } -}) -``` - -### Item Keys Filtering - -Select specific fields using `itemKeys`: - -```typescript -// Only load 'name' field from each record -const table = await loadJsonTable({ - path: "data.json", - dialect: { itemKeys: ["name"] } -}) -``` - -### Array Format Handling - -Handle CSV-style array data with `itemType: "array"`: - -```typescript -// Input: [["id", "name"], [1, "Alice"], [2, "Bob"]] -const table = await loadJsonTable({ - path: "data.json", - dialect: { itemType: "array" } -}) -``` diff --git a/portal/typescript/content/docs/guides/jupyter.md b/portal/typescript/content/docs/guides/jupyter.md deleted file mode 100644 index 117171d6..00000000 --- a/portal/typescript/content/docs/guides/jupyter.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Using dpkit in Jupyter Notebooks -sidebar: - label: Jupyter Notebooks - order: 10 ---- - -For data scientists and data engineers, [Jupyter Notebooks](https://docs.jupyter.org/en/latest/) provide a powerful and flexible environment for exploring, visualizing, and analyzing data. - -## Installation - -1. **Install Jupyter:** `pip install jupyterlab` - Installs Jupyter Notebook, a web-based interactive computing environment for data science and data engineering. You can user another UI such as Jupyter CLI or Jupyter Desktop. - -1. **Install Deno:** `curl -fsSL https://deno.land/install.sh | sh` - Downloads and installs the Deno runtime required for TypeScript execution in Jupyter notebooks. Visit [Deno](https://deno.com) for official documentation. - -1. **Install Deno Jupyter Kernel:** `deno jupyter --install` - Sets up the Deno kernel for Jupyter, enabling TypeScript support within notebooks. - -1. **Activate Deno Jupyter Kernel:** `deno jupyter --unstable` - Enables the Deno kernel for Jupyter notebook usage. - -## Usage - -1. **Run Jupyter Notebooks:** `jupyter-lab` - Launches the Jupyter Notebook server in the current working directory, which allows you to create and run Jupyter notebooks. - -1. **Select Deno Kernel in Notebook:** Choose the Deno kernel from your notebook's kernel selection menu. VSCode users may need to install the default Jupyter kernel extensions. - -![dpkit in Jupyter Notebooks](./assets/jupyter.png) - -## References - -- [Typescript Jupyter Notebooks: How to set them up](https://alex-goff.medium.com/typescript-jupyter-notebooks-how-to-set-them-up-1ab9fd464ea4) -- [Bringing Modern JavaScript to the Jupyter Notebook](https://blog.jupyter.org/bringing-modern-javascript-to-the-jupyter-notebook-fc998095081e) diff --git a/portal/typescript/content/docs/guides/ods.md b/portal/typescript/content/docs/guides/ods.md deleted file mode 100644 index 9b2373c3..00000000 --- a/portal/typescript/content/docs/guides/ods.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -title: Working with ODS -sidebar: - label: ODS - order: 2 ---- -Comprehensive OpenDocument Spreadsheet (ODS) file handling with sheet selection, advanced header processing, and high-performance data operations. - -## Introduction - -> [!TIP] -> You can use `loadTable` and `saveTable` from `dpkit` instead of `@dpkit/ods` to load and save ODS files if the framework can infer that files are in the `ods` format. - -The ODS plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - -- `loadOdsTable` -- `saveOdsTable` - -For example: - -```typescript -import { loadOdsTable, saveOdsTable } from "@dpkit/ods" - -const table = await loadOdsTable({path: "table.ods"}) -// the field types will be automatically inferred -// or you can provide a Table Schema - -await saveOdsTable(table, {path: "output.ods"}) -``` - -## Basic Usage - -### Reading ODS Files - -> [!TIP] -> The ouput of `loadOdsTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. - -```typescript -import { loadOdsTable } from "@dpkit/ods" - -// Load a simple ODS file -const table = await loadOdsTable({ path: "data.ods" }) - -// Load with custom dialect (specify sheet) -const table = await loadOdsTable({ - path: "data.ods", - dialect: { - sheetName: "Sheet2", - header: true - } -}) - -// Load multiple ODS files (concatenated) -const table = await loadOdsTable({ - path: ["part1.ods", "part2.ods", "part3.ods"] -}) - -// Table is a Polars LazyDataFrame -const frame = table.collect() -frame.describe() -``` - -### Saving ODS Files - -```typescript -import { saveOdsTable } from "@dpkit/ods" - -// Save with default options -await saveOdsTable(table, { path: "output.ods" }) - -// Save with custom sheet name -await saveOdsTable(table, { - path: "output.ods", - dialect: { - sheetName: "Data" - } -}) -``` - -## Advanced Features - -### Sheet Selection - -```typescript -import { loadOdsTable } from "@dpkit/ods" - -// Select by sheet number (1-indexed) -const table = await loadOdsTable({ - path: "workbook.ods", - dialect: { - sheetNumber: 2 // Load second sheet - } -}) - -// Select by sheet name -const table = await loadOdsTable({ - path: "workbook.ods", - dialect: { - sheetName: "Sales Data" - } -}) -``` - -### Multi-Header Row Processing - -```typescript -import { loadOdsTable } from "@dpkit/ods" - -// ODS with multiple header rows: -// Year | 2023 | 2023 | 2024 | 2024 -// Quarter | Q1 | Q2 | Q1 | Q2 -// Revenue | 100 | 120 | 110 | 130 - -const table = await loadOdsTable({ - path: "multi-header.ods", - dialect: { - headerRows: [1, 2], - headerJoin: "_" - } -}) -// Resulting columns: ["Year_Quarter", "2023_Q1", "2023_Q2", "2024_Q1", "2024_Q2"] -``` - -### Comment Row Handling - -```typescript -import { loadOdsTable } from "@dpkit/ods" - -// ODS with comment rows -const table = await loadOdsTable({ - path: "with-comments.ods", - dialect: { - commentRows: [1, 2], // Skip first two rows - header: true - } -}) - -// Skip rows with comment character -const table = await loadOdsTable({ - path: "data.ods", - dialect: { - commentChar: "#" // Skip rows starting with # - } -}) -``` - -### Remote File Loading - -```typescript -import { loadOdsTable } from "@dpkit/ods" - -// Load from URL -const table = await loadOdsTable({ - path: "https://example.com/data.ods" -}) - -// Load multiple remote files -const table = await loadOdsTable({ - path: [ - "https://api.example.com/data-2023.ods", - "https://api.example.com/data-2024.ods" - ] -}) -``` - -### Header Options - -```typescript -import { loadOdsTable } from "@dpkit/ods" - -// No header row (use generated column names) -const table = await loadOdsTable({ - path: "data.ods", - dialect: { - header: false - } -}) -// Columns will be: field1, field2, field3, etc. - -// Custom header row offset -const table = await loadOdsTable({ - path: "data.ods", - dialect: { - headerRows: [3] // Use third row as header - } -}) -``` diff --git a/portal/typescript/content/docs/guides/parquet.md b/portal/typescript/content/docs/guides/parquet.md deleted file mode 100644 index 36634cd1..00000000 --- a/portal/typescript/content/docs/guides/parquet.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: Working with Parquet -sidebar: - label: Parquet - order: 4 ---- - -The `@dpkit/parquet` package provides efficient support for loading and saving data in Apache Parquet format. It uses Polars DataFrames for high-performance columnar data processing. - -## Installation - -```bash -npm install @dpkit/parquet -``` - -## Basic Usage - -### Loading Data - -```typescript -import { loadParquetTable } from "@dpkit/parquet" - -// Load from local file -const table = await loadParquetTable({ path: "data.parquet" }) - -// Load from remote URL -const table = await loadParquetTable({ - path: "https://example.com/data.parquet" -}) - -// Load multiple files (concatenated) -const table = await loadParquetTable({ - path: ["file1.parquet", "file2.parquet"] -}) -``` - -### Saving Data - -```typescript -import { saveParquetTable } from "@dpkit/parquet" - -// Save as Parquet format -await saveParquetTable(table, { path: "output.parquet" }) -``` diff --git a/portal/typescript/content/docs/guides/service.md b/portal/typescript/content/docs/guides/service.md deleted file mode 100644 index 7811874d..00000000 --- a/portal/typescript/content/docs/guides/service.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: Self-Hosting dpkit Service -sidebar: - label: dpkit Service - order: 10 ---- - -This document provides a brief overview of self-hosting **dpkit Service** on your own infrastructure, a OpenAPI-compatible web-server powering [dpkit Cloud](https://cloud.dpkit.dev). - -## Instructions - -1. Install `@dpkit/engine` package. -2. Within your project, create a `main.ts` file: - -```ts -import { createServer } from "@dpkit/engine/node" - -createServer({ start: true }) -``` -``` -Listening on http://localhost:8080/api -``` - -## Usage - -Visit `http://localhost:8080` to see the OpenAPI documentation for using your service. You can use your new API as any other OpenAPI-compatible service. - -## Support - -We happy to help with a deployment of any complexity. Please [reach out to us to get a quote](https://www.linkedin.com/in/evgeny-karev/). diff --git a/portal/typescript/content/docs/guides/table.md b/portal/typescript/content/docs/guides/table.md deleted file mode 100644 index 886ef28f..00000000 --- a/portal/typescript/content/docs/guides/table.md +++ /dev/null @@ -1,227 +0,0 @@ ---- -title: Working with tabular data -sidebar: - label: Tabular Data - order: 6 ---- - -The `@dpkit/table` package provides high-performance data validation and processing capabilities for tabular data. Built on top of **nodejs-polars** (a Rust-based DataFrame library), it offers robust schema validation, type inference, and error handling for CSV, Excel, and other tabular data formats. - -## Examples - -### Basic Table Validation - -```typescript -import * as pl from "nodejs-polars" -import { validateTable } from "@dpkit/table" -import type { Schema } from "@dpkit/metadata" - -// Create a table from data -const table = DataFrame({ - id: [1, 2, 3], - name: ["John", "Jane", "Bob"], - email: ["john@example.com", "jane@example.com", "bob@example.com"] -}).lazy() - -// Define schema with constraints -const schema: Schema = { - fields: [ - { name: "id", type: "integer", constraints: { required: true, unique: true } }, - { name: "name", type: "string", constraints: { required: true } }, - { name: "email", type: "string", constraints: { pattern: "^[^@]+@[^@]+\\.[^@]+$" } } - ] -} - -// validate the table -const errors = await validateTable(table, { schema }) -console.log(errors) // Array of validation errors -``` - -### Schema Inference - -```typescript -import { inferSchema } from "@dpkit/table" - -// Automatically infer schema from data patterns -const table = DataFrame({ - id: ["1", "2", "3"], - price: ["10.50", "25.00", "15.75"], - date: ["2023-01-15", "2023-02-20", "2023-03-25"], - active: ["true", "false", "true"] -}).lazy() - -const inferredSchema = await inferSchema(table, { - sampleRows: 100, // Sample size for inference - confidence: 0.9, // Confidence threshold - monthFirst: false, // Date format preference - commaDecimal: false // Decimal separator preference -}) - -// Result: automatically detected integer, number, date, and boolean types -``` - -### Field Matching Strategies - -```typescript -// Subset matching - data can have extra fields -const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" } - ] -} - -// Equal matching - field names must match regardless of order -const equalSchema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" } - ] -} -``` - -### Table Processing - -```typescript -import { normalizeTable } from "@dpkit/table" - -// Process table with schema (converts string columns to proper types) -const table = DataFrame({ - id: ["1", "2", "3"], // String data - price: ["10.50", "25.00", "15.75"], - active: ["true", "false", "true"], - date: ["2023-01-15", "2023-02-20", "2023-03-25"] -}).lazy() - -const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "price", type: "number" }, - { name: "active", type: "boolean" }, - { name: "date", type: "date" } - ] -} - -const processedTable = await normalizeTable(table, { schema }) -const result = await processedTable.collect() - -// Result will have properly typed columns: -// { id: 1, price: 10.50, active: true, date: Date('2023-01-15') } -// { id: 2, price: 25.00, active: false, date: Date('2023-02-20') } -// { id: 3, price: 15.75, active: true, date: Date('2023-03-25') } -``` - -### Error Handling - -```typescript -const result = await validateTable(table, { schema }) - -result.errors.forEach(error => { - switch (error.type) { - case "cell/required": - console.log(`Required field missing in row ${error.rowNumber}: '${error.fieldName}'`) - break - case "cell/unique": - console.log(`Duplicate value in row ${error.rowNumber}: '${error.cell}'`) - break - case "cell/pattern": - console.log(`Pattern mismatch: '${error.cell}' doesn't match ${error.constraint}`) - break - } -}) -``` - -## Core Architecture - -### Table Type -The package uses `LazyDataFrame` from nodejs-polars as its core table representation, enabling lazy evaluation and efficient processing of large datasets through vectorized operations. - -### Schema Integration -Integrates seamlessly with `@dpkit/metadata` schemas, bridging Data Package field definitions with Polars data types for comprehensive validation workflows. - -## Key Features - -### 1. Multi-Level Validation System - -**Field-Level Validation:** -- **Type Validation**: Converts and validates data types (string → integer, etc.) -- **Name Validation**: Ensures field names match schema requirements -- **Constraint Validation**: Enforces required, unique, enum, pattern, min/max values, and length constraints - -**Table-Level Validation:** -- **Field Presence**: Validates missing/extra fields based on flexible matching strategies -- **Schema Compatibility**: Ensures data structure aligns with schema definitions - -**Row-Level Validation:** -- **Primary Key Uniqueness**: Validates unique identifiers -- **Composite Keys**: Supports multi-column unique constraints - -### 2. Comprehensive Field Types - -**Primitive Types:** -- `string`, `integer`, `number`, `boolean` - -**Temporal Types:** -- `date`, `datetime`, `time`, `year`, `yearmonth`, `duration` - -**Spatial Types:** -- `geopoint`, `geojson` - -**Complex Types:** -- `array`, `list`, `object` - -### 3. Smart Schema Inference - -Automatically infers field types and formats from data using: -- Regex pattern matching with configurable confidence thresholds -- Locale-specific format detection (comma decimals, date formats) -- Complex type recognition (objects, arrays, temporal data) - -### 4. Flexible Field Matching Strategies - -- **exact**: Fields must match exactly in order and count -- **equal**: Same fields in any order -- **subset**: Data must contain all schema fields (extras allowed) -- **superset**: Schema must contain all data fields -- **partial**: At least one field must match - -### 5. Advanced Data Processing - -**Format-Aware Parsing:** -- Handles missing values at schema and field levels -- Supports group/decimal character customization -- Processes currency symbols and whitespace -- Parses complex formats (ISO dates, scientific notation) - -**Performance Optimizations:** -- Sample-based validation for large datasets -- Lazy evaluation for memory efficiency -- Vectorized constraint checking -- Configurable error limits and batch processing - -## Error Handling - -### Comprehensive Error Taxonomy - -**Field Errors:** -- Name mismatches between schema and data -- Type conversion failures -- Missing or extra field violations - -**Cell Errors:** -- Type validation failures with specific conversion details -- Constraint violations (required, unique, enum, pattern, range, length) -- Format parsing errors with problematic values - -**Row Errors:** -- Unique key constraint violations -- Composite constraint failures - -### Error Details -Each error includes: -- Precise row and column locations -- Actual vs expected values -- Specific error type classification -- Actionable error messages for debugging diff --git a/portal/typescript/content/docs/guides/visidata.md b/portal/typescript/content/docs/guides/visidata.md deleted file mode 100644 index 20fd3354..00000000 --- a/portal/typescript/content/docs/guides/visidata.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Using dpkit with VisiData -sidebar: - label: VisiData - order: 11 ---- - -[VisiData](https://www.visidata.org/) provide a powerful and flexible environment for data exploration, visualization, and analysis. dpkit can be used as a tool that prepared data for following usage in VisiData. - -## Installation - -- [Install dpkit CLI](https://dpkit.dev/overview/getting-started/) -- [Install VisiData](https://www.visidata.org/install/) - -## Usage - -For example, we can use dpkit to copy a remote Data Package to a local folder and then load it in VisiData. - -```bash -dpkit copy https://zenodo.org/records/7559361 --toFolder dataset --withRemote -vd dataset/*.csv -``` - -> [!TIP] -> Functionality allowing to copy a Data Package to a SQLite database is under development. Once it is ready, it will be possible to prepare data to VisiData in more type-safe manner. - -## References - -- [Visidata](https://www.visidata.org/) diff --git a/portal/typescript/content/docs/guides/xlsx.md b/portal/typescript/content/docs/guides/xlsx.md deleted file mode 100644 index 7f7ba4c6..00000000 --- a/portal/typescript/content/docs/guides/xlsx.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -title: Working with XLSX -sidebar: - label: XLSX - order: 1 ---- -Comprehensive Excel (.xlsx) file handling with sheet selection, advanced header processing, and high-performance data operations. - -## Introduction - -> [!TIP] -> You can use `loadTable` and `saveTable` from `dpkit` instead of `@dpkit/xlsx` to load and save XLSX files if the framework can infer that files are in the `xlsx` format. - -The XLSX plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - -- `loadXlsxTable` -- `saveXlsxTable` - -For example: - -```typescript -import { loadXlsxTable, saveXlsxTable } from "@dpkit/xlsx" - -const table = await loadXlsxTable({path: "table.xlsx"}) -// the field types will be automatically inferred -// or you can provide a Table Schema - -await saveXlsxTable(table, {path: "output.xlsx"}) -``` - -## Basic Usage - -### Reading XLSX Files - -> [!TIP] -> The ouput of `loadXlsxTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. - -```typescript -import { loadXlsxTable } from "@dpkit/xlsx" - -// Load a simple XLSX file -const table = await loadXlsxTable({ path: "data.xlsx" }) - -// Load with custom dialect (specify sheet) -const table = await loadXlsxTable({ - path: "data.xlsx", - dialect: { - sheetName: "Sheet2", - header: true - } -}) - -// Load multiple XLSX files (concatenated) -const table = await loadXlsxTable({ - path: ["part1.xlsx", "part2.xlsx", "part3.xlsx"] -}) - -// Table is a Polars LazyDataFrame -const frame = table.collect() -frame.describe() -``` - -### Saving XLSX Files - -```typescript -import { saveXlsxTable } from "@dpkit/xlsx" - -// Save with default options -await saveXlsxTable(table, { path: "output.xlsx" }) - -// Save with custom sheet name -await saveXlsxTable(table, { - path: "output.xlsx", - dialect: { - sheetName: "Data" - } -}) -``` - -## Advanced Features - -### Sheet Selection - -```typescript -import { loadXlsxTable } from "@dpkit/xlsx" - -// Select by sheet number (1-indexed) -const table = await loadXlsxTable({ - path: "workbook.xlsx", - dialect: { - sheetNumber: 2 // Load second sheet - } -}) - -// Select by sheet name -const table = await loadXlsxTable({ - path: "workbook.xlsx", - dialect: { - sheetName: "Sales Data" - } -}) -``` - -### Multi-Header Row Processing - -```typescript -import { loadXlsxTable } from "@dpkit/xlsx" - -// XLSX with multiple header rows: -// Year | 2023 | 2023 | 2024 | 2024 -// Quarter | Q1 | Q2 | Q1 | Q2 -// Revenue | 100 | 120 | 110 | 130 - -const table = await loadXlsxTable({ - path: "multi-header.xlsx", - dialect: { - headerRows: [1, 2], - headerJoin: "_" - } -}) -// Resulting columns: ["Year_Quarter", "2023_Q1", "2023_Q2", "2024_Q1", "2024_Q2"] -``` - -### Comment Row Handling - -```typescript -import { loadXlsxTable } from "@dpkit/xlsx" - -// XLSX with comment rows -const table = await loadXlsxTable({ - path: "with-comments.xlsx", - dialect: { - commentRows: [1, 2], // Skip first two rows - header: true - } -}) - -// Skip rows with comment character -const table = await loadXlsxTable({ - path: "data.xlsx", - dialect: { - commentChar: "#" // Skip rows starting with # - } -}) -``` - -### Remote File Loading - -```typescript -import { loadXlsxTable } from "@dpkit/xlsx" - -// Load from URL -const table = await loadXlsxTable({ - path: "https://example.com/data.xlsx" -}) - -// Load multiple remote files -const table = await loadXlsxTable({ - path: [ - "https://api.example.com/data-2023.xlsx", - "https://api.example.com/data-2024.xlsx" - ] -}) -``` - -### Header Options - -```typescript -import { loadXlsxTable } from "@dpkit/xlsx" - -// No header row (use generated column names) -const table = await loadXlsxTable({ - path: "data.xlsx", - dialect: { - header: false - } -}) -// Columns will be: field1, field2, field3, etc. - -// Custom header row offset -const table = await loadXlsxTable({ - path: "data.xlsx", - dialect: { - headerRows: [3] // Use third row as header - } -}) -``` diff --git a/portal/typescript/content/docs/index.md b/portal/typescript/content/docs/index.md deleted file mode 100644 index 59fa6a4e..00000000 --- a/portal/typescript/content/docs/index.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -title: dpkit TypeScript -sidebar: - order: 1 - label: Getting Started ---- - -This guide will help you get started with dpkit in TypeScript. If you are new to the core framework's tecnhologies, please take a look at the [Data Package standard](https://datapackage.org/) and [Polars DataFrames](https://pola.rs/) documentation. - -## Runtimes - -> [!TIP] -> - It is possible to use dpkit in [Jupyter Notebooks](/guides/jupyter)! - -dpkit and all its packages support all the prominent TypeScript runtimes: - -- **Node v22+** -- **Deno v2+** -- **Bun v1+** - -The core package `@dpkit/metadata` additionally supports browser environments: - -- **Edge v92+** -- **Chrome v92+** -- **Firefox v90+** -- and others - -## Installation - -> [!NOTE] -> The documentation uses `npm` command to install packages. If you are using other package managers, please adjust the commands accordingly. - -The framework can be installed as one package including CLI: - -```bash -npm install dpkit -``` - -If you don't need CLI, you can install it as: - -```bash -npm install @dpkit/lib -``` - -You car cherry-pick from individual packages: - -```bash -npm install @dpkit/metadata @dpkit/zenodo -``` - -In the browser, the core package can be just imported using NPM CDNs: - -```js -import { loadPackageDescriptor } from "https://esm.sh/@dpkit/metadata" -``` - -## TypeScript - -> [!TIP] -> Use **Node v24+** to be able to run TypeScript files directly with the `node` binary like `node my-data-script.ts` - -dpkit is built with type safety in mind. It uses TypeScript to provide type definitions for all packages and to enforce type safety throughout the framework. It's highly reccomended to setup a TypeScript aware environment to work with the project. - -## Examples - -Loading a Data Package from Zenodo merging system Zenodo metadata into a user data package and validating its metadata: - -```ts -import { loadPackage } from "dpkit" - -const { dataPackage } = await loadPackage("https://zenodo.org/records/10053903") - -console.log(dataPackage) -//{ -// id: 'https://doi.org/10.5281/zenodo.10053903', -// profile: 'tabular-data-package', -// ... -//} - -``` - -Validating an in-memory package descriptor: - -```ts -import { validatePackageDescriptor } from "dpkit" - -const { valid, errors } = await validatePackageDescriptor({ name: "package" }) - -console.log(valid) -// false -console.log(errors) -//[ -// { -// instancePath: '', -// schemaPath: '#/required', -// keyword: 'required', -// params: { missingProperty: 'resources' }, -// message: "must have required property 'resources'", -// type: 'descriptor' -// } -//] -``` - -Loading a package from a remote descriptor and saving it locally as a zip archive, and then using it as a local data package: - -```ts -import { - loadPackageDescriptor, - loadPackageFromZip, - savePackageToZip, - getTempFilePath, -} from "dpkit" - -const archivePath = getTempFilePath() -const sourcePath = await loadPackageDescriptor( - "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/datapackage.json", -) - -await savePackageToZip(sourcePackage, { archivePath }) -const targetPackage = await loadPackageFromZip(archivePath) -console.log(targetPackage) -``` - -Reading a CSV table: - -```ts -import { loadTable } from "dpkit" - -const table = await loadTable({ path: "data.csv" }) - -// Load with custom dialect -const table = await loadTable({ - path: "data.csv", - dialect: { - delimiter: ";", - header: true, - skipInitialSpace: true - } -}) -``` - -## Reference - -See **API Reference** of each individual package for more details. Note, that `dpkit` and `@dpkit/lib` packages re-export most of the functionality. diff --git a/portal/typescript/content/docs/overview/contributing.md b/portal/typescript/content/docs/overview/contributing.md deleted file mode 100644 index 81fee01b..00000000 --- a/portal/typescript/content/docs/overview/contributing.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: Contributing -sidebar: - order: 2 ---- -Thank you for your interest in contributing to dpkit! This document provides guidelines and instructions for contributing to this project. - -## Project Overview - -Project is a monorepo with the following packages: - -- `@dpkit/metadata`: Core metadata functionality -- `@dpkit/dataset`: File-related functionality -- `@dpkit/table`: Table-related functionality -- `@dpkit/`: Domain-specific functionality -- `@dpkit/library`: All-in-one package that re-exports all functionality -- `@dpkit/terminal`: Terminal interface for running tasks -- `@dpkit/website`: Website-related functionality -- `dpkit`: Meta-package that re-exports the underlying functionality - -## Development Environment - -### Prerequisites - -> [!TIP] -> For CLI compilation and development, we recommend using [Bun](https://bun.sh/). For example, to run CLI in dev `bun cli/main.ts` - -- **Node.js**: v24.0.0 or higher -- **PNPM**: v10.0.0 or higher - -### Setup - -1. Clone the repository - ```bash - git clone https://github.com/yourusername/dpkit.git dpkit - cd dpkit - ``` - -2. Install dependencies - ```bash - pnpm install - ``` -## Development Workflow - -### Code Style and Quality - -We use Biome for linting and formatting, and TypeScript for type checking: - -- **Lint**: Check for code issues - ```bash - pnpm run lint - ``` - -- **Format**: Auto-fix formatting issues - ```bash - pnpm run format - ``` - -- **Type Check**: Verify TypeScript types - ```bash - pnpm run type - ``` - -- **Comprehensive Check**: Run lint and type checking - ```bash - pnpm run check - ``` - -### Testing - -Tests are located in `__spec__` directories and use Vitest: - -- **Run All Tests**: (includes linting and type checking) - ```bash - pnpm test - ``` - -- **Run Tests Only**: (without linting/type checking) - ```bash - pnpm run spec - ``` - -- **Run a Specific Test**: - ```bash - pnpm exec vitest run core/actions/__spec__/findTask.ts - ``` - -### Dependencies - -Update all dependencies to their latest versions: - -```bash -pnpm run bump -``` -## Code Style Guidelines - -- Use TypeScript with strict type checking -- Follow ES modules pattern (`import`/`export`) -- Tests should be placed in `__spec__` directories -- Use semicolons as needed (not required everywhere) -- Use arrow function parentheses as needed (omitted for single parameters) - -## Making Changes to the Meta-Package - -When adding new functionality: - -1. Add it to the appropriate package first (`core`, `cli`, or `cron`) -2. Ensure it's properly exported from that package -3. No additional work is needed for the meta-package as it automatically re-exports everything - -## Submitting Changes - -1. Create a feature branch (`git checkout -b feature/your-feature`) -2. Make your changes with appropriate tests -3. Ensure the code passes all checks: `pnpm test` -4. Commit your changes with a descriptive message -5. Submit a pull request - -## License - -By contributing to dpkit, you agree that your contributions will be licensed under the project's license. - -Thank you for your contribution! diff --git a/portal/typescript/content/docs/overview/funding.md b/portal/typescript/content/docs/overview/funding.md deleted file mode 100644 index fdea7321..00000000 --- a/portal/typescript/content/docs/overview/funding.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Funding ---- - -This project is funded through [NGI Zero Commons Fund](https://nlnet.nl/commonsfund/), a fund established by [NLnet](https://nlnet.nl) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu) program. Learn more at the [NLnet project page](https://nlnet.nl/project/DataPackage-TS/). - - diff --git a/portal/typescript/content/docs/reference/AnyConstraints.md b/portal/typescript/content/docs/reference/AnyConstraints.md deleted file mode 100644 index 25c0ac61..00000000 --- a/portal/typescript/content/docs/reference/AnyConstraints.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "AnyConstraints" ---- - -Defined in: metadata/build/field/types/Any.d.ts:14 - -Any field constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `any`[] - -Defined in: metadata/build/field/types/Any.d.ts:19 - -Restrict values to a specified set -For any field type, can be an array of any values - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/AnyField.md b/portal/typescript/content/docs/reference/AnyField.md deleted file mode 100644 index 743f1e62..00000000 --- a/portal/typescript/content/docs/reference/AnyField.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "AnyField" ---- - -Defined in: metadata/build/field/types/Any.d.ts:5 - -Any field type (unspecified/mixed) - -## Extends - -- `BaseField`\<[`AnyConstraints`](/reference/anyconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`AnyConstraints`](/reference/anyconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`description`](/reference/arrayfield/#description) - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`example`](/reference/arrayfield/#example) - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`examples`](/reference/arrayfield/#examples) - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`format`](/reference/arrayfield/#format) - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`name`](/reference/arrayfield/#name) - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`rdfType`](/reference/arrayfield/#rdftype) - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -[`ArrayField`](/reference/arrayfield/).[`title`](/reference/arrayfield/#title) - -*** - -### type? - -> `optional` **type**: `"any"` - -Defined in: metadata/build/field/types/Any.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/ArrayConstraints.md b/portal/typescript/content/docs/reference/ArrayConstraints.md deleted file mode 100644 index 7bec28f5..00000000 --- a/portal/typescript/content/docs/reference/ArrayConstraints.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ArrayConstraints" ---- - -Defined in: metadata/build/field/types/Array.d.ts:14 - -Array-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `any`[][] - -Defined in: metadata/build/field/types/Array.d.ts:31 - -Restrict values to a specified set of arrays -Serialized as JSON strings or parsed array objects - -*** - -### jsonSchema? - -> `optional` **jsonSchema**: `Record`\<`string`, `any`\> - -Defined in: metadata/build/field/types/Array.d.ts:26 - -JSON Schema object for validating array items - -*** - -### maxLength? - -> `optional` **maxLength**: `number` - -Defined in: metadata/build/field/types/Array.d.ts:22 - -Maximum array length - -*** - -### minLength? - -> `optional` **minLength**: `number` - -Defined in: metadata/build/field/types/Array.d.ts:18 - -Minimum array length - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/ArrayField.md b/portal/typescript/content/docs/reference/ArrayField.md deleted file mode 100644 index 55a4c228..00000000 --- a/portal/typescript/content/docs/reference/ArrayField.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ArrayField" ---- - -Defined in: metadata/build/field/types/Array.d.ts:5 - -Array field type (serialized JSON array) - -## Extends - -- `BaseField`\<[`ArrayConstraints`](/reference/arrayconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`ArrayConstraints`](/reference/arrayconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"array"` - -Defined in: metadata/build/field/types/Array.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/ArrowPlugin.md b/portal/typescript/content/docs/reference/ArrowPlugin.md deleted file mode 100644 index 5962af1e..00000000 --- a/portal/typescript/content/docs/reference/ArrowPlugin.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ArrowPlugin" ---- - -Defined in: table/build/plugins/arrow/plugin.d.ts:4 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new ArrowPlugin**(): `ArrowPlugin` - -#### Returns - -`ArrowPlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/arrow/plugin.d.ts:5 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugins/arrow/plugin.d.ts:6 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/BaseCellError.md b/portal/typescript/content/docs/reference/BaseCellError.md deleted file mode 100644 index a5e97a20..00000000 --- a/portal/typescript/content/docs/reference/BaseCellError.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BaseCellError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:4 - -## Extends - -- `BaseError` - -## Extended by - -- [`CellEnumError`](/reference/cellenumerror/) -- [`CellExclusiveMaximumError`](/reference/cellexclusivemaximumerror/) -- [`CellExclusiveMinimumError`](/reference/cellexclusiveminimumerror/) -- [`CellJsonSchemaError`](/reference/celljsonschemaerror/) -- [`CellMaxLengthError`](/reference/cellmaxlengtherror/) -- [`CellMaximumError`](/reference/cellmaximumerror/) -- [`CellMinLengthError`](/reference/cellminlengtherror/) -- [`CellMinimumError`](/reference/cellminimumerror/) -- [`CellPatternError`](/reference/cellpatternerror/) -- [`CellRequiredError`](/reference/cellrequirederror/) -- [`CellTypeError`](/reference/celltypeerror/) -- [`CellUniqueError`](/reference/celluniqueerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -*** - -### type - -> **type**: `string` - -Defined in: metadata/build/error/types/Base.d.ts:2 - -#### Inherited from - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/BaseFieldError.md b/portal/typescript/content/docs/reference/BaseFieldError.md deleted file mode 100644 index 43d1b8cc..00000000 --- a/portal/typescript/content/docs/reference/BaseFieldError.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BaseFieldError" ---- - -Defined in: metadata/build/error/types/Field.d.ts:4 - -## Extends - -- `BaseError` - -## Extended by - -- [`FieldNameError`](/reference/fieldnameerror/) -- [`FieldTypeError`](/reference/fieldtypeerror/) - -## Properties - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Field.d.ts:5 - -*** - -### type - -> **type**: `string` - -Defined in: metadata/build/error/types/Base.d.ts:2 - -#### Inherited from - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/BaseFieldsError.md b/portal/typescript/content/docs/reference/BaseFieldsError.md deleted file mode 100644 index edc11f3c..00000000 --- a/portal/typescript/content/docs/reference/BaseFieldsError.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BaseFieldsError" ---- - -Defined in: metadata/build/error/types/Fields.d.ts:3 - -## Extends - -- `BaseError` - -## Extended by - -- [`FieldsExtraError`](/reference/fieldsextraerror/) -- [`FieldsMissingError`](/reference/fieldsmissingerror/) - -## Properties - -### fieldNames - -> **fieldNames**: `string`[] - -Defined in: metadata/build/error/types/Fields.d.ts:4 - -*** - -### type - -> **type**: `string` - -Defined in: metadata/build/error/types/Base.d.ts:2 - -#### Inherited from - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/BaseRowError.md b/portal/typescript/content/docs/reference/BaseRowError.md deleted file mode 100644 index b89ffb87..00000000 --- a/portal/typescript/content/docs/reference/BaseRowError.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BaseRowError" ---- - -Defined in: metadata/build/error/types/Row.d.ts:3 - -## Extends - -- `BaseError` - -## Extended by - -- [`RowUniqueError`](/reference/rowuniqueerror/) - -## Properties - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Row.d.ts:4 - -*** - -### type - -> **type**: `string` - -Defined in: metadata/build/error/types/Base.d.ts:2 - -#### Inherited from - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/BooleanConstraints.md b/portal/typescript/content/docs/reference/BooleanConstraints.md deleted file mode 100644 index fe01782c..00000000 --- a/portal/typescript/content/docs/reference/BooleanConstraints.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BooleanConstraints" ---- - -Defined in: metadata/build/field/types/Boolean.d.ts:22 - -Boolean-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `boolean`[] - -Defined in: metadata/build/field/types/Boolean.d.ts:27 - -Restrict values to a specified set -Can be an array of booleans or strings that parse to booleans - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/BooleanField.md b/portal/typescript/content/docs/reference/BooleanField.md deleted file mode 100644 index b98d06c3..00000000 --- a/portal/typescript/content/docs/reference/BooleanField.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BooleanField" ---- - -Defined in: metadata/build/field/types/Boolean.d.ts:5 - -Boolean field type - -## Extends - -- `BaseField`\<[`BooleanConstraints`](/reference/booleanconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`BooleanConstraints`](/reference/booleanconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### falseValues? - -> `optional` **falseValues**: `string`[] - -Defined in: metadata/build/field/types/Boolean.d.ts:17 - -Values that represent false - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### trueValues? - -> `optional` **trueValues**: `string`[] - -Defined in: metadata/build/field/types/Boolean.d.ts:13 - -Values that represent true - -*** - -### type - -> **type**: `"boolean"` - -Defined in: metadata/build/field/types/Boolean.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/BoundError.md b/portal/typescript/content/docs/reference/BoundError.md deleted file mode 100644 index 335c6842..00000000 --- a/portal/typescript/content/docs/reference/BoundError.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BoundError" ---- - -> **BoundError** = [`UnboundError`](/reference/unbounderror/) & `object` - -Defined in: metadata/build/error/Bound.d.ts:2 - -## Type declaration - -### resource - -> **resource**: `string` diff --git a/portal/typescript/content/docs/reference/BytesError.md b/portal/typescript/content/docs/reference/BytesError.md deleted file mode 100644 index 67d5501a..00000000 --- a/portal/typescript/content/docs/reference/BytesError.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "BytesError" ---- - -Defined in: metadata/build/error/types/Bytes.d.ts:2 - -## Extends - -- `BaseError` - -## Properties - -### actualBytes - -> **actualBytes**: `number` - -Defined in: metadata/build/error/types/Bytes.d.ts:5 - -*** - -### bytes - -> **bytes**: `number` - -Defined in: metadata/build/error/types/Bytes.d.ts:4 - -*** - -### type - -> **type**: `"file/bytes"` - -Defined in: metadata/build/error/types/Bytes.d.ts:3 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/CellEnumError.md b/portal/typescript/content/docs/reference/CellEnumError.md deleted file mode 100644 index 52510a20..00000000 --- a/portal/typescript/content/docs/reference/CellEnumError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellEnumError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:48 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### enum - -> **enum**: `string`[] - -Defined in: metadata/build/error/types/Cell.d.ts:50 - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/enum"` - -Defined in: metadata/build/error/types/Cell.d.ts:49 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellError.md b/portal/typescript/content/docs/reference/CellError.md deleted file mode 100644 index 331cef5f..00000000 --- a/portal/typescript/content/docs/reference/CellError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellError" ---- - -> **CellError** = [`CellTypeError`](/reference/celltypeerror/) \| [`CellRequiredError`](/reference/cellrequirederror/) \| [`CellMinimumError`](/reference/cellminimumerror/) \| [`CellMaximumError`](/reference/cellmaximumerror/) \| [`CellExclusiveMinimumError`](/reference/cellexclusiveminimumerror/) \| [`CellExclusiveMaximumError`](/reference/cellexclusivemaximumerror/) \| [`CellMinLengthError`](/reference/cellminlengtherror/) \| [`CellMaxLengthError`](/reference/cellmaxlengtherror/) \| [`CellPatternError`](/reference/cellpatternerror/) \| [`CellUniqueError`](/reference/celluniqueerror/) \| [`CellEnumError`](/reference/cellenumerror/) \| [`CellJsonSchemaError`](/reference/celljsonschemaerror/) - -Defined in: metadata/build/error/types/Cell.d.ts:3 diff --git a/portal/typescript/content/docs/reference/CellExclusiveMaximumError.md b/portal/typescript/content/docs/reference/CellExclusiveMaximumError.md deleted file mode 100644 index 99ad5c28..00000000 --- a/portal/typescript/content/docs/reference/CellExclusiveMaximumError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellExclusiveMaximumError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:29 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### maximum - -> **maximum**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:31 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/exclusiveMaximum"` - -Defined in: metadata/build/error/types/Cell.d.ts:30 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellExclusiveMinimumError.md b/portal/typescript/content/docs/reference/CellExclusiveMinimumError.md deleted file mode 100644 index d7aa5b73..00000000 --- a/portal/typescript/content/docs/reference/CellExclusiveMinimumError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellExclusiveMinimumError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:25 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### minimum - -> **minimum**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:27 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/exclusiveMinimum"` - -Defined in: metadata/build/error/types/Cell.d.ts:26 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellJsonSchemaError.md b/portal/typescript/content/docs/reference/CellJsonSchemaError.md deleted file mode 100644 index 8b356c2e..00000000 --- a/portal/typescript/content/docs/reference/CellJsonSchemaError.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellJsonSchemaError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:52 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### message - -> **message**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:55 - -*** - -### pointer - -> **pointer**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:54 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/jsonSchema"` - -Defined in: metadata/build/error/types/Cell.d.ts:53 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellMaxLengthError.md b/portal/typescript/content/docs/reference/CellMaxLengthError.md deleted file mode 100644 index 3fcf2d4d..00000000 --- a/portal/typescript/content/docs/reference/CellMaxLengthError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellMaxLengthError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:37 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### maxLength - -> **maxLength**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:39 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/maxLength"` - -Defined in: metadata/build/error/types/Cell.d.ts:38 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellMaximumError.md b/portal/typescript/content/docs/reference/CellMaximumError.md deleted file mode 100644 index a1814ca8..00000000 --- a/portal/typescript/content/docs/reference/CellMaximumError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellMaximumError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:21 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### maximum - -> **maximum**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:23 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/maximum"` - -Defined in: metadata/build/error/types/Cell.d.ts:22 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellMinLengthError.md b/portal/typescript/content/docs/reference/CellMinLengthError.md deleted file mode 100644 index 8734fd3a..00000000 --- a/portal/typescript/content/docs/reference/CellMinLengthError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellMinLengthError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:33 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### minLength - -> **minLength**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:35 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/minLength"` - -Defined in: metadata/build/error/types/Cell.d.ts:34 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellMinimumError.md b/portal/typescript/content/docs/reference/CellMinimumError.md deleted file mode 100644 index c1431bae..00000000 --- a/portal/typescript/content/docs/reference/CellMinimumError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellMinimumError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:17 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### minimum - -> **minimum**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:19 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/minimum"` - -Defined in: metadata/build/error/types/Cell.d.ts:18 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellPatternError.md b/portal/typescript/content/docs/reference/CellPatternError.md deleted file mode 100644 index adfb4a86..00000000 --- a/portal/typescript/content/docs/reference/CellPatternError.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellPatternError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:41 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### pattern - -> **pattern**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:43 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/pattern"` - -Defined in: metadata/build/error/types/Cell.d.ts:42 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellRequiredError.md b/portal/typescript/content/docs/reference/CellRequiredError.md deleted file mode 100644 index bc0799bf..00000000 --- a/portal/typescript/content/docs/reference/CellRequiredError.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellRequiredError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:14 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/required"` - -Defined in: metadata/build/error/types/Cell.d.ts:15 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellTypeError.md b/portal/typescript/content/docs/reference/CellTypeError.md deleted file mode 100644 index 254e3434..00000000 --- a/portal/typescript/content/docs/reference/CellTypeError.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellTypeError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:9 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldFormat? - -> `optional` **fieldFormat**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:12 - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### fieldType - -> **fieldType**: [`FieldType`](/reference/fieldtype/) - -Defined in: metadata/build/error/types/Cell.d.ts:11 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/type"` - -Defined in: metadata/build/error/types/Cell.d.ts:10 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CellUniqueError.md b/portal/typescript/content/docs/reference/CellUniqueError.md deleted file mode 100644 index 89d7687f..00000000 --- a/portal/typescript/content/docs/reference/CellUniqueError.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CellUniqueError" ---- - -Defined in: metadata/build/error/types/Cell.d.ts:45 - -## Extends - -- [`BaseCellError`](/reference/basecellerror/) - -## Properties - -### cell - -> **cell**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:7 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`cell`](/reference/basecellerror/#cell) - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Cell.d.ts:5 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`fieldName`](/reference/basecellerror/#fieldname) - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Cell.d.ts:6 - -#### Inherited from - -[`BaseCellError`](/reference/basecellerror/).[`rowNumber`](/reference/basecellerror/#rownumber) - -*** - -### type - -> **type**: `"cell/unique"` - -Defined in: metadata/build/error/types/Cell.d.ts:46 - -#### Overrides - -[`BaseCellError`](/reference/basecellerror/).[`type`](/reference/basecellerror/#type) diff --git a/portal/typescript/content/docs/reference/CkanPlugin.md b/portal/typescript/content/docs/reference/CkanPlugin.md deleted file mode 100644 index a8eb3264..00000000 --- a/portal/typescript/content/docs/reference/CkanPlugin.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CkanPlugin" ---- - -Defined in: dataset/build/plugins/ckan/plugin.d.ts:2 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new CkanPlugin**(): `CkanPlugin` - -#### Returns - -`CkanPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| \{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: dataset/build/plugins/ckan/plugin.d.ts:3 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| \{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) diff --git a/portal/typescript/content/docs/reference/Contributor.md b/portal/typescript/content/docs/reference/Contributor.md deleted file mode 100644 index 8c562a69..00000000 --- a/portal/typescript/content/docs/reference/Contributor.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Contributor" ---- - -Defined in: metadata/build/package/Contributor.d.ts:4 - -Contributor information - -## Properties - -### email? - -> `optional` **email**: `string` - -Defined in: metadata/build/package/Contributor.d.ts:12 - -Email address of the contributor - -*** - -### path? - -> `optional` **path**: `string` - -Defined in: metadata/build/package/Contributor.d.ts:16 - -Path to relevant contributor information - -*** - -### role? - -> `optional` **role**: `string` - -Defined in: metadata/build/package/Contributor.d.ts:20 - -Role of the contributor - -*** - -### title - -> **title**: `string` - -Defined in: metadata/build/package/Contributor.d.ts:8 - -Full name of the contributor diff --git a/portal/typescript/content/docs/reference/CsvPlugin.md b/portal/typescript/content/docs/reference/CsvPlugin.md deleted file mode 100644 index 71006c49..00000000 --- a/portal/typescript/content/docs/reference/CsvPlugin.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "CsvPlugin" ---- - -Defined in: table/build/plugins/csv/plugin.d.ts:5 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new CsvPlugin**(): `CsvPlugin` - -#### Returns - -`CsvPlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/csv/plugin.d.ts:6 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugins/csv/plugin.d.ts:7 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/DataError.md b/portal/typescript/content/docs/reference/DataError.md deleted file mode 100644 index 2ff8da76..00000000 --- a/portal/typescript/content/docs/reference/DataError.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DataError" ---- - -Defined in: metadata/build/error/types/Data.d.ts:2 - -## Extends - -- `BaseError` - -## Properties - -### message - -> **message**: `string` - -Defined in: metadata/build/error/types/Data.d.ts:4 - -*** - -### type - -> **type**: `"data"` - -Defined in: metadata/build/error/types/Data.d.ts:3 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/DataRecord.md b/portal/typescript/content/docs/reference/DataRecord.md deleted file mode 100644 index 3d0105da..00000000 --- a/portal/typescript/content/docs/reference/DataRecord.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DataRecord" ---- - -> **DataRecord** = `Record`\<`string`, `unknown`\> - -Defined in: table/build/data/record.d.ts:3 diff --git a/portal/typescript/content/docs/reference/DataRow.md b/portal/typescript/content/docs/reference/DataRow.md deleted file mode 100644 index a59a13bd..00000000 --- a/portal/typescript/content/docs/reference/DataRow.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DataRow" ---- - -> **DataRow** = `unknown`[] - -Defined in: table/build/data/record.d.ts:2 diff --git a/portal/typescript/content/docs/reference/DatabasePlugin.md b/portal/typescript/content/docs/reference/DatabasePlugin.md deleted file mode 100644 index 9991ea32..00000000 --- a/portal/typescript/content/docs/reference/DatabasePlugin.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DatabasePlugin" ---- - -Defined in: database/build/plugin.d.ts:5 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new DatabasePlugin**(): `DatabasePlugin` - -#### Returns - -`DatabasePlugin` - -## Methods - -### inferSchema() - -> **inferSchema**(`resource`): `Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> - -Defined in: database/build/plugin.d.ts:12 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -#### Returns - -`Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`inferSchema`](/reference/tableplugin/#inferschema) - -*** - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: database/build/plugin.d.ts:11 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadPackage`](/reference/tableplugin/#loadpackage) - -*** - -### loadTable() - -> **loadTable**(`resource`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: database/build/plugin.d.ts:13 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### savePackage() - -> **savePackage**(`dataPackage`, `options`): `Promise`\<`undefined` \| \{ `path`: `string`; \}\> - -Defined in: database/build/plugin.d.ts:6 - -#### Parameters - -##### dataPackage - -[`Package`](/reference/package/) - -##### options - -[`SavePackageOptions`](/reference/savepackageoptions/) & `object` - -#### Returns - -`Promise`\<`undefined` \| \{ `path`: `string`; \}\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`savePackage`](/reference/tableplugin/#savepackage) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: database/build/plugin.d.ts:14 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/DatahubPlugin.md b/portal/typescript/content/docs/reference/DatahubPlugin.md deleted file mode 100644 index ab11d86a..00000000 --- a/portal/typescript/content/docs/reference/DatahubPlugin.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DatahubPlugin" ---- - -Defined in: dataset/build/plugins/datahub/plugin.d.ts:2 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new DatahubPlugin**(): `DatahubPlugin` - -#### Returns - -`DatahubPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/datahub/plugin.d.ts:3 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) diff --git a/portal/typescript/content/docs/reference/DatasetPlugin.md b/portal/typescript/content/docs/reference/DatasetPlugin.md deleted file mode 100644 index 828c8fca..00000000 --- a/portal/typescript/content/docs/reference/DatasetPlugin.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DatasetPlugin" ---- - -Defined in: dataset/build/plugin.d.ts:6 - -## Extended by - -- [`TablePlugin`](/reference/tableplugin/) - -## Methods - -### loadPackage()? - -> `optional` **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: dataset/build/plugin.d.ts:7 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -*** - -### savePackage()? - -> `optional` **savePackage**(`dataPackage`, `options`): `Promise`\<`undefined` \| \{ `path?`: `string`; \}\> - -Defined in: dataset/build/plugin.d.ts:8 - -#### Parameters - -##### dataPackage - -[`Package`](/reference/package/) - -##### options - -[`SavePackageOptions`](/reference/savepackageoptions/) - -#### Returns - -`Promise`\<`undefined` \| \{ `path?`: `string`; \}\> diff --git a/portal/typescript/content/docs/reference/DateConstraints.md b/portal/typescript/content/docs/reference/DateConstraints.md deleted file mode 100644 index bd06f728..00000000 --- a/portal/typescript/content/docs/reference/DateConstraints.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DateConstraints" ---- - -Defined in: metadata/build/field/types/Date.d.ts:21 - -Date-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] - -Defined in: metadata/build/field/types/Date.d.ts:42 - -Restrict values to a specified set of dates -Should be in string date format (e.g., "YYYY-MM-DD") - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` - -Defined in: metadata/build/field/types/Date.d.ts:37 - -Exclusive maximum date value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` - -Defined in: metadata/build/field/types/Date.d.ts:33 - -Exclusive minimum date value - -*** - -### maximum? - -> `optional` **maximum**: `string` - -Defined in: metadata/build/field/types/Date.d.ts:29 - -Maximum allowed date value - -*** - -### minimum? - -> `optional` **minimum**: `string` - -Defined in: metadata/build/field/types/Date.d.ts:25 - -Minimum allowed date value - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/DateField.md b/portal/typescript/content/docs/reference/DateField.md deleted file mode 100644 index e4341a85..00000000 --- a/portal/typescript/content/docs/reference/DateField.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DateField" ---- - -Defined in: metadata/build/field/types/Date.d.ts:5 - -Date field type - -## Extends - -- `BaseField`\<[`DateConstraints`](/reference/dateconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`DateConstraints`](/reference/dateconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Date.d.ts:16 - -Format of the date -- default: YYYY-MM-DD -- any: flexible date parsing (not recommended) -- Or custom strptime/strftime format string - -#### Overrides - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"date"` - -Defined in: metadata/build/field/types/Date.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/DatetimeConstraints.md b/portal/typescript/content/docs/reference/DatetimeConstraints.md deleted file mode 100644 index 11295ee8..00000000 --- a/portal/typescript/content/docs/reference/DatetimeConstraints.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DatetimeConstraints" ---- - -Defined in: metadata/build/field/types/Datetime.d.ts:21 - -Datetime-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] - -Defined in: metadata/build/field/types/Datetime.d.ts:42 - -Restrict values to a specified set of datetimes -Should be in string datetime format (e.g., ISO8601) - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` - -Defined in: metadata/build/field/types/Datetime.d.ts:37 - -Exclusive maximum datetime value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` - -Defined in: metadata/build/field/types/Datetime.d.ts:33 - -Exclusive minimum datetime value - -*** - -### maximum? - -> `optional` **maximum**: `string` - -Defined in: metadata/build/field/types/Datetime.d.ts:29 - -Maximum allowed datetime value - -*** - -### minimum? - -> `optional` **minimum**: `string` - -Defined in: metadata/build/field/types/Datetime.d.ts:25 - -Minimum allowed datetime value - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/DatetimeField.md b/portal/typescript/content/docs/reference/DatetimeField.md deleted file mode 100644 index 6d6840e9..00000000 --- a/portal/typescript/content/docs/reference/DatetimeField.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DatetimeField" ---- - -Defined in: metadata/build/field/types/Datetime.d.ts:5 - -Datetime field type - -## Extends - -- `BaseField`\<[`DatetimeConstraints`](/reference/datetimeconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`DatetimeConstraints`](/reference/datetimeconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Datetime.d.ts:16 - -Format of the datetime -- default: ISO8601 format -- any: flexible datetime parsing (not recommended) -- Or custom strptime/strftime format string - -#### Overrides - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"datetime"` - -Defined in: metadata/build/field/types/Datetime.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/DenormalizeFieldOptions.md b/portal/typescript/content/docs/reference/DenormalizeFieldOptions.md deleted file mode 100644 index 02f77c1e..00000000 --- a/portal/typescript/content/docs/reference/DenormalizeFieldOptions.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DenormalizeFieldOptions" ---- - -> **DenormalizeFieldOptions** = `object` - -Defined in: table/build/field/denormalize.d.ts:3 - -## Properties - -### nativeTypes? - -> `optional` **nativeTypes**: `Exclude`\<[`Field`](/reference/field/)\[`"type"`\], `undefined`\>[] - -Defined in: table/build/field/denormalize.d.ts:4 diff --git a/portal/typescript/content/docs/reference/Descriptor.md b/portal/typescript/content/docs/reference/Descriptor.md deleted file mode 100644 index 6dc76a84..00000000 --- a/portal/typescript/content/docs/reference/Descriptor.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Descriptor" ---- - -> **Descriptor** = `Record`\<`string`, `unknown`\> - -Defined in: metadata/build/descriptor/Descriptor.d.ts:1 diff --git a/portal/typescript/content/docs/reference/DescriptorPlugin.md b/portal/typescript/content/docs/reference/DescriptorPlugin.md deleted file mode 100644 index 2bd17057..00000000 --- a/portal/typescript/content/docs/reference/DescriptorPlugin.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DescriptorPlugin" ---- - -Defined in: dataset/build/plugins/descriptor/plugin.d.ts:3 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new DescriptorPlugin**(): `DescriptorPlugin` - -#### Returns - -`DescriptorPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/descriptor/plugin.d.ts:4 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) - -*** - -### savePackage() - -> **savePackage**(`dataPackage`, `options`): `Promise`\<`undefined` \| \{ `path`: `string`; \}\> - -Defined in: dataset/build/plugins/descriptor/plugin.d.ts:5 - -#### Parameters - -##### dataPackage - -[`Package`](/reference/package/) - -##### options - -###### target - -`string` - -###### withRemote? - -`boolean` - -#### Returns - -`Promise`\<`undefined` \| \{ `path`: `string`; \}\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`savePackage`](/reference/datasetplugin/#savepackage) diff --git a/portal/typescript/content/docs/reference/Dialect.md b/portal/typescript/content/docs/reference/Dialect.md deleted file mode 100644 index b668080f..00000000 --- a/portal/typescript/content/docs/reference/Dialect.md +++ /dev/null @@ -1,221 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Dialect" ---- - -Defined in: metadata/build/dialect/Dialect.d.ts:7 - -Descriptor that describes the structure of tabular data, such as delimiters, -headers, and other features. Following the Data Package standard: -https://datapackage.org/standard/table-dialect/ - -## Extends - -- `Metadata` - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### $schema? - -> `optional` **$schema**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:11 - -JSON schema profile URL for validation - -*** - -### commentChar? - -> `optional` **commentChar**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:35 - -Character sequence denoting the start of a comment line - -*** - -### commentRows? - -> `optional` **commentRows**: `number`[] - -Defined in: metadata/build/dialect/Dialect.d.ts:31 - -Specific rows to be excluded from the data (zero-based) - -*** - -### delimiter? - -> `optional` **delimiter**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:39 - -The character used to separate fields in the data - -*** - -### doubleQuote? - -> `optional` **doubleQuote**: `boolean` - -Defined in: metadata/build/dialect/Dialect.d.ts:51 - -Controls whether a sequence of two quote characters represents a single quote - -*** - -### escapeChar? - -> `optional` **escapeChar**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:55 - -Character used to escape the delimiter or quote characters - -*** - -### header? - -> `optional` **header**: `boolean` - -Defined in: metadata/build/dialect/Dialect.d.ts:19 - -Whether the file includes a header row with field names - -*** - -### headerJoin? - -> `optional` **headerJoin**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:27 - -The character used to join multi-line headers - -*** - -### headerRows? - -> `optional` **headerRows**: `number`[] - -Defined in: metadata/build/dialect/Dialect.d.ts:23 - -Row numbers (zero-based) that are considered header rows - -*** - -### itemKeys? - -> `optional` **itemKeys**: `string`[] - -Defined in: metadata/build/dialect/Dialect.d.ts:76 - -For object-based data items, specifies which object properties to extract as values - -*** - -### itemType? - -> `optional` **itemType**: `"object"` \| `"array"` - -Defined in: metadata/build/dialect/Dialect.d.ts:72 - -The type of data item in the source: 'array' for rows represented as arrays, -or 'object' for rows represented as objects - -*** - -### lineTerminator? - -> `optional` **lineTerminator**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:43 - -Character sequence used to terminate rows - -*** - -### name? - -> `optional` **name**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:15 - -The name of this dialect - -*** - -### nullSequence? - -> `optional` **nullSequence**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:59 - -Character sequence representing null or missing values in the data - -*** - -### property? - -> `optional` **property**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:67 - -For JSON data, the property name containing the data array - -*** - -### quoteChar? - -> `optional` **quoteChar**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:47 - -Character used to quote fields - -*** - -### sheetName? - -> `optional` **sheetName**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:84 - -For spreadsheet data, the sheet name to read - -*** - -### sheetNumber? - -> `optional` **sheetNumber**: `number` - -Defined in: metadata/build/dialect/Dialect.d.ts:80 - -For spreadsheet data, the sheet number to read (zero-based) - -*** - -### skipInitialSpace? - -> `optional` **skipInitialSpace**: `boolean` - -Defined in: metadata/build/dialect/Dialect.d.ts:63 - -Whether to ignore whitespace immediately following the delimiter - -*** - -### table? - -> `optional` **table**: `string` - -Defined in: metadata/build/dialect/Dialect.d.ts:88 - -For database sources, the table name to read diff --git a/portal/typescript/content/docs/reference/DialectOptions.md b/portal/typescript/content/docs/reference/DialectOptions.md deleted file mode 100644 index 8f444c45..00000000 --- a/portal/typescript/content/docs/reference/DialectOptions.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DialectOptions" ---- - -Defined in: table/build/dialect/Options.d.ts:1 - -## Extended by - -- [`InferDialectOptions`](/reference/inferdialectoptions/) - -## Properties - -### delimiter? - -> `optional` **delimiter**: `string` - -Defined in: table/build/dialect/Options.d.ts:2 diff --git a/portal/typescript/content/docs/reference/Document.md b/portal/typescript/content/docs/reference/Document.md deleted file mode 100644 index 90a826f3..00000000 --- a/portal/typescript/content/docs/reference/Document.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Document" ---- - -> **Document** = [`JsonDocument`](/reference/jsondocument/) - -Defined in: document/build/document/Document.d.ts:2 diff --git a/portal/typescript/content/docs/reference/DocumentError.md b/portal/typescript/content/docs/reference/DocumentError.md deleted file mode 100644 index b41fb13a..00000000 --- a/portal/typescript/content/docs/reference/DocumentError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DocumentError" ---- - -> **DocumentError** = [`JsonDocumentError`](/reference/jsondocumenterror/) - -Defined in: metadata/build/error/types/Document.d.ts:2 diff --git a/portal/typescript/content/docs/reference/DpkitError.md b/portal/typescript/content/docs/reference/DpkitError.md deleted file mode 100644 index 3a54048c..00000000 --- a/portal/typescript/content/docs/reference/DpkitError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DpkitError" ---- - -> **DpkitError** = [`BoundError`](/reference/bounderror/) \| [`UnboundError`](/reference/unbounderror/) - -Defined in: metadata/build/error/Error.d.ts:3 diff --git a/portal/typescript/content/docs/reference/DurationConstraints.md b/portal/typescript/content/docs/reference/DurationConstraints.md deleted file mode 100644 index fb775f11..00000000 --- a/portal/typescript/content/docs/reference/DurationConstraints.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DurationConstraints" ---- - -Defined in: metadata/build/field/types/Duration.d.ts:14 - -Duration-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] - -Defined in: metadata/build/field/types/Duration.d.ts:27 - -Restrict values to a specified set of durations -Should be in ISO 8601 duration format - -*** - -### maximum? - -> `optional` **maximum**: `string` - -Defined in: metadata/build/field/types/Duration.d.ts:22 - -Maximum allowed duration (ISO 8601 format) - -*** - -### minimum? - -> `optional` **minimum**: `string` - -Defined in: metadata/build/field/types/Duration.d.ts:18 - -Minimum allowed duration (ISO 8601 format) - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/DurationField.md b/portal/typescript/content/docs/reference/DurationField.md deleted file mode 100644 index 5742cdb6..00000000 --- a/portal/typescript/content/docs/reference/DurationField.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "DurationField" ---- - -Defined in: metadata/build/field/types/Duration.d.ts:5 - -Duration field type (ISO 8601 duration) - -## Extends - -- `BaseField`\<[`DurationConstraints`](/reference/durationconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`DurationConstraints`](/reference/durationconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"duration"` - -Defined in: metadata/build/field/types/Duration.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/EncodingError.md b/portal/typescript/content/docs/reference/EncodingError.md deleted file mode 100644 index c4f3543d..00000000 --- a/portal/typescript/content/docs/reference/EncodingError.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "EncodingError" ---- - -Defined in: metadata/build/error/types/Encoding.d.ts:2 - -## Extends - -- `BaseError` - -## Properties - -### actualEncoding - -> **actualEncoding**: `string` - -Defined in: metadata/build/error/types/Encoding.d.ts:5 - -*** - -### encoding - -> **encoding**: `string` - -Defined in: metadata/build/error/types/Encoding.d.ts:4 - -*** - -### type - -> **type**: `"file/encoding"` - -Defined in: metadata/build/error/types/Encoding.d.ts:3 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/Field.md b/portal/typescript/content/docs/reference/Field.md deleted file mode 100644 index c4137aa5..00000000 --- a/portal/typescript/content/docs/reference/Field.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Field" ---- - -> **Field** = [`StringField`](/reference/stringfield/) \| [`NumberField`](/reference/numberfield/) \| [`IntegerField`](/reference/integerfield/) \| [`BooleanField`](/reference/booleanfield/) \| [`ObjectField`](/reference/objectfield/) \| [`ArrayField`](/reference/arrayfield/) \| [`ListField`](/reference/listfield/) \| [`DateField`](/reference/datefield/) \| [`TimeField`](/reference/timefield/) \| [`DatetimeField`](/reference/datetimefield/) \| [`YearField`](/reference/yearfield/) \| [`YearmonthField`](/reference/yearmonthfield/) \| [`DurationField`](/reference/durationfield/) \| [`GeopointField`](/reference/geopointfield/) \| [`GeojsonField`](/reference/geojsonfield/) \| [`AnyField`](/reference/anyfield/) - -Defined in: metadata/build/field/Field.d.ts:5 - -A Table Schema field diff --git a/portal/typescript/content/docs/reference/FieldError.md b/portal/typescript/content/docs/reference/FieldError.md deleted file mode 100644 index a296fcf2..00000000 --- a/portal/typescript/content/docs/reference/FieldError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldError" ---- - -> **FieldError** = [`FieldNameError`](/reference/fieldnameerror/) \| [`FieldTypeError`](/reference/fieldtypeerror/) - -Defined in: metadata/build/error/types/Field.d.ts:3 diff --git a/portal/typescript/content/docs/reference/FieldNameError.md b/portal/typescript/content/docs/reference/FieldNameError.md deleted file mode 100644 index 8a653742..00000000 --- a/portal/typescript/content/docs/reference/FieldNameError.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldNameError" ---- - -Defined in: metadata/build/error/types/Field.d.ts:7 - -## Extends - -- [`BaseFieldError`](/reference/basefielderror/) - -## Properties - -### actualFieldName - -> **actualFieldName**: `string` - -Defined in: metadata/build/error/types/Field.d.ts:9 - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Field.d.ts:5 - -#### Inherited from - -[`BaseFieldError`](/reference/basefielderror/).[`fieldName`](/reference/basefielderror/#fieldname) - -*** - -### type - -> **type**: `"field/name"` - -Defined in: metadata/build/error/types/Field.d.ts:8 - -#### Overrides - -[`BaseFieldError`](/reference/basefielderror/).[`type`](/reference/basefielderror/#type) diff --git a/portal/typescript/content/docs/reference/FieldType.md b/portal/typescript/content/docs/reference/FieldType.md deleted file mode 100644 index aed8353f..00000000 --- a/portal/typescript/content/docs/reference/FieldType.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldType" ---- - -> **FieldType** = `Exclude`\<[`Field`](/reference/field/)\[`"type"`\], `undefined`\> - -Defined in: metadata/build/field/Type.d.ts:2 diff --git a/portal/typescript/content/docs/reference/FieldTypeError.md b/portal/typescript/content/docs/reference/FieldTypeError.md deleted file mode 100644 index 5e5c6a12..00000000 --- a/portal/typescript/content/docs/reference/FieldTypeError.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldTypeError" ---- - -Defined in: metadata/build/error/types/Field.d.ts:11 - -## Extends - -- [`BaseFieldError`](/reference/basefielderror/) - -## Properties - -### actualFieldType - -> **actualFieldType**: [`FieldType`](/reference/fieldtype/) - -Defined in: metadata/build/error/types/Field.d.ts:14 - -*** - -### fieldName - -> **fieldName**: `string` - -Defined in: metadata/build/error/types/Field.d.ts:5 - -#### Inherited from - -[`BaseFieldError`](/reference/basefielderror/).[`fieldName`](/reference/basefielderror/#fieldname) - -*** - -### fieldType - -> **fieldType**: [`FieldType`](/reference/fieldtype/) - -Defined in: metadata/build/error/types/Field.d.ts:13 - -*** - -### type - -> **type**: `"field/type"` - -Defined in: metadata/build/error/types/Field.d.ts:12 - -#### Overrides - -[`BaseFieldError`](/reference/basefielderror/).[`type`](/reference/basefielderror/#type) diff --git a/portal/typescript/content/docs/reference/FieldsError.md b/portal/typescript/content/docs/reference/FieldsError.md deleted file mode 100644 index 4894b796..00000000 --- a/portal/typescript/content/docs/reference/FieldsError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldsError" ---- - -> **FieldsError** = [`FieldsMissingError`](/reference/fieldsmissingerror/) \| [`FieldsExtraError`](/reference/fieldsextraerror/) - -Defined in: metadata/build/error/types/Fields.d.ts:2 diff --git a/portal/typescript/content/docs/reference/FieldsExtraError.md b/portal/typescript/content/docs/reference/FieldsExtraError.md deleted file mode 100644 index f395380c..00000000 --- a/portal/typescript/content/docs/reference/FieldsExtraError.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldsExtraError" ---- - -Defined in: metadata/build/error/types/Fields.d.ts:9 - -## Extends - -- [`BaseFieldsError`](/reference/basefieldserror/) - -## Properties - -### fieldNames - -> **fieldNames**: `string`[] - -Defined in: metadata/build/error/types/Fields.d.ts:4 - -#### Inherited from - -[`BaseFieldsError`](/reference/basefieldserror/).[`fieldNames`](/reference/basefieldserror/#fieldnames) - -*** - -### type - -> **type**: `"fields/extra"` - -Defined in: metadata/build/error/types/Fields.d.ts:10 - -#### Overrides - -[`BaseFieldsError`](/reference/basefieldserror/).[`type`](/reference/basefieldserror/#type) diff --git a/portal/typescript/content/docs/reference/FieldsMissingError.md b/portal/typescript/content/docs/reference/FieldsMissingError.md deleted file mode 100644 index 98d00e22..00000000 --- a/portal/typescript/content/docs/reference/FieldsMissingError.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FieldsMissingError" ---- - -Defined in: metadata/build/error/types/Fields.d.ts:6 - -## Extends - -- [`BaseFieldsError`](/reference/basefieldserror/) - -## Properties - -### fieldNames - -> **fieldNames**: `string`[] - -Defined in: metadata/build/error/types/Fields.d.ts:4 - -#### Inherited from - -[`BaseFieldsError`](/reference/basefieldserror/).[`fieldNames`](/reference/basefieldserror/#fieldnames) - -*** - -### type - -> **type**: `"fields/missing"` - -Defined in: metadata/build/error/types/Fields.d.ts:7 - -#### Overrides - -[`BaseFieldsError`](/reference/basefieldserror/).[`type`](/reference/basefieldserror/#type) diff --git a/portal/typescript/content/docs/reference/FileError.md b/portal/typescript/content/docs/reference/FileError.md deleted file mode 100644 index 818f0481..00000000 --- a/portal/typescript/content/docs/reference/FileError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FileError" ---- - -> **FileError** = [`BytesError`](/reference/byteserror/) \| [`HashError`](/reference/hasherror/) \| [`EncodingError`](/reference/encodingerror/) - -Defined in: metadata/build/error/types/File.d.ts:4 diff --git a/portal/typescript/content/docs/reference/FolderPlugin.md b/portal/typescript/content/docs/reference/FolderPlugin.md deleted file mode 100644 index eead27fe..00000000 --- a/portal/typescript/content/docs/reference/FolderPlugin.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "FolderPlugin" ---- - -Defined in: dataset/build/plugins/folder/plugin.d.ts:2 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new FolderPlugin**(): `FolderPlugin` - -#### Returns - -`FolderPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/folder/plugin.d.ts:3 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) diff --git a/portal/typescript/content/docs/reference/ForeignKeyError.md b/portal/typescript/content/docs/reference/ForeignKeyError.md deleted file mode 100644 index bec64e3b..00000000 --- a/portal/typescript/content/docs/reference/ForeignKeyError.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ForeignKeyError" ---- - -Defined in: metadata/build/error/types/ForeignKey.d.ts:3 - -## Extends - -- `BaseError` - -## Properties - -### cells - -> **cells**: `string`[] - -Defined in: metadata/build/error/types/ForeignKey.d.ts:6 - -*** - -### foreignKey - -> **foreignKey**: `ForeignKey` - -Defined in: metadata/build/error/types/ForeignKey.d.ts:5 - -*** - -### type - -> **type**: `"foreignKey"` - -Defined in: metadata/build/error/types/ForeignKey.d.ts:4 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/Frame.md b/portal/typescript/content/docs/reference/Frame.md deleted file mode 100644 index bc69d5f1..00000000 --- a/portal/typescript/content/docs/reference/Frame.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Frame" ---- - -> **Frame** = `pl.DataFrame` - -Defined in: table/build/table/Frame.d.ts:2 diff --git a/portal/typescript/content/docs/reference/GeojsonConstraints.md b/portal/typescript/content/docs/reference/GeojsonConstraints.md deleted file mode 100644 index 8628e373..00000000 --- a/portal/typescript/content/docs/reference/GeojsonConstraints.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "GeojsonConstraints" ---- - -Defined in: metadata/build/field/types/Geojson.d.ts:20 - -GeoJSON-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `Record`\<`string`, `any`\>[] - -Defined in: metadata/build/field/types/Geojson.d.ts:29 - -Restrict values to a specified set of GeoJSON objects -Serialized as strings or GeoJSON object literals - -*** - -### jsonSchema? - -> `optional` **jsonSchema**: `Record`\<`string`, `any`\> - -Defined in: metadata/build/field/types/Geojson.d.ts:24 - -JSON Schema object for validating the object structure and properties - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/GeojsonField.md b/portal/typescript/content/docs/reference/GeojsonField.md deleted file mode 100644 index f2740189..00000000 --- a/portal/typescript/content/docs/reference/GeojsonField.md +++ /dev/null @@ -1,158 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "GeojsonField" ---- - -Defined in: metadata/build/field/types/Geojson.d.ts:5 - -GeoJSON field type - -## Extends - -- `BaseField`\<[`GeojsonConstraints`](/reference/geojsonconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`GeojsonConstraints`](/reference/geojsonconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `"default"` \| `"topojson"` - -Defined in: metadata/build/field/types/Geojson.d.ts:15 - -Format of the geojson -- default: standard GeoJSON -- topojson: TopoJSON format - -#### Overrides - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"geojson"` - -Defined in: metadata/build/field/types/Geojson.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/GeopointConstraints.md b/portal/typescript/content/docs/reference/GeopointConstraints.md deleted file mode 100644 index 20fa54d5..00000000 --- a/portal/typescript/content/docs/reference/GeopointConstraints.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "GeopointConstraints" ---- - -Defined in: metadata/build/field/types/Geopoint.d.ts:21 - -Geopoint-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `number`[][] \| `Record`\<`string`, `number`\>[] - -Defined in: metadata/build/field/types/Geopoint.d.ts:26 - -Restrict values to a specified set of geopoints -Format depends on the field's format setting - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/GeopointField.md b/portal/typescript/content/docs/reference/GeopointField.md deleted file mode 100644 index 7bf9c1d3..00000000 --- a/portal/typescript/content/docs/reference/GeopointField.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "GeopointField" ---- - -Defined in: metadata/build/field/types/Geopoint.d.ts:5 - -Geopoint field type - -## Extends - -- `BaseField`\<[`GeopointConstraints`](/reference/geopointconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`GeopointConstraints`](/reference/geopointconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `"object"` \| `"default"` \| `"array"` - -Defined in: metadata/build/field/types/Geopoint.d.ts:16 - -Format of the geopoint -- default: "lon,lat" string with comma separator -- array: [lon,lat] array -- object: {lon:x, lat:y} object - -#### Overrides - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"geopoint"` - -Defined in: metadata/build/field/types/Geopoint.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/GithubPlugin.md b/portal/typescript/content/docs/reference/GithubPlugin.md deleted file mode 100644 index 138437c1..00000000 --- a/portal/typescript/content/docs/reference/GithubPlugin.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "GithubPlugin" ---- - -Defined in: dataset/build/plugins/github/plugin.d.ts:2 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new GithubPlugin**(): `GithubPlugin` - -#### Returns - -`GithubPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| \{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: dataset/build/plugins/github/plugin.d.ts:3 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| \{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) diff --git a/portal/typescript/content/docs/reference/HashError.md b/portal/typescript/content/docs/reference/HashError.md deleted file mode 100644 index 87003e9e..00000000 --- a/portal/typescript/content/docs/reference/HashError.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "HashError" ---- - -Defined in: metadata/build/error/types/Hash.d.ts:2 - -## Extends - -- `BaseError` - -## Properties - -### actualHash - -> **actualHash**: `string` - -Defined in: metadata/build/error/types/Hash.d.ts:5 - -*** - -### hash - -> **hash**: `string` - -Defined in: metadata/build/error/types/Hash.d.ts:4 - -*** - -### type - -> **type**: `"file/hash"` - -Defined in: metadata/build/error/types/Hash.d.ts:3 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/InferDialectOptions.md b/portal/typescript/content/docs/reference/InferDialectOptions.md deleted file mode 100644 index a2cbe2bd..00000000 --- a/portal/typescript/content/docs/reference/InferDialectOptions.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "InferDialectOptions" ---- - -Defined in: table/build/dialect/infer.d.ts:2 - -## Extends - -- [`DialectOptions`](/reference/dialectoptions/) - -## Properties - -### delimiter? - -> `optional` **delimiter**: `string` - -Defined in: table/build/dialect/Options.d.ts:2 - -#### Inherited from - -[`DialectOptions`](/reference/dialectoptions/).[`delimiter`](/reference/dialectoptions/#delimiter) - -*** - -### sampleBytes? - -> `optional` **sampleBytes**: `number` - -Defined in: table/build/dialect/infer.d.ts:3 diff --git a/portal/typescript/content/docs/reference/InferSchemaOptions.md b/portal/typescript/content/docs/reference/InferSchemaOptions.md deleted file mode 100644 index 19864f9b..00000000 --- a/portal/typescript/content/docs/reference/InferSchemaOptions.md +++ /dev/null @@ -1,256 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "InferSchemaOptions" ---- - -Defined in: table/build/schema/infer.d.ts:5 - -## Extends - -- [`SchemaOptions`](/reference/schemaoptions/) - -## Properties - -### arrayType? - -> `optional` **arrayType**: `"array"` \| `"list"` - -Defined in: table/build/schema/Options.d.ts:17 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`arrayType`](/reference/schemaoptions/#arraytype) - -*** - -### bareNumber? - -> `optional` **bareNumber**: `boolean` - -Defined in: table/build/schema/Options.d.ts:11 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`bareNumber`](/reference/schemaoptions/#barenumber) - -*** - -### commaDecimal? - -> `optional` **commaDecimal**: `boolean` - -Defined in: table/build/schema/infer.d.ts:8 - -*** - -### confidence? - -> `optional` **confidence**: `number` - -Defined in: table/build/schema/infer.d.ts:7 - -*** - -### dateFormat? - -> `optional` **dateFormat**: `string` - -Defined in: table/build/schema/Options.d.ts:15 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`dateFormat`](/reference/schemaoptions/#dateformat) - -*** - -### datetimeFormat? - -> `optional` **datetimeFormat**: `string` - -Defined in: table/build/schema/Options.d.ts:14 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`datetimeFormat`](/reference/schemaoptions/#datetimeformat) - -*** - -### decimalChar? - -> `optional` **decimalChar**: `string` - -Defined in: table/build/schema/Options.d.ts:9 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`decimalChar`](/reference/schemaoptions/#decimalchar) - -*** - -### falseValues? - -> `optional` **falseValues**: `string`[] - -Defined in: table/build/schema/Options.d.ts:13 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`falseValues`](/reference/schemaoptions/#falsevalues) - -*** - -### fieldNames? - -> `optional` **fieldNames**: `string`[] - -Defined in: table/build/schema/Options.d.ts:5 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`fieldNames`](/reference/schemaoptions/#fieldnames) - -*** - -### fieldTypes? - -> `optional` **fieldTypes**: `Record`\<`string`, [`FieldType`](/reference/fieldtype/)\> - -Defined in: table/build/schema/Options.d.ts:6 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`fieldTypes`](/reference/schemaoptions/#fieldtypes) - -*** - -### geojsonFormat? - -> `optional` **geojsonFormat**: `"default"` \| `"topojson"` - -Defined in: table/build/schema/Options.d.ts:21 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`geojsonFormat`](/reference/schemaoptions/#geojsonformat) - -*** - -### geopointFormat? - -> `optional` **geopointFormat**: `"object"` \| `"default"` \| `"array"` - -Defined in: table/build/schema/Options.d.ts:20 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`geopointFormat`](/reference/schemaoptions/#geopointformat) - -*** - -### groupChar? - -> `optional` **groupChar**: `string` - -Defined in: table/build/schema/Options.d.ts:10 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`groupChar`](/reference/schemaoptions/#groupchar) - -*** - -### keepStrings? - -> `optional` **keepStrings**: `boolean` - -Defined in: table/build/schema/infer.d.ts:10 - -*** - -### listDelimiter? - -> `optional` **listDelimiter**: `string` - -Defined in: table/build/schema/Options.d.ts:18 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`listDelimiter`](/reference/schemaoptions/#listdelimiter) - -*** - -### listItemType? - -> `optional` **listItemType**: `"string"` \| `"number"` \| `"boolean"` \| `"integer"` \| `"date"` \| `"time"` \| `"datetime"` - -Defined in: table/build/schema/Options.d.ts:19 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`listItemType`](/reference/schemaoptions/#listitemtype) - -*** - -### missingValues? - -> `optional` **missingValues**: `string`[] - -Defined in: table/build/schema/Options.d.ts:7 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`missingValues`](/reference/schemaoptions/#missingvalues) - -*** - -### monthFirst? - -> `optional` **monthFirst**: `boolean` - -Defined in: table/build/schema/infer.d.ts:9 - -*** - -### sampleRows? - -> `optional` **sampleRows**: `number` - -Defined in: table/build/schema/infer.d.ts:6 - -*** - -### stringFormat? - -> `optional` **stringFormat**: `"email"` \| `"uri"` \| `"binary"` \| `"uuid"` - -Defined in: table/build/schema/Options.d.ts:8 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`stringFormat`](/reference/schemaoptions/#stringformat) - -*** - -### timeFormat? - -> `optional` **timeFormat**: `string` - -Defined in: table/build/schema/Options.d.ts:16 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`timeFormat`](/reference/schemaoptions/#timeformat) - -*** - -### trueValues? - -> `optional` **trueValues**: `string`[] - -Defined in: table/build/schema/Options.d.ts:12 - -#### Inherited from - -[`SchemaOptions`](/reference/schemaoptions/).[`trueValues`](/reference/schemaoptions/#truevalues) diff --git a/portal/typescript/content/docs/reference/InlinePlugin.md b/portal/typescript/content/docs/reference/InlinePlugin.md deleted file mode 100644 index c7bb8ecd..00000000 --- a/portal/typescript/content/docs/reference/InlinePlugin.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "InlinePlugin" ---- - -Defined in: table/build/plugins/inline/plugin.d.ts:3 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new InlinePlugin**(): `InlinePlugin` - -#### Returns - -`InlinePlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<\{\[`x`: `string`\]: `any`; \}\>\> - -Defined in: table/build/plugins/inline/plugin.d.ts:4 - -#### Parameters - -##### resource - -[`Resource`](/reference/resource/) - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<\{\[`x`: `string`\]: `any`; \}\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) diff --git a/portal/typescript/content/docs/reference/IntegerConstraints.md b/portal/typescript/content/docs/reference/IntegerConstraints.md deleted file mode 100644 index 33fee678..00000000 --- a/portal/typescript/content/docs/reference/IntegerConstraints.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "IntegerConstraints" ---- - -Defined in: metadata/build/field/types/Integer.d.ts:35 - -**`Internal`** - -Integer-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `number`[] - -Defined in: metadata/build/field/types/Integer.d.ts:56 - -Restrict values to a specified set -Can be an array of integers or strings that parse to integers - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` \| `number` - -Defined in: metadata/build/field/types/Integer.d.ts:51 - -Exclusive maximum allowed value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` \| `number` - -Defined in: metadata/build/field/types/Integer.d.ts:47 - -Exclusive minimum allowed value - -*** - -### maximum? - -> `optional` **maximum**: `string` \| `number` - -Defined in: metadata/build/field/types/Integer.d.ts:43 - -Maximum allowed value - -*** - -### minimum? - -> `optional` **minimum**: `string` \| `number` - -Defined in: metadata/build/field/types/Integer.d.ts:39 - -Minimum allowed value - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/IntegerField.md b/portal/typescript/content/docs/reference/IntegerField.md deleted file mode 100644 index 6c509396..00000000 --- a/portal/typescript/content/docs/reference/IntegerField.md +++ /dev/null @@ -1,197 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "IntegerField" ---- - -Defined in: metadata/build/field/types/Integer.d.ts:5 - -Integer field type - -## Extends - -- `BaseField`\<[`IntegerConstraints`](/reference/integerconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### bareNumber? - -> `optional` **bareNumber**: `boolean` - -Defined in: metadata/build/field/types/Integer.d.ts:17 - -Whether number is presented without currency symbols or percent signs - -*** - -### categories? - -> `optional` **categories**: `number`[] \| `object`[] - -Defined in: metadata/build/field/types/Integer.d.ts:22 - -Categories for enum values -Can be an array of values or an array of {value, label} objects - -*** - -### categoriesOrdered? - -> `optional` **categoriesOrdered**: `boolean` - -Defined in: metadata/build/field/types/Integer.d.ts:29 - -Whether categories should be considered to have a natural order - -*** - -### constraints? - -> `optional` **constraints**: [`IntegerConstraints`](/reference/integerconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### groupChar? - -> `optional` **groupChar**: `string` - -Defined in: metadata/build/field/types/Integer.d.ts:13 - -Character used as thousands separator - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"integer"` - -Defined in: metadata/build/field/types/Integer.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/JsonDocument.md b/portal/typescript/content/docs/reference/JsonDocument.md deleted file mode 100644 index 1427cb1c..00000000 --- a/portal/typescript/content/docs/reference/JsonDocument.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "JsonDocument" ---- - -Defined in: document/build/document/types/Json.d.ts:2 - -## Extends - -- `BaseDocument` - -## Properties - -### data - -> **data**: `Record`\<`string`, `any`\> - -Defined in: document/build/document/types/Json.d.ts:4 - -*** - -### type - -> **type**: `"json"` - -Defined in: document/build/document/types/Json.d.ts:3 - -#### Overrides - -`BaseDocument.type` diff --git a/portal/typescript/content/docs/reference/JsonDocumentError.md b/portal/typescript/content/docs/reference/JsonDocumentError.md deleted file mode 100644 index ca20ebb1..00000000 --- a/portal/typescript/content/docs/reference/JsonDocumentError.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "JsonDocumentError" ---- - -Defined in: metadata/build/error/types/Document.d.ts:3 - -## Extends - -- `BaseError` - -## Properties - -### message - -> **message**: `string` - -Defined in: metadata/build/error/types/Document.d.ts:6 - -*** - -### pointer - -> **pointer**: `string` - -Defined in: metadata/build/error/types/Document.d.ts:5 - -*** - -### type - -> **type**: `"document/json"` - -Defined in: metadata/build/error/types/Document.d.ts:4 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/JsonPlugin.md b/portal/typescript/content/docs/reference/JsonPlugin.md deleted file mode 100644 index ef8020ca..00000000 --- a/portal/typescript/content/docs/reference/JsonPlugin.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "JsonPlugin" ---- - -Defined in: table/build/plugins/json/plugin.d.ts:4 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new JsonPlugin**(): `JsonPlugin` - -#### Returns - -`JsonPlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/json/plugin.d.ts:5 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugins/json/plugin.d.ts:6 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/License.md b/portal/typescript/content/docs/reference/License.md deleted file mode 100644 index 4cb5f66c..00000000 --- a/portal/typescript/content/docs/reference/License.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "License" ---- - -Defined in: metadata/build/resource/License.d.ts:4 - -License information - -## Properties - -### name? - -> `optional` **name**: `string` - -Defined in: metadata/build/resource/License.d.ts:9 - -The name of the license - -#### Example - -```ts -"MIT", "Apache-2.0" -``` - -*** - -### path? - -> `optional` **path**: `string` - -Defined in: metadata/build/resource/License.d.ts:13 - -A URL to the license text - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/resource/License.d.ts:17 - -Human-readable title of the license diff --git a/portal/typescript/content/docs/reference/ListConstraints.md b/portal/typescript/content/docs/reference/ListConstraints.md deleted file mode 100644 index f5739b43..00000000 --- a/portal/typescript/content/docs/reference/ListConstraints.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ListConstraints" ---- - -Defined in: metadata/build/field/types/List.d.ts:22 - -List-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `any`[][] - -Defined in: metadata/build/field/types/List.d.ts:35 - -Restrict values to a specified set of lists -Either as delimited strings or arrays - -*** - -### maxLength? - -> `optional` **maxLength**: `number` - -Defined in: metadata/build/field/types/List.d.ts:30 - -Maximum number of list items - -*** - -### minLength? - -> `optional` **minLength**: `number` - -Defined in: metadata/build/field/types/List.d.ts:26 - -Minimum number of list items - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/ListField.md b/portal/typescript/content/docs/reference/ListField.md deleted file mode 100644 index 071a29ea..00000000 --- a/portal/typescript/content/docs/reference/ListField.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ListField" ---- - -Defined in: metadata/build/field/types/List.d.ts:5 - -List field type (primitive values ordered collection) - -## Extends - -- `BaseField`\<[`ListConstraints`](/reference/listconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`ListConstraints`](/reference/listconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### delimiter? - -> `optional` **delimiter**: `string` - -Defined in: metadata/build/field/types/List.d.ts:13 - -Character used to separate values in the list - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### itemType? - -> `optional` **itemType**: `"string"` \| `"number"` \| `"boolean"` \| `"integer"` \| `"date"` \| `"time"` \| `"datetime"` - -Defined in: metadata/build/field/types/List.d.ts:17 - -Type of items in the list - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"list"` - -Defined in: metadata/build/field/types/List.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/LoadTableOptions.md b/portal/typescript/content/docs/reference/LoadTableOptions.md deleted file mode 100644 index 2689f195..00000000 --- a/portal/typescript/content/docs/reference/LoadTableOptions.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "LoadTableOptions" ---- - -> **LoadTableOptions** = [`InferDialectOptions`](/reference/inferdialectoptions/) & [`InferSchemaOptions`](/reference/inferschemaoptions/) & `object` - -Defined in: table/build/plugin.d.ts:7 - -## Type declaration - -### denormalized? - -> `optional` **denormalized**: `boolean` diff --git a/portal/typescript/content/docs/reference/MetadataError.md b/portal/typescript/content/docs/reference/MetadataError.md deleted file mode 100644 index 48eadac4..00000000 --- a/portal/typescript/content/docs/reference/MetadataError.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "MetadataError" ---- - -Defined in: metadata/build/error/types/Metadata.d.ts:5 - -A descriptor error - -## Extends - -- `BaseError` - -## Properties - -### message - -> **message**: `string` - -Defined in: metadata/build/error/types/Metadata.d.ts:8 - -*** - -### pointer - -> **pointer**: `string` - -Defined in: metadata/build/error/types/Metadata.d.ts:7 - -*** - -### type - -> **type**: `"metadata"` - -Defined in: metadata/build/error/types/Metadata.d.ts:6 - -#### Overrides - -`BaseError.type` diff --git a/portal/typescript/content/docs/reference/NumberConstraints.md b/portal/typescript/content/docs/reference/NumberConstraints.md deleted file mode 100644 index c820ec0c..00000000 --- a/portal/typescript/content/docs/reference/NumberConstraints.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "NumberConstraints" ---- - -Defined in: metadata/build/field/types/Number.d.ts:26 - -Number-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `number`[] - -Defined in: metadata/build/field/types/Number.d.ts:47 - -Restrict values to a specified set -Can be an array of numbers or strings that parse to numbers - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` \| `number` - -Defined in: metadata/build/field/types/Number.d.ts:42 - -Exclusive maximum allowed value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` \| `number` - -Defined in: metadata/build/field/types/Number.d.ts:38 - -Exclusive minimum allowed value - -*** - -### maximum? - -> `optional` **maximum**: `string` \| `number` - -Defined in: metadata/build/field/types/Number.d.ts:34 - -Maximum allowed value - -*** - -### minimum? - -> `optional` **minimum**: `string` \| `number` - -Defined in: metadata/build/field/types/Number.d.ts:30 - -Minimum allowed value - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/NumberField.md b/portal/typescript/content/docs/reference/NumberField.md deleted file mode 100644 index 59154f67..00000000 --- a/portal/typescript/content/docs/reference/NumberField.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "NumberField" ---- - -Defined in: metadata/build/field/types/Number.d.ts:5 - -Number field type - -## Extends - -- `BaseField`\<[`NumberConstraints`](/reference/numberconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### bareNumber? - -> `optional` **bareNumber**: `boolean` - -Defined in: metadata/build/field/types/Number.d.ts:21 - -Whether number is presented without currency symbols or percent signs - -*** - -### constraints? - -> `optional` **constraints**: [`NumberConstraints`](/reference/numberconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### decimalChar? - -> `optional` **decimalChar**: `string` - -Defined in: metadata/build/field/types/Number.d.ts:13 - -Character used as decimal separator - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### groupChar? - -> `optional` **groupChar**: `string` - -Defined in: metadata/build/field/types/Number.d.ts:17 - -Character used as thousands separator - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"number"` - -Defined in: metadata/build/field/types/Number.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/ObjectConstraints.md b/portal/typescript/content/docs/reference/ObjectConstraints.md deleted file mode 100644 index 75862775..00000000 --- a/portal/typescript/content/docs/reference/ObjectConstraints.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ObjectConstraints" ---- - -Defined in: metadata/build/field/types/Object.d.ts:14 - -Object-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `Record`\<`string`, `any`\>[] - -Defined in: metadata/build/field/types/Object.d.ts:31 - -Restrict values to a specified set of objects -Serialized as JSON strings or object literals - -*** - -### jsonSchema? - -> `optional` **jsonSchema**: `Record`\<`string`, `any`\> - -Defined in: metadata/build/field/types/Object.d.ts:26 - -JSON Schema object for validating the object structure and properties - -*** - -### maxLength? - -> `optional` **maxLength**: `number` - -Defined in: metadata/build/field/types/Object.d.ts:22 - -Maximum number of properties - -*** - -### minLength? - -> `optional` **minLength**: `number` - -Defined in: metadata/build/field/types/Object.d.ts:18 - -Minimum number of properties - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/ObjectField.md b/portal/typescript/content/docs/reference/ObjectField.md deleted file mode 100644 index a48e8d8b..00000000 --- a/portal/typescript/content/docs/reference/ObjectField.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ObjectField" ---- - -Defined in: metadata/build/field/types/Object.d.ts:5 - -Object field type (serialized JSON object) - -## Extends - -- `BaseField`\<[`ObjectConstraints`](/reference/objectconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`ObjectConstraints`](/reference/objectconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"object"` - -Defined in: metadata/build/field/types/Object.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/OdsPlugin.md b/portal/typescript/content/docs/reference/OdsPlugin.md deleted file mode 100644 index b4fc2526..00000000 --- a/portal/typescript/content/docs/reference/OdsPlugin.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "OdsPlugin" ---- - -Defined in: table/build/plugins/ods/plugin.d.ts:4 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new OdsPlugin**(): `OdsPlugin` - -#### Returns - -`OdsPlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/ods/plugin.d.ts:5 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugins/ods/plugin.d.ts:6 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/Package.md b/portal/typescript/content/docs/reference/Package.md deleted file mode 100644 index 33e7abeb..00000000 --- a/portal/typescript/content/docs/reference/Package.md +++ /dev/null @@ -1,163 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Package" ---- - -Defined in: metadata/build/package/Package.d.ts:8 - -Data Package interface built on top of the Frictionless Data specification - -## See - -https://datapackage.org/standard/data-package/ - -## Extends - -- `Metadata` - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### $schema? - -> `optional` **$schema**: `string` - -Defined in: metadata/build/package/Package.d.ts:12 - -URL of profile (optional) - -*** - -### contributors? - -> `optional` **contributors**: [`Contributor`](/reference/contributor/)[] - -Defined in: metadata/build/package/Package.d.ts:46 - -List of contributors - -*** - -### created? - -> `optional` **created**: `string` - -Defined in: metadata/build/package/Package.d.ts:59 - -Create time of the package - -#### Format - -ISO 8601 format - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/package/Package.d.ts:29 - -A description of the package - -*** - -### homepage? - -> `optional` **homepage**: `string` - -Defined in: metadata/build/package/Package.d.ts:33 - -A URL for the home page of the package - -*** - -### image? - -> `optional` **image**: `string` - -Defined in: metadata/build/package/Package.d.ts:63 - -Package image - -*** - -### keywords? - -> `optional` **keywords**: `string`[] - -Defined in: metadata/build/package/Package.d.ts:54 - -Keywords for the package - -*** - -### licenses? - -> `optional` **licenses**: [`License`](/reference/license/)[] - -Defined in: metadata/build/package/Package.d.ts:42 - -License information - -*** - -### name? - -> `optional` **name**: `string` - -Defined in: metadata/build/package/Package.d.ts:21 - -Unique package identifier -Should use lowercase alphanumeric characters, periods, hyphens, and underscores - -*** - -### resources - -> **resources**: [`Resource`](/reference/resource/)[] - -Defined in: metadata/build/package/Package.d.ts:16 - -Data resources in this package (required) - -*** - -### sources? - -> `optional` **sources**: [`Source`](/reference/source/)[] - -Defined in: metadata/build/package/Package.d.ts:50 - -Data sources for this package - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/package/Package.d.ts:25 - -Human-readable title - -*** - -### version? - -> `optional` **version**: `string` - -Defined in: metadata/build/package/Package.d.ts:38 - -Version of the package using SemVer - -#### Example - -```ts -"1.0.0" -``` diff --git a/portal/typescript/content/docs/reference/ParquetPlugin.md b/portal/typescript/content/docs/reference/ParquetPlugin.md deleted file mode 100644 index cd0c313f..00000000 --- a/portal/typescript/content/docs/reference/ParquetPlugin.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ParquetPlugin" ---- - -Defined in: table/build/plugins/parquet/plugin.d.ts:4 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new ParquetPlugin**(): `ParquetPlugin` - -#### Returns - -`ParquetPlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/parquet/plugin.d.ts:5 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugins/parquet/plugin.d.ts:6 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/Plugin.md b/portal/typescript/content/docs/reference/Plugin.md deleted file mode 100644 index 1c040b3e..00000000 --- a/portal/typescript/content/docs/reference/Plugin.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Plugin" ---- - -> **Plugin** = [`TablePlugin`](/reference/tableplugin/) - -Defined in: library/build/plugin.d.ts:2 diff --git a/portal/typescript/content/docs/reference/PolarsField.md b/portal/typescript/content/docs/reference/PolarsField.md deleted file mode 100644 index 96c5f118..00000000 --- a/portal/typescript/content/docs/reference/PolarsField.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "PolarsField" ---- - -> **PolarsField** = `object` - -Defined in: table/build/field/Field.d.ts:2 - -## Properties - -### name - -> **name**: `string` - -Defined in: table/build/field/Field.d.ts:3 - -*** - -### type - -> **type**: `pl.DataType` - -Defined in: table/build/field/Field.d.ts:4 diff --git a/portal/typescript/content/docs/reference/PolarsSchema.md b/portal/typescript/content/docs/reference/PolarsSchema.md deleted file mode 100644 index affa77e6..00000000 --- a/portal/typescript/content/docs/reference/PolarsSchema.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "PolarsSchema" ---- - -Defined in: table/build/schema/Schema.d.ts:2 - -## Properties - -### fields - -> **fields**: [`PolarsField`](/reference/polarsfield/)[] - -Defined in: table/build/schema/Schema.d.ts:3 diff --git a/portal/typescript/content/docs/reference/README.md b/portal/typescript/content/docs/reference/README.md deleted file mode 100644 index 6ffcb065..00000000 --- a/portal/typescript/content/docs/reference/README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "dpkit" ---- - -# dpkit - -dpkit CLI is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/portal/typescript/content/docs/reference/Report.md b/portal/typescript/content/docs/reference/Report.md deleted file mode 100644 index ab5fbe0c..00000000 --- a/portal/typescript/content/docs/reference/Report.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Report" ---- - -Defined in: metadata/build/report/Report.d.ts:2 - -## Type Parameters - -### T - -`T` *extends* [`DpkitError`](/reference/dpkiterror/) = [`DpkitError`](/reference/dpkiterror/) - -## Properties - -### errors - -> **errors**: `T`[] - -Defined in: metadata/build/report/Report.d.ts:4 - -*** - -### valid - -> **valid**: `boolean` - -Defined in: metadata/build/report/Report.d.ts:3 diff --git a/portal/typescript/content/docs/reference/Resource.md b/portal/typescript/content/docs/reference/Resource.md deleted file mode 100644 index a622aad2..00000000 --- a/portal/typescript/content/docs/reference/Resource.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Resource" ---- - -Defined in: metadata/build/resource/Resource.d.ts:11 - -Data Resource interface built on top of the Data Package standard and Polars DataFrames - -## See - -https://datapackage.org/standard/data-resource/ - -## Extends - -- `Metadata` - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### $schema? - -> `optional` **$schema**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:15 - -JSON schema profile URL for validation - -*** - -### bytes? - -> `optional` **bytes**: `number` - -Defined in: metadata/build/resource/Resource.d.ts:62 - -Size of the file in bytes - -*** - -### data? - -> `optional` **data**: `unknown` - -Defined in: metadata/build/resource/Resource.d.ts:30 - -Inline data content instead of referencing an external file -Either path or data must be provided - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:58 - -A description of the resource - -*** - -### dialect? - -> `optional` **dialect**: `string` \| [`Dialect`](/reference/dialect/) - -Defined in: metadata/build/resource/Resource.d.ts:80 - -Table dialect specification -Describes delimiters, quote characters, etc. - -#### See - -https://datapackage.org/standard/table-dialect/ - -*** - -### encoding? - -> `optional` **encoding**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:50 - -Character encoding of the resource - -#### Default - -```ts -"utf-8" -``` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:40 - -The file format - -#### Example - -```ts -"csv", "json", "xlsx" -``` - -*** - -### hash? - -> `optional` **hash**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:66 - -Hash of the resource data - -*** - -### jsonSchema? - -> `optional` **jsonSchema**: `string` \| [`Descriptor`](/reference/descriptor/) - -Defined in: metadata/build/resource/Resource.d.ts:92 - -Schema for the json data -Describes fields in the json, constraints, etc. - -#### See - -https://json-schema.org/ - -*** - -### licenses? - -> `optional` **licenses**: [`License`](/reference/license/)[] - -Defined in: metadata/build/resource/Resource.d.ts:74 - -License information - -*** - -### mediatype? - -> `optional` **mediatype**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:45 - -The media type of the resource - -#### Example - -```ts -"text/csv", "application/json" -``` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:20 - -Unique resource identifier -Should use lowercase alphanumeric characters, periods, hyphens, and underscores - -*** - -### path? - -> `optional` **path**: `string` \| `string`[] - -Defined in: metadata/build/resource/Resource.d.ts:25 - -A reference to the data itself, can be a path URL or array of paths -Either path or data must be provided - -*** - -### schema? - -> `optional` **schema**: `string` \| [`Schema`](/reference/schema/) - -Defined in: metadata/build/resource/Resource.d.ts:86 - -Schema for the tabular data -Describes fields in the table, constraints, etc. - -#### See - -https://datapackage.org/standard/table-schema/ - -*** - -### sources? - -> `optional` **sources**: [`Source`](/reference/source/)[] - -Defined in: metadata/build/resource/Resource.d.ts:70 - -Data sources - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/resource/Resource.d.ts:54 - -Human-readable title - -*** - -### type? - -> `optional` **type**: `"table"` - -Defined in: metadata/build/resource/Resource.d.ts:35 - -The resource type - -#### Example - -```ts -"table" -``` diff --git a/portal/typescript/content/docs/reference/RowError.md b/portal/typescript/content/docs/reference/RowError.md deleted file mode 100644 index 444da9ec..00000000 --- a/portal/typescript/content/docs/reference/RowError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "RowError" ---- - -> **RowError** = [`RowUniqueError`](/reference/rowuniqueerror/) - -Defined in: metadata/build/error/types/Row.d.ts:2 diff --git a/portal/typescript/content/docs/reference/RowUniqueError.md b/portal/typescript/content/docs/reference/RowUniqueError.md deleted file mode 100644 index e9114041..00000000 --- a/portal/typescript/content/docs/reference/RowUniqueError.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "RowUniqueError" ---- - -Defined in: metadata/build/error/types/Row.d.ts:6 - -## Extends - -- [`BaseRowError`](/reference/baserowerror/) - -## Properties - -### fieldNames - -> **fieldNames**: `string`[] - -Defined in: metadata/build/error/types/Row.d.ts:8 - -*** - -### rowNumber - -> **rowNumber**: `number` - -Defined in: metadata/build/error/types/Row.d.ts:4 - -#### Inherited from - -[`BaseRowError`](/reference/baserowerror/).[`rowNumber`](/reference/baserowerror/#rownumber) - -*** - -### type - -> **type**: `"row/unique"` - -Defined in: metadata/build/error/types/Row.d.ts:7 - -#### Overrides - -[`BaseRowError`](/reference/baserowerror/).[`type`](/reference/baserowerror/#type) diff --git a/portal/typescript/content/docs/reference/SavePackageOptions.md b/portal/typescript/content/docs/reference/SavePackageOptions.md deleted file mode 100644 index 06cc7973..00000000 --- a/portal/typescript/content/docs/reference/SavePackageOptions.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "SavePackageOptions" ---- - -> **SavePackageOptions** = `object` - -Defined in: dataset/build/plugin.d.ts:2 - -## Properties - -### target - -> **target**: `string` - -Defined in: dataset/build/plugin.d.ts:3 - -*** - -### withRemote? - -> `optional` **withRemote**: `boolean` - -Defined in: dataset/build/plugin.d.ts:4 diff --git a/portal/typescript/content/docs/reference/SaveTableOptions.md b/portal/typescript/content/docs/reference/SaveTableOptions.md deleted file mode 100644 index 8ea92420..00000000 --- a/portal/typescript/content/docs/reference/SaveTableOptions.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "SaveTableOptions" ---- - -> **SaveTableOptions** = [`DialectOptions`](/reference/dialectoptions/) & [`SchemaOptions`](/reference/schemaoptions/) & `object` - -Defined in: table/build/plugin.d.ts:10 - -## Type declaration - -### dialect? - -> `optional` **dialect**: [`Dialect`](/reference/dialect/) - -### format? - -> `optional` **format**: `string` - -### overwrite? - -> `optional` **overwrite**: `boolean` - -### path - -> **path**: `string` - -### schema? - -> `optional` **schema**: [`Schema`](/reference/schema/) diff --git a/portal/typescript/content/docs/reference/Schema.md b/portal/typescript/content/docs/reference/Schema.md deleted file mode 100644 index cbe6c0c7..00000000 --- a/portal/typescript/content/docs/reference/Schema.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Schema" ---- - -Defined in: metadata/build/schema/Schema.d.ts:8 - -Table Schema definition -Based on the specification at https://datapackage.org/standard/table-schema/ - -## Extends - -- `Metadata` - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### $schema? - -> `optional` **$schema**: `string` - -Defined in: metadata/build/schema/Schema.d.ts:12 - -URL of profile (optional) - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/schema/Schema.d.ts:24 - -Description of schema (optional) - -*** - -### fields - -> **fields**: [`Field`](/reference/field/)[] - -Defined in: metadata/build/schema/Schema.d.ts:28 - -Fields in this schema (required) - -*** - -### fieldsMatch? - -> `optional` **fieldsMatch**: `"exact"` \| `"equal"` \| `"subset"` \| `"superset"` \| `"partial"` - -Defined in: metadata/build/schema/Schema.d.ts:33 - -Field matching rule (optional) -Default: "exact" - -*** - -### foreignKeys? - -> `optional` **foreignKeys**: `ForeignKey`[] - -Defined in: metadata/build/schema/Schema.d.ts:55 - -Foreign key relationships (optional) - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/schema/Schema.d.ts:40 - -Values representing missing data (optional) -Default: [""] -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -*** - -### name? - -> `optional` **name**: `string` - -Defined in: metadata/build/schema/Schema.d.ts:16 - -Name of schema (optional) - -*** - -### primaryKey? - -> `optional` **primaryKey**: `string`[] - -Defined in: metadata/build/schema/Schema.d.ts:47 - -Fields uniquely identifying each row (optional) - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/schema/Schema.d.ts:20 - -Title of schema (optional) - -*** - -### uniqueKeys? - -> `optional` **uniqueKeys**: `string`[][] - -Defined in: metadata/build/schema/Schema.d.ts:51 - -Field combinations that must be unique (optional) diff --git a/portal/typescript/content/docs/reference/SchemaOptions.md b/portal/typescript/content/docs/reference/SchemaOptions.md deleted file mode 100644 index 39ff5159..00000000 --- a/portal/typescript/content/docs/reference/SchemaOptions.md +++ /dev/null @@ -1,148 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "SchemaOptions" ---- - -Defined in: table/build/schema/Options.d.ts:4 - -## Extended by - -- [`InferSchemaOptions`](/reference/inferschemaoptions/) - -## Properties - -### arrayType? - -> `optional` **arrayType**: `"array"` \| `"list"` - -Defined in: table/build/schema/Options.d.ts:17 - -*** - -### bareNumber? - -> `optional` **bareNumber**: `boolean` - -Defined in: table/build/schema/Options.d.ts:11 - -*** - -### dateFormat? - -> `optional` **dateFormat**: `string` - -Defined in: table/build/schema/Options.d.ts:15 - -*** - -### datetimeFormat? - -> `optional` **datetimeFormat**: `string` - -Defined in: table/build/schema/Options.d.ts:14 - -*** - -### decimalChar? - -> `optional` **decimalChar**: `string` - -Defined in: table/build/schema/Options.d.ts:9 - -*** - -### falseValues? - -> `optional` **falseValues**: `string`[] - -Defined in: table/build/schema/Options.d.ts:13 - -*** - -### fieldNames? - -> `optional` **fieldNames**: `string`[] - -Defined in: table/build/schema/Options.d.ts:5 - -*** - -### fieldTypes? - -> `optional` **fieldTypes**: `Record`\<`string`, [`FieldType`](/reference/fieldtype/)\> - -Defined in: table/build/schema/Options.d.ts:6 - -*** - -### geojsonFormat? - -> `optional` **geojsonFormat**: `"default"` \| `"topojson"` - -Defined in: table/build/schema/Options.d.ts:21 - -*** - -### geopointFormat? - -> `optional` **geopointFormat**: `"object"` \| `"default"` \| `"array"` - -Defined in: table/build/schema/Options.d.ts:20 - -*** - -### groupChar? - -> `optional` **groupChar**: `string` - -Defined in: table/build/schema/Options.d.ts:10 - -*** - -### listDelimiter? - -> `optional` **listDelimiter**: `string` - -Defined in: table/build/schema/Options.d.ts:18 - -*** - -### listItemType? - -> `optional` **listItemType**: `"string"` \| `"number"` \| `"boolean"` \| `"integer"` \| `"date"` \| `"time"` \| `"datetime"` - -Defined in: table/build/schema/Options.d.ts:19 - -*** - -### missingValues? - -> `optional` **missingValues**: `string`[] - -Defined in: table/build/schema/Options.d.ts:7 - -*** - -### stringFormat? - -> `optional` **stringFormat**: `"email"` \| `"uri"` \| `"binary"` \| `"uuid"` - -Defined in: table/build/schema/Options.d.ts:8 - -*** - -### timeFormat? - -> `optional` **timeFormat**: `string` - -Defined in: table/build/schema/Options.d.ts:16 - -*** - -### trueValues? - -> `optional` **trueValues**: `string`[] - -Defined in: table/build/schema/Options.d.ts:12 diff --git a/portal/typescript/content/docs/reference/Source.md b/portal/typescript/content/docs/reference/Source.md deleted file mode 100644 index 936d24b7..00000000 --- a/portal/typescript/content/docs/reference/Source.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Source" ---- - -Defined in: metadata/build/resource/Source.d.ts:4 - -Source information - -## Properties - -### email? - -> `optional` **email**: `string` - -Defined in: metadata/build/resource/Source.d.ts:16 - -Email contact for the source - -*** - -### path? - -> `optional` **path**: `string` - -Defined in: metadata/build/resource/Source.d.ts:12 - -URL or path to the source - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/resource/Source.d.ts:8 - -Human-readable title of the source diff --git a/portal/typescript/content/docs/reference/StringConstraints.md b/portal/typescript/content/docs/reference/StringConstraints.md deleted file mode 100644 index d5b3c8ec..00000000 --- a/portal/typescript/content/docs/reference/StringConstraints.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "StringConstraints" ---- - -Defined in: metadata/build/field/types/String.d.ts:34 - -String-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] - -Defined in: metadata/build/field/types/String.d.ts:50 - -Restrict values to a specified set of strings - -*** - -### maxLength? - -> `optional` **maxLength**: `number` - -Defined in: metadata/build/field/types/String.d.ts:42 - -Maximum string length - -*** - -### minLength? - -> `optional` **minLength**: `number` - -Defined in: metadata/build/field/types/String.d.ts:38 - -Minimum string length - -*** - -### pattern? - -> `optional` **pattern**: `string` - -Defined in: metadata/build/field/types/String.d.ts:46 - -Regular expression pattern to match - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/StringField.md b/portal/typescript/content/docs/reference/StringField.md deleted file mode 100644 index 5d1ec98f..00000000 --- a/portal/typescript/content/docs/reference/StringField.md +++ /dev/null @@ -1,181 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "StringField" ---- - -Defined in: metadata/build/field/types/String.d.ts:5 - -String field type - -## Extends - -- `BaseField`\<[`StringConstraints`](/reference/stringconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### categories? - -> `optional` **categories**: `string`[] \| `object`[] - -Defined in: metadata/build/field/types/String.d.ts:22 - -Categories for enum values -Can be an array of string values or an array of {value, label} objects - -*** - -### categoriesOrdered? - -> `optional` **categoriesOrdered**: `boolean` - -Defined in: metadata/build/field/types/String.d.ts:29 - -Whether categories should be considered to have a natural order - -*** - -### constraints? - -> `optional` **constraints**: [`StringConstraints`](/reference/stringconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `"email"` \| `"uri"` \| `"binary"` \| `"uuid"` - -Defined in: metadata/build/field/types/String.d.ts:17 - -Format of the string -- email: valid email address -- uri: valid URI -- binary: base64 encoded string -- uuid: valid UUID string - -#### Overrides - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"string"` - -Defined in: metadata/build/field/types/String.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/System.md b/portal/typescript/content/docs/reference/System.md deleted file mode 100644 index 234dd030..00000000 --- a/portal/typescript/content/docs/reference/System.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "System" ---- - -Defined in: library/build/system.d.ts:2 - -## Constructors - -### Constructor - -> **new System**(): `System` - -#### Returns - -`System` - -## Properties - -### plugins - -> **plugins**: [`TablePlugin`](/reference/tableplugin/)[] - -Defined in: library/build/system.d.ts:3 - -## Methods - -### register() - -> **register**(`PluginClass`): `void` - -Defined in: library/build/system.d.ts:4 - -#### Parameters - -##### PluginClass - -() => [`TablePlugin`](/reference/tableplugin/) - -#### Returns - -`void` diff --git a/portal/typescript/content/docs/reference/Table.md b/portal/typescript/content/docs/reference/Table.md deleted file mode 100644 index a727a50d..00000000 --- a/portal/typescript/content/docs/reference/Table.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "Table" ---- - -> **Table** = `pl.LazyDataFrame` - -Defined in: table/build/table/Table.d.ts:2 diff --git a/portal/typescript/content/docs/reference/TableError.md b/portal/typescript/content/docs/reference/TableError.md deleted file mode 100644 index 5e5ae6a1..00000000 --- a/portal/typescript/content/docs/reference/TableError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "TableError" ---- - -> **TableError** = [`FieldsError`](/reference/fieldserror/) \| [`FieldError`](/reference/fielderror/) \| [`RowError`](/reference/rowerror/) \| [`CellError`](/reference/cellerror/) \| [`ForeignKeyError`](/reference/foreignkeyerror/) - -Defined in: metadata/build/error/types/Table.d.ts:6 diff --git a/portal/typescript/content/docs/reference/TablePlugin.md b/portal/typescript/content/docs/reference/TablePlugin.md deleted file mode 100644 index 1ecda5bc..00000000 --- a/portal/typescript/content/docs/reference/TablePlugin.md +++ /dev/null @@ -1,148 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "TablePlugin" ---- - -Defined in: table/build/plugin.d.ts:17 - -## Extends - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Methods - -### inferDialect()? - -> `optional` **inferDialect**(`resource`, `options?`): `Promise`\<`undefined` \| [`Dialect`](/reference/dialect/)\> - -Defined in: table/build/plugin.d.ts:23 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`InferDialectOptions`](/reference/inferdialectoptions/) - -#### Returns - -`Promise`\<`undefined` \| [`Dialect`](/reference/dialect/)\> - -*** - -### inferSchema()? - -> `optional` **inferSchema**(`resource`, `options?`): `Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> - -Defined in: table/build/plugin.d.ts:24 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`InferSchemaOptions`](/reference/inferschemaoptions/) - -#### Returns - -`Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> - -*** - -### loadPackage()? - -> `optional` **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: dataset/build/plugin.d.ts:7 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -#### Inherited from - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) - -*** - -### loadTable()? - -> `optional` **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| [`Table`](/reference/table/)\> - -Defined in: table/build/plugin.d.ts:25 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| [`Table`](/reference/table/)\> - -*** - -### savePackage()? - -> `optional` **savePackage**(`dataPackage`, `options`): `Promise`\<`undefined` \| \{ `path?`: `string`; \}\> - -Defined in: table/build/plugin.d.ts:18 - -#### Parameters - -##### dataPackage - -[`Package`](/reference/package/) - -##### options - -[`SavePackageOptions`](/reference/savepackageoptions/) & `object` - -#### Returns - -`Promise`\<`undefined` \| \{ `path?`: `string`; \}\> - -#### Overrides - -[`DatasetPlugin`](/reference/datasetplugin/).[`savePackage`](/reference/datasetplugin/#savepackage) - -*** - -### saveTable()? - -> `optional` **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugin.d.ts:26 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> diff --git a/portal/typescript/content/docs/reference/TimeConstraints.md b/portal/typescript/content/docs/reference/TimeConstraints.md deleted file mode 100644 index 605a01fb..00000000 --- a/portal/typescript/content/docs/reference/TimeConstraints.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "TimeConstraints" ---- - -Defined in: metadata/build/field/types/Time.d.ts:21 - -Time-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] - -Defined in: metadata/build/field/types/Time.d.ts:42 - -Restrict values to a specified set of times -Should be in string time format (e.g., "HH:MM:SS") - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` - -Defined in: metadata/build/field/types/Time.d.ts:37 - -Exclusive maximum time value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` - -Defined in: metadata/build/field/types/Time.d.ts:33 - -Exclusive minimum time value - -*** - -### maximum? - -> `optional` **maximum**: `string` - -Defined in: metadata/build/field/types/Time.d.ts:29 - -Maximum allowed time value - -*** - -### minimum? - -> `optional` **minimum**: `string` - -Defined in: metadata/build/field/types/Time.d.ts:25 - -Minimum allowed time value - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/TimeField.md b/portal/typescript/content/docs/reference/TimeField.md deleted file mode 100644 index 6903c3f1..00000000 --- a/portal/typescript/content/docs/reference/TimeField.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "TimeField" ---- - -Defined in: metadata/build/field/types/Time.d.ts:5 - -Time field type - -## Extends - -- `BaseField`\<[`TimeConstraints`](/reference/timeconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`TimeConstraints`](/reference/timeconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Time.d.ts:16 - -Format of the time -- default: HH:MM:SS -- any: flexible time parsing (not recommended) -- Or custom strptime/strftime format string - -#### Overrides - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"time"` - -Defined in: metadata/build/field/types/Time.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/UnboundError.md b/portal/typescript/content/docs/reference/UnboundError.md deleted file mode 100644 index 8ec2cd3c..00000000 --- a/portal/typescript/content/docs/reference/UnboundError.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "UnboundError" ---- - -> **UnboundError** = [`MetadataError`](/reference/metadataerror/) \| [`DataError`](/reference/dataerror/) \| [`FileError`](/reference/fileerror/) \| [`TableError`](/reference/tableerror/) \| [`DocumentError`](/reference/documenterror/) - -Defined in: metadata/build/error/Unbound.d.ts:6 diff --git a/portal/typescript/content/docs/reference/XlsxPlugin.md b/portal/typescript/content/docs/reference/XlsxPlugin.md deleted file mode 100644 index 3988e287..00000000 --- a/portal/typescript/content/docs/reference/XlsxPlugin.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "XlsxPlugin" ---- - -Defined in: table/build/plugins/xlxs/plugin.d.ts:4 - -## Implements - -- [`TablePlugin`](/reference/tableplugin/) - -## Constructors - -### Constructor - -> **new XlsxPlugin**(): `XlsxPlugin` - -#### Returns - -`XlsxPlugin` - -## Methods - -### loadTable() - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/xlxs/plugin.d.ts:5 - -#### Parameters - -##### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -##### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `LazyDataFrame`\<`any`\>\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`loadTable`](/reference/tableplugin/#loadtable) - -*** - -### saveTable() - -> **saveTable**(`table`, `options`): `Promise`\<`undefined` \| `string`\> - -Defined in: table/build/plugins/xlxs/plugin.d.ts:6 - -#### Parameters - -##### table - -[`Table`](/reference/table/) - -##### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -#### Returns - -`Promise`\<`undefined` \| `string`\> - -#### Implementation of - -[`TablePlugin`](/reference/tableplugin/).[`saveTable`](/reference/tableplugin/#savetable) diff --git a/portal/typescript/content/docs/reference/YearConstraints.md b/portal/typescript/content/docs/reference/YearConstraints.md deleted file mode 100644 index bfeb8767..00000000 --- a/portal/typescript/content/docs/reference/YearConstraints.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "YearConstraints" ---- - -Defined in: metadata/build/field/types/Year.d.ts:14 - -Year-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] \| `number`[] - -Defined in: metadata/build/field/types/Year.d.ts:35 - -Restrict values to a specified set of years -Can be an array of numbers or strings that parse to years - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` \| `number` - -Defined in: metadata/build/field/types/Year.d.ts:30 - -Exclusive maximum year value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` \| `number` - -Defined in: metadata/build/field/types/Year.d.ts:26 - -Exclusive minimum year value - -*** - -### maximum? - -> `optional` **maximum**: `string` \| `number` - -Defined in: metadata/build/field/types/Year.d.ts:22 - -Maximum allowed year - -*** - -### minimum? - -> `optional` **minimum**: `string` \| `number` - -Defined in: metadata/build/field/types/Year.d.ts:18 - -Minimum allowed year - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/YearField.md b/portal/typescript/content/docs/reference/YearField.md deleted file mode 100644 index a6d9bcf7..00000000 --- a/portal/typescript/content/docs/reference/YearField.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "YearField" ---- - -Defined in: metadata/build/field/types/Year.d.ts:5 - -Year field type - -## Extends - -- `BaseField`\<[`YearConstraints`](/reference/yearconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`YearConstraints`](/reference/yearconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"year"` - -Defined in: metadata/build/field/types/Year.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/YearmonthConstraints.md b/portal/typescript/content/docs/reference/YearmonthConstraints.md deleted file mode 100644 index 5f545ff6..00000000 --- a/portal/typescript/content/docs/reference/YearmonthConstraints.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "YearmonthConstraints" ---- - -Defined in: metadata/build/field/types/Yearmonth.d.ts:14 - -Yearmonth-specific constraints - -## Extends - -- `BaseConstraints` - -## Properties - -### enum? - -> `optional` **enum**: `string`[] - -Defined in: metadata/build/field/types/Yearmonth.d.ts:35 - -Restrict values to a specified set of yearmonths -Should be in string format (e.g., "YYYY-MM") - -*** - -### exclusiveMaximum? - -> `optional` **exclusiveMaximum**: `string` - -Defined in: metadata/build/field/types/Yearmonth.d.ts:30 - -Exclusive maximum yearmonth value - -*** - -### exclusiveMinimum? - -> `optional` **exclusiveMinimum**: `string` - -Defined in: metadata/build/field/types/Yearmonth.d.ts:26 - -Exclusive minimum yearmonth value - -*** - -### maximum? - -> `optional` **maximum**: `string` - -Defined in: metadata/build/field/types/Yearmonth.d.ts:22 - -Maximum allowed yearmonth value (format: YYYY-MM) - -*** - -### minimum? - -> `optional` **minimum**: `string` - -Defined in: metadata/build/field/types/Yearmonth.d.ts:18 - -Minimum allowed yearmonth value (format: YYYY-MM) - -*** - -### required? - -> `optional` **required**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:55 - -Indicates if field is allowed to be null/empty - -#### Inherited from - -`BaseConstraints.required` - -*** - -### unique? - -> `optional` **unique**: `boolean` - -Defined in: metadata/build/field/types/Base.d.ts:59 - -Indicates if values must be unique within the column - -#### Inherited from - -`BaseConstraints.unique` diff --git a/portal/typescript/content/docs/reference/YearmonthField.md b/portal/typescript/content/docs/reference/YearmonthField.md deleted file mode 100644 index 9557327d..00000000 --- a/portal/typescript/content/docs/reference/YearmonthField.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "YearmonthField" ---- - -Defined in: metadata/build/field/types/Yearmonth.d.ts:5 - -Year and month field type - -## Extends - -- `BaseField`\<[`YearmonthConstraints`](/reference/yearmonthconstraints/)\> - -## Indexable - -\[`key`: `` `${string}:${string}` ``\]: `any` - -## Properties - -### constraints? - -> `optional` **constraints**: [`YearmonthConstraints`](/reference/yearmonthconstraints/) - -Defined in: metadata/build/field/types/Base.d.ts:46 - -Validation constraints applied to values - -#### Inherited from - -`BaseField.constraints` - -*** - -### description? - -> `optional` **description**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:21 - -Human-readable description - -#### Inherited from - -`BaseField.description` - -*** - -### example? - -> `optional` **example**: `any` - -Defined in: metadata/build/field/types/Base.d.ts:25 - -Example value for this field - -#### Inherited from - -`BaseField.example` - -*** - -### examples? - -> `optional` **examples**: `any`[] - -Defined in: metadata/build/field/types/Base.d.ts:29 - -Examples for this field - -#### Inherited from - -`BaseField.examples` - -*** - -### format? - -> `optional` **format**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:13 - -Field format -- optional addition to the type - -#### Inherited from - -`BaseField.format` - -*** - -### missingValues? - -> `optional` **missingValues**: (`string` \| \{ `label`: `string`; `value`: `string`; \})[] - -Defined in: metadata/build/field/types/Base.d.ts:39 - -Values representing missing data for this field -Can be a simple array of strings or an array of {value, label} objects -where label provides context for why the data is missing - -#### Inherited from - -`BaseField.missingValues` - -*** - -### name - -> **name**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:9 - -Name of the field matching the column name - -#### Inherited from - -`BaseField.name` - -*** - -### rdfType? - -> `optional` **rdfType**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:33 - -URI for semantic type (RDF) - -#### Inherited from - -`BaseField.rdfType` - -*** - -### title? - -> `optional` **title**: `string` - -Defined in: metadata/build/field/types/Base.d.ts:17 - -Human-readable title - -#### Inherited from - -`BaseField.title` - -*** - -### type - -> **type**: `"yearmonth"` - -Defined in: metadata/build/field/types/Yearmonth.d.ts:9 - -Field type - discriminator property diff --git a/portal/typescript/content/docs/reference/ZenodoPlugin.md b/portal/typescript/content/docs/reference/ZenodoPlugin.md deleted file mode 100644 index 22af9cc5..00000000 --- a/portal/typescript/content/docs/reference/ZenodoPlugin.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ZenodoPlugin" ---- - -Defined in: dataset/build/plugins/zenodo/plugin.d.ts:2 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new ZenodoPlugin**(): `ZenodoPlugin` - -#### Returns - -`ZenodoPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| \{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: dataset/build/plugins/zenodo/plugin.d.ts:3 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| \{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) diff --git a/portal/typescript/content/docs/reference/ZipPlugin.md b/portal/typescript/content/docs/reference/ZipPlugin.md deleted file mode 100644 index 977ceed3..00000000 --- a/portal/typescript/content/docs/reference/ZipPlugin.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "ZipPlugin" ---- - -Defined in: dataset/build/plugins/zip/plugin.d.ts:3 - -## Implements - -- [`DatasetPlugin`](/reference/datasetplugin/) - -## Constructors - -### Constructor - -> **new ZipPlugin**(): `ZipPlugin` - -#### Returns - -`ZipPlugin` - -## Methods - -### loadPackage() - -> **loadPackage**(`source`): `Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/zip/plugin.d.ts:4 - -#### Parameters - -##### source - -`string` - -#### Returns - -`Promise`\<`undefined` \| [`Package`](/reference/package/)\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`loadPackage`](/reference/datasetplugin/#loadpackage) - -*** - -### savePackage() - -> **savePackage**(`dataPackage`, `options`): `Promise`\<`undefined` \| \{ `path`: `undefined`; \}\> - -Defined in: dataset/build/plugins/zip/plugin.d.ts:5 - -#### Parameters - -##### dataPackage - -[`Package`](/reference/package/) - -##### options - -###### target - -`string` - -###### withRemote? - -`boolean` - -#### Returns - -`Promise`\<`undefined` \| \{ `path`: `undefined`; \}\> - -#### Implementation of - -[`DatasetPlugin`](/reference/datasetplugin/).[`savePackage`](/reference/datasetplugin/#savepackage) diff --git a/portal/typescript/content/docs/reference/assertLocalPathVacant.md b/portal/typescript/content/docs/reference/assertLocalPathVacant.md deleted file mode 100644 index a7aca77d..00000000 --- a/portal/typescript/content/docs/reference/assertLocalPathVacant.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "assertLocalPathVacant" ---- - -> **assertLocalPathVacant**(`path`): `Promise`\<`void`\> - -Defined in: dataset/build/file/path.d.ts:2 - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<`void`\> diff --git a/portal/typescript/content/docs/reference/convertPackageToDescriptor.md b/portal/typescript/content/docs/reference/convertPackageToDescriptor.md deleted file mode 100644 index b2f54503..00000000 --- a/portal/typescript/content/docs/reference/convertPackageToDescriptor.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "convertPackageToDescriptor" ---- - -> **convertPackageToDescriptor**(`dataPackage`, `options?`): [`Descriptor`](/reference/descriptor/) - -Defined in: metadata/build/package/convert/toDescriptor.d.ts:3 - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options? - -#### basepath? - -`string` - -## Returns - -[`Descriptor`](/reference/descriptor/) diff --git a/portal/typescript/content/docs/reference/convertResourceToDescriptor.md b/portal/typescript/content/docs/reference/convertResourceToDescriptor.md deleted file mode 100644 index d1eaacd6..00000000 --- a/portal/typescript/content/docs/reference/convertResourceToDescriptor.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "convertResourceToDescriptor" ---- - -> **convertResourceToDescriptor**(`resource`, `options?`): [`Descriptor`](/reference/descriptor/) - -Defined in: metadata/build/resource/convert/toDescriptor.d.ts:3 - -## Parameters - -### resource - -[`Resource`](/reference/resource/) - -### options? - -#### basepath? - -`string` - -## Returns - -[`Descriptor`](/reference/descriptor/) diff --git a/portal/typescript/content/docs/reference/convertSchemaFromJsonSchema.md b/portal/typescript/content/docs/reference/convertSchemaFromJsonSchema.md deleted file mode 100644 index fd0c1614..00000000 --- a/portal/typescript/content/docs/reference/convertSchemaFromJsonSchema.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "convertSchemaFromJsonSchema" ---- - -> **convertSchemaFromJsonSchema**(`jsonSchema`): [`Schema`](/reference/schema/) - -Defined in: metadata/build/schema/convert/fromJsonSchema.d.ts:3 - -## Parameters - -### jsonSchema - -`JSONSchema7` - -## Returns - -[`Schema`](/reference/schema/) diff --git a/portal/typescript/content/docs/reference/convertSchemaToHtml.md b/portal/typescript/content/docs/reference/convertSchemaToHtml.md deleted file mode 100644 index fe039095..00000000 --- a/portal/typescript/content/docs/reference/convertSchemaToHtml.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "convertSchemaToHtml" ---- - -> **convertSchemaToHtml**(`schema`, `options?`): `string` - -Defined in: document/build/schema/convert/toHtml.d.ts:2 - -## Parameters - -### schema - -[`Schema`](/reference/schema/) - -### options? - -#### frontmatter? - -`boolean` - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/convertSchemaToJsonSchema.md b/portal/typescript/content/docs/reference/convertSchemaToJsonSchema.md deleted file mode 100644 index 98b14546..00000000 --- a/portal/typescript/content/docs/reference/convertSchemaToJsonSchema.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "convertSchemaToJsonSchema" ---- - -> **convertSchemaToJsonSchema**(`schema`): `JSONSchema7` - -Defined in: metadata/build/schema/convert/toJsonSchema.d.ts:3 - -## Parameters - -### schema - -[`Schema`](/reference/schema/) - -## Returns - -`JSONSchema7` diff --git a/portal/typescript/content/docs/reference/convertSchemaToMarkdown.md b/portal/typescript/content/docs/reference/convertSchemaToMarkdown.md deleted file mode 100644 index 9fde6db6..00000000 --- a/portal/typescript/content/docs/reference/convertSchemaToMarkdown.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "convertSchemaToMarkdown" ---- - -> **convertSchemaToMarkdown**(`schema`, `options?`): `string` - -Defined in: document/build/schema/convert/toMarkdown.d.ts:2 - -## Parameters - -### schema - -[`Schema`](/reference/schema/) - -### options? - -#### frontmatter? - -`boolean` - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/copyFile.md b/portal/typescript/content/docs/reference/copyFile.md deleted file mode 100644 index 4f1eb061..00000000 --- a/portal/typescript/content/docs/reference/copyFile.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "copyFile" ---- - -> **copyFile**(`options`): `Promise`\<`void`\> - -Defined in: dataset/build/file/copy.d.ts:1 - -## Parameters - -### options - -#### sourcePath - -`string` - -#### targetPath - -`string` - -## Returns - -`Promise`\<`void`\> diff --git a/portal/typescript/content/docs/reference/createReport.md b/portal/typescript/content/docs/reference/createReport.md deleted file mode 100644 index 7253d5bf..00000000 --- a/portal/typescript/content/docs/reference/createReport.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "createReport" ---- - -> **createReport**\<`T`\>(`errors?`, `options?`): `object` - -Defined in: metadata/build/report/create.d.ts:2 - -## Type Parameters - -### T - -`T` = [`DpkitError`](/reference/dpkiterror/) - -## Parameters - -### errors? - -`T`[] - -### options? - -#### maxErrors? - -`number` - -## Returns - -`object` - -### errors - -> **errors**: `T`[] - -### valid - -> **valid**: `boolean` diff --git a/portal/typescript/content/docs/reference/denormalizePath.md b/portal/typescript/content/docs/reference/denormalizePath.md deleted file mode 100644 index db605bfc..00000000 --- a/portal/typescript/content/docs/reference/denormalizePath.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "denormalizePath" ---- - -> **denormalizePath**(`path`, `options`): `string` - -Defined in: metadata/build/path/denormalize.d.ts:1 - -## Parameters - -### path - -`string` - -### options - -#### basepath? - -`string` - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/denormalizeTable.md b/portal/typescript/content/docs/reference/denormalizeTable.md deleted file mode 100644 index 0b67357a..00000000 --- a/portal/typescript/content/docs/reference/denormalizeTable.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "denormalizeTable" ---- - -> **denormalizeTable**(`table`, `schema`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/table/denormalize.d.ts:5 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### schema - -[`Schema`](/reference/schema/) - -### options? - -[`DenormalizeFieldOptions`](/reference/denormalizefieldoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/describeFile.md b/portal/typescript/content/docs/reference/describeFile.md deleted file mode 100644 index 9d96dfc2..00000000 --- a/portal/typescript/content/docs/reference/describeFile.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "describeFile" ---- - -> **describeFile**(`path`, `options?`): `Promise`\<\{ `bytes`: `number`; `hash`: `string`; \}\> - -Defined in: dataset/build/file/describe.d.ts:2 - -## Parameters - -### path - -`string` - -### options? - -#### hashType? - -`HashType` - -## Returns - -`Promise`\<\{ `bytes`: `number`; `hash`: `string`; \}\> diff --git a/portal/typescript/content/docs/reference/getBasepath.md b/portal/typescript/content/docs/reference/getBasepath.md deleted file mode 100644 index 7abbffc4..00000000 --- a/portal/typescript/content/docs/reference/getBasepath.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "getBasepath" ---- - -> **getBasepath**(`path`): `string` - -Defined in: metadata/build/path/basepath.d.ts:2 - -## Parameters - -### path - -`string` - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/getFilename.md b/portal/typescript/content/docs/reference/getFilename.md deleted file mode 100644 index bcdf27b0..00000000 --- a/portal/typescript/content/docs/reference/getFilename.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "getFilename" ---- - -> **getFilename**(`path`): `undefined` \| `string` - -Defined in: metadata/build/path/path.d.ts:5 - -## Parameters - -### path - -`string` - -## Returns - -`undefined` \| `string` diff --git a/portal/typescript/content/docs/reference/getFormat.md b/portal/typescript/content/docs/reference/getFormat.md deleted file mode 100644 index f22ee3e4..00000000 --- a/portal/typescript/content/docs/reference/getFormat.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "getFormat" ---- - -> **getFormat**(`filename?`): `undefined` \| `string` - -Defined in: metadata/build/path/path.d.ts:4 - -## Parameters - -### filename? - -`string` - -## Returns - -`undefined` \| `string` diff --git a/portal/typescript/content/docs/reference/getName.md b/portal/typescript/content/docs/reference/getName.md deleted file mode 100644 index b7335c42..00000000 --- a/portal/typescript/content/docs/reference/getName.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "getName" ---- - -> **getName**(`filename?`): `undefined` \| `string` - -Defined in: metadata/build/path/path.d.ts:2 - -## Parameters - -### filename? - -`string` - -## Returns - -`undefined` \| `string` diff --git a/portal/typescript/content/docs/reference/getTempFilePath.md b/portal/typescript/content/docs/reference/getTempFilePath.md deleted file mode 100644 index d630492b..00000000 --- a/portal/typescript/content/docs/reference/getTempFilePath.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "getTempFilePath" ---- - -> **getTempFilePath**(`options?`): `string` - -Defined in: dataset/build/file/temp.d.ts:7 - -## Parameters - -### options? - -#### filename? - -`string` - -#### format? - -`string` - -#### persist? - -`boolean` - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/getTempFolderPath.md b/portal/typescript/content/docs/reference/getTempFolderPath.md deleted file mode 100644 index 5449cfc5..00000000 --- a/portal/typescript/content/docs/reference/getTempFolderPath.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "getTempFolderPath" ---- - -> **getTempFolderPath**(`options?`): `string` - -Defined in: dataset/build/folder/temp.d.ts:1 - -## Parameters - -### options? - -#### persist? - -`boolean` - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/inferBytes.md b/portal/typescript/content/docs/reference/inferBytes.md deleted file mode 100644 index 646fe4df..00000000 --- a/portal/typescript/content/docs/reference/inferBytes.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferBytes" ---- - -> **inferBytes**(`resource`): `Promise`\<`number`\> - -Defined in: dataset/build/file/infer.d.ts:3 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -## Returns - -`Promise`\<`number`\> diff --git a/portal/typescript/content/docs/reference/inferDatabaseSchema.md b/portal/typescript/content/docs/reference/inferDatabaseSchema.md deleted file mode 100644 index 9abd899a..00000000 --- a/portal/typescript/content/docs/reference/inferDatabaseSchema.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferDatabaseSchema" ---- - -> **inferDatabaseSchema**(`resource`): `Promise`\<[`Schema`](/reference/schema/)\> - -Defined in: database/build/schema/infer.d.ts:2 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> & `object` - -## Returns - -`Promise`\<[`Schema`](/reference/schema/)\> diff --git a/portal/typescript/content/docs/reference/inferDialect.md b/portal/typescript/content/docs/reference/inferDialect.md deleted file mode 100644 index ac453379..00000000 --- a/portal/typescript/content/docs/reference/inferDialect.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferDialect" ---- - -> **inferDialect**(`resource`, `options?`): `Promise`\<[`Dialect`](/reference/dialect/)\> - -Defined in: library/build/dialect/infer.d.ts:3 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`InferDialectOptions`](/reference/inferdialectoptions/) - -## Returns - -`Promise`\<[`Dialect`](/reference/dialect/)\> diff --git a/portal/typescript/content/docs/reference/inferEncoding.md b/portal/typescript/content/docs/reference/inferEncoding.md deleted file mode 100644 index cbc6db69..00000000 --- a/portal/typescript/content/docs/reference/inferEncoding.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferEncoding" ---- - -> **inferEncoding**(`resource`, `options?`): `Promise`\<`undefined` \| `string`\> - -Defined in: dataset/build/file/infer.d.ts:7 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -#### confidencePercent? - -`number` - -#### sampleBytes? - -`number` - -## Returns - -`Promise`\<`undefined` \| `string`\> diff --git a/portal/typescript/content/docs/reference/inferFormat.md b/portal/typescript/content/docs/reference/inferFormat.md deleted file mode 100644 index 14405bbf..00000000 --- a/portal/typescript/content/docs/reference/inferFormat.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferFormat" ---- - -> **inferFormat**(`resource`): `undefined` \| `string` - -Defined in: metadata/build/resource/infer.d.ts:3 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -## Returns - -`undefined` \| `string` diff --git a/portal/typescript/content/docs/reference/inferHash.md b/portal/typescript/content/docs/reference/inferHash.md deleted file mode 100644 index 15eb07c1..00000000 --- a/portal/typescript/content/docs/reference/inferHash.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferHash" ---- - -> **inferHash**(`resource`, `options?`): `Promise`\<`string`\> - -Defined in: dataset/build/file/infer.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -#### hashType? - -`HashType` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/inferName.md b/portal/typescript/content/docs/reference/inferName.md deleted file mode 100644 index 6f2c6d0b..00000000 --- a/portal/typescript/content/docs/reference/inferName.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferName" ---- - -> **inferName**(`resource`): `string` - -Defined in: metadata/build/resource/infer.d.ts:2 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/inferPackage.md b/portal/typescript/content/docs/reference/inferPackage.md deleted file mode 100644 index 5ba70965..00000000 --- a/portal/typescript/content/docs/reference/inferPackage.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferPackage" ---- - -> **inferPackage**(`dataPackage`, `options?`): `Promise`\<\{ `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: `object`[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: library/build/package/infer.d.ts:7 - -## Parameters - -### dataPackage - -`PartialPackage` - -### options? - -[`InferDialectOptions`](/reference/inferdialectoptions/) & [`InferSchemaOptions`](/reference/inferschemaoptions/) - -## Returns - -`Promise`\<\{ `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: `object`[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> diff --git a/portal/typescript/content/docs/reference/inferResource.md b/portal/typescript/content/docs/reference/inferResource.md deleted file mode 100644 index 61dbe0db..00000000 --- a/portal/typescript/content/docs/reference/inferResource.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferResource" ---- - -> **inferResource**(`resource`, `options?`): `Promise`\<\{ `$schema?`: `string`; `bytes?`: `number`; `data?`: `unknown`; `description?`: `string`; `dialect?`: `string` \| [`Dialect`](/reference/dialect/); `encoding?`: `string`; `format?`: `string`; `hash?`: `string`; `jsonSchema?`: `string` \| [`Descriptor`](/reference/descriptor/); `licenses?`: [`License`](/reference/license/)[]; `mediatype?`: `string`; `name`: `string`; `path?`: `string` \| `string`[]; `schema?`: `string` \| [`Schema`](/reference/schema/); `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `type?`: `"table"`; \}\> - -Defined in: library/build/resource/infer.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`InferDialectOptions`](/reference/inferdialectoptions/) & [`InferSchemaOptions`](/reference/inferschemaoptions/) - -## Returns - -`Promise`\<\{ `$schema?`: `string`; `bytes?`: `number`; `data?`: `unknown`; `description?`: `string`; `dialect?`: `string` \| [`Dialect`](/reference/dialect/); `encoding?`: `string`; `format?`: `string`; `hash?`: `string`; `jsonSchema?`: `string` \| [`Descriptor`](/reference/descriptor/); `licenses?`: [`License`](/reference/license/)[]; `mediatype?`: `string`; `name`: `string`; `path?`: `string` \| `string`[]; `schema?`: `string` \| [`Schema`](/reference/schema/); `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `type?`: `"table"`; \}\> diff --git a/portal/typescript/content/docs/reference/inferSchema.md b/portal/typescript/content/docs/reference/inferSchema.md deleted file mode 100644 index 1e96af60..00000000 --- a/portal/typescript/content/docs/reference/inferSchema.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferSchema" ---- - -> **inferSchema**(`resource`, `options?`): `Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> - -Defined in: library/build/schema/infer.d.ts:3 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`InferSchemaOptions`](/reference/inferschemaoptions/) - -## Returns - -`Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> diff --git a/portal/typescript/content/docs/reference/inferSchemaFromSample.md b/portal/typescript/content/docs/reference/inferSchemaFromSample.md deleted file mode 100644 index d1034caa..00000000 --- a/portal/typescript/content/docs/reference/inferSchemaFromSample.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferSchemaFromSample" ---- - -> **inferSchemaFromSample**(`sample`, `options?`): [`Schema`](/reference/schema/) - -Defined in: table/build/schema/infer.d.ts:13 - -## Parameters - -### sample - -`DataFrame` - -### options? - -[`InferSchemaOptions`](/reference/inferschemaoptions/) - -## Returns - -[`Schema`](/reference/schema/) diff --git a/portal/typescript/content/docs/reference/inferSchemaFromTable.md b/portal/typescript/content/docs/reference/inferSchemaFromTable.md deleted file mode 100644 index c13a9d0f..00000000 --- a/portal/typescript/content/docs/reference/inferSchemaFromTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inferSchemaFromTable" ---- - -> **inferSchemaFromTable**(`table`, `options?`): `Promise`\<[`Schema`](/reference/schema/)\> - -Defined in: table/build/schema/infer.d.ts:12 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options? - -[`InferSchemaOptions`](/reference/inferschemaoptions/) - -## Returns - -`Promise`\<[`Schema`](/reference/schema/)\> diff --git a/portal/typescript/content/docs/reference/inspectJsonValue.md b/portal/typescript/content/docs/reference/inspectJsonValue.md deleted file mode 100644 index d898425a..00000000 --- a/portal/typescript/content/docs/reference/inspectJsonValue.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inspectJsonValue" ---- - -> **inspectJsonValue**(`value`, `options`): `Promise`\<`object`[]\> - -Defined in: metadata/build/json/inspect/value.d.ts:6 - -Validate a value against a JSON Schema -It uses Ajv for JSON Schema validation under the hood - -## Parameters - -### value - -`unknown` - -### options - -#### jsonSchema - -`string` \| [`Descriptor`](/reference/descriptor/) - -## Returns - -`Promise`\<`object`[]\> diff --git a/portal/typescript/content/docs/reference/inspectTable.md b/portal/typescript/content/docs/reference/inspectTable.md deleted file mode 100644 index 7b0eb11f..00000000 --- a/portal/typescript/content/docs/reference/inspectTable.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "inspectTable" ---- - -> **inspectTable**(`table`, `options?`): `Promise`\<[`TableError`](/reference/tableerror/)[]\> - -Defined in: table/build/table/inspect.d.ts:4 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options? - -#### maxErrors? - -`number` - -#### sampleRows? - -`number` - -#### schema? - -[`Schema`](/reference/schema/) - -## Returns - -`Promise`\<[`TableError`](/reference/tableerror/)[]\> diff --git a/portal/typescript/content/docs/reference/isLocalPathExist.md b/portal/typescript/content/docs/reference/isLocalPathExist.md deleted file mode 100644 index 15727819..00000000 --- a/portal/typescript/content/docs/reference/isLocalPathExist.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "isLocalPathExist" ---- - -> **isLocalPathExist**(`path`): `Promise`\<`boolean`\> - -Defined in: dataset/build/file/path.d.ts:1 - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<`boolean`\> diff --git a/portal/typescript/content/docs/reference/isRemotePath.md b/portal/typescript/content/docs/reference/isRemotePath.md deleted file mode 100644 index d23a474c..00000000 --- a/portal/typescript/content/docs/reference/isRemotePath.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "isRemotePath" ---- - -> **isRemotePath**(`path`): `boolean` - -Defined in: metadata/build/path/path.d.ts:1 - -## Parameters - -### path - -`string` - -## Returns - -`boolean` diff --git a/portal/typescript/content/docs/reference/isRemoteResource.md b/portal/typescript/content/docs/reference/isRemoteResource.md deleted file mode 100644 index cb00da96..00000000 --- a/portal/typescript/content/docs/reference/isRemoteResource.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "isRemoteResource" ---- - -> **isRemoteResource**(`resource`): `boolean` - -Defined in: metadata/build/resource/helpers.d.ts:2 - -## Parameters - -### resource - -[`Resource`](/reference/resource/) - -## Returns - -`boolean` diff --git a/portal/typescript/content/docs/reference/loadArrowTable.md b/portal/typescript/content/docs/reference/loadArrowTable.md deleted file mode 100644 index b8753b24..00000000 --- a/portal/typescript/content/docs/reference/loadArrowTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadArrowTable" ---- - -> **loadArrowTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/arrow/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/loadCsvTable.md b/portal/typescript/content/docs/reference/loadCsvTable.md deleted file mode 100644 index 70805608..00000000 --- a/portal/typescript/content/docs/reference/loadCsvTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadCsvTable" ---- - -> **loadCsvTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/csv/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> & `object` - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/loadDatabaseTable.md b/portal/typescript/content/docs/reference/loadDatabaseTable.md deleted file mode 100644 index 317c511c..00000000 --- a/portal/typescript/content/docs/reference/loadDatabaseTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadDatabaseTable" ---- - -> **loadDatabaseTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: database/build/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> & `object` - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/loadDescriptor.md b/portal/typescript/content/docs/reference/loadDescriptor.md deleted file mode 100644 index 0b5deb6c..00000000 --- a/portal/typescript/content/docs/reference/loadDescriptor.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadDescriptor" ---- - -> **loadDescriptor**(`path`, `options?`): `Promise`\<`Record`\<`string`, `any`\>\> - -Defined in: metadata/build/descriptor/load.d.ts:6 - -Load a descriptor (JSON Object) from a file or URL -Uses dynamic imports to work in both Node.js and browser environments -Supports HTTP, HTTPS, FTP, and FTPS protocols - -## Parameters - -### path - -`string` - -### options? - -#### onlyRemote? - -`boolean` - -## Returns - -`Promise`\<`Record`\<`string`, `any`\>\> diff --git a/portal/typescript/content/docs/reference/loadDialect.md b/portal/typescript/content/docs/reference/loadDialect.md deleted file mode 100644 index f2cc2265..00000000 --- a/portal/typescript/content/docs/reference/loadDialect.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadDialect" ---- - -> **loadDialect**(`path`): `Promise`\<[`Dialect`](/reference/dialect/)\> - -Defined in: metadata/build/dialect/load.d.ts:5 - -Load a Dialect descriptor (JSON Object) from a file or URL -Ensures the descriptor is valid against its profile - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<[`Dialect`](/reference/dialect/)\> diff --git a/portal/typescript/content/docs/reference/loadFile.md b/portal/typescript/content/docs/reference/loadFile.md deleted file mode 100644 index 24b1a7ec..00000000 --- a/portal/typescript/content/docs/reference/loadFile.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadFile" ---- - -> **loadFile**(`path`, `options?`): `Promise`\<`Buffer`\<`ArrayBufferLike`\>\> - -Defined in: dataset/build/file/load.d.ts:1 - -## Parameters - -### path - -`string` - -### options? - -#### maxBytes? - -`number` - -## Returns - -`Promise`\<`Buffer`\<`ArrayBufferLike`\>\> diff --git a/portal/typescript/content/docs/reference/loadFileStream.md b/portal/typescript/content/docs/reference/loadFileStream.md deleted file mode 100644 index 4eddc80f..00000000 --- a/portal/typescript/content/docs/reference/loadFileStream.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadFileStream" ---- - -> **loadFileStream**(`path`, `options?`): `Promise`\<`Readable` \| `ReadStream`\> - -Defined in: dataset/build/stream/load.d.ts:2 - -## Parameters - -### path - -`string` | `string`[] - -### options? - -#### index? - -`number` - -#### maxBytes? - -`number` - -## Returns - -`Promise`\<`Readable` \| `ReadStream`\> diff --git a/portal/typescript/content/docs/reference/loadInlineTable.md b/portal/typescript/content/docs/reference/loadInlineTable.md deleted file mode 100644 index 808916e2..00000000 --- a/portal/typescript/content/docs/reference/loadInlineTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadInlineTable" ---- - -> **loadInlineTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<\{\[`x`: `string`\]: `any`; \}\>\> - -Defined in: table/build/plugins/inline/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<\{\[`x`: `string`\]: `any`; \}\>\> diff --git a/portal/typescript/content/docs/reference/loadJsonTable.md b/portal/typescript/content/docs/reference/loadJsonTable.md deleted file mode 100644 index a5aeeae0..00000000 --- a/portal/typescript/content/docs/reference/loadJsonTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadJsonTable" ---- - -> **loadJsonTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/json/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> & `object` - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/loadOdsTable.md b/portal/typescript/content/docs/reference/loadOdsTable.md deleted file mode 100644 index aebb4609..00000000 --- a/portal/typescript/content/docs/reference/loadOdsTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadOdsTable" ---- - -> **loadOdsTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/ods/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/loadPackage.md b/portal/typescript/content/docs/reference/loadPackage.md deleted file mode 100644 index 80286c74..00000000 --- a/portal/typescript/content/docs/reference/loadPackage.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackage" ---- - -> **loadPackage**(`source`): `Promise`\<[`Package`](/reference/package/)\> - -Defined in: library/build/package/load.d.ts:1 - -## Parameters - -### source - -`string` - -## Returns - -`Promise`\<[`Package`](/reference/package/)\> diff --git a/portal/typescript/content/docs/reference/loadPackageDescriptor.md b/portal/typescript/content/docs/reference/loadPackageDescriptor.md deleted file mode 100644 index f50a06b4..00000000 --- a/portal/typescript/content/docs/reference/loadPackageDescriptor.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageDescriptor" ---- - -> **loadPackageDescriptor**(`path`): `Promise`\<[`Package`](/reference/package/)\> - -Defined in: metadata/build/package/load.d.ts:5 - -Load a Package descriptor (JSON Object) from a file or URL -Ensures the descriptor is valid against its profile - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<[`Package`](/reference/package/)\> diff --git a/portal/typescript/content/docs/reference/loadPackageFromCkan.md b/portal/typescript/content/docs/reference/loadPackageFromCkan.md deleted file mode 100644 index c879d2bc..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromCkan.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromCkan" ---- - -> **loadPackageFromCkan**(`datasetUrl`): `Promise`\<\{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: dataset/build/plugins/ckan/package/load.d.ts:6 - -Load a package from a CKAN instance - -## Parameters - -### datasetUrl - -`string` - -## Returns - -`Promise`\<\{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Package object and cleanup function diff --git a/portal/typescript/content/docs/reference/loadPackageFromDatabase.md b/portal/typescript/content/docs/reference/loadPackageFromDatabase.md deleted file mode 100644 index a9ae238d..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromDatabase.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromDatabase" ---- - -> **loadPackageFromDatabase**(`connectionString`, `options`): `Promise`\<[`Package`](/reference/package/)\> - -Defined in: database/build/package/load.d.ts:3 - -## Parameters - -### connectionString - -`string` - -### options - -#### excludeTables? - -`string`[] - -#### format - -`DatabaseFormat` - -#### includeTables? - -`string`[] - -## Returns - -`Promise`\<[`Package`](/reference/package/)\> diff --git a/portal/typescript/content/docs/reference/loadPackageFromDatahub.md b/portal/typescript/content/docs/reference/loadPackageFromDatahub.md deleted file mode 100644 index e16c173f..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromDatahub.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromDatahub" ---- - -> **loadPackageFromDatahub**(`datasetUrl`): `Promise`\<[`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/datahub/package/load.d.ts:1 - -## Parameters - -### datasetUrl - -`string` - -## Returns - -`Promise`\<[`Package`](/reference/package/)\> diff --git a/portal/typescript/content/docs/reference/loadPackageFromFolder.md b/portal/typescript/content/docs/reference/loadPackageFromFolder.md deleted file mode 100644 index e66abb01..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromFolder.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromFolder" ---- - -> **loadPackageFromFolder**(`folderPath`): `Promise`\<[`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/folder/package/load.d.ts:1 - -## Parameters - -### folderPath - -`string` - -## Returns - -`Promise`\<[`Package`](/reference/package/)\> diff --git a/portal/typescript/content/docs/reference/loadPackageFromGithub.md b/portal/typescript/content/docs/reference/loadPackageFromGithub.md deleted file mode 100644 index 24e1f32c..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromGithub.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromGithub" ---- - -> **loadPackageFromGithub**(`repoUrl`, `options?`): `Promise`\<\{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: dataset/build/plugins/github/package/load.d.ts:6 - -Load a package from a Github repository - -## Parameters - -### repoUrl - -`string` - -### options? - -#### apiKey? - -`string` - -## Returns - -`Promise`\<\{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Package object diff --git a/portal/typescript/content/docs/reference/loadPackageFromZenodo.md b/portal/typescript/content/docs/reference/loadPackageFromZenodo.md deleted file mode 100644 index e993adf2..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromZenodo.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromZenodo" ---- - -> **loadPackageFromZenodo**(`datasetUrl`, `options?`): `Promise`\<\{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Defined in: dataset/build/plugins/zenodo/package/load.d.ts:6 - -Load a package from a Zenodo deposit - -## Parameters - -### datasetUrl - -`string` - -### options? - -#### apiKey? - -`string` - -## Returns - -`Promise`\<\{\[`x`: `` `${string}:${string}` ``\]: `any`; `$schema?`: `string`; `contributors?`: [`Contributor`](/reference/contributor/)[]; `created?`: `string`; `description?`: `string`; `homepage?`: `string`; `image?`: `string`; `keywords?`: `string`[]; `licenses?`: [`License`](/reference/license/)[]; `name?`: `string`; `resources`: [`Resource`](/reference/resource/)[]; `sources?`: [`Source`](/reference/source/)[]; `title?`: `string`; `version?`: `string`; \}\> - -Package object diff --git a/portal/typescript/content/docs/reference/loadPackageFromZip.md b/portal/typescript/content/docs/reference/loadPackageFromZip.md deleted file mode 100644 index bcf6ca8a..00000000 --- a/portal/typescript/content/docs/reference/loadPackageFromZip.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadPackageFromZip" ---- - -> **loadPackageFromZip**(`archivePath`): `Promise`\<[`Package`](/reference/package/)\> - -Defined in: dataset/build/plugins/zip/package/load.d.ts:1 - -## Parameters - -### archivePath - -`string` - -## Returns - -`Promise`\<[`Package`](/reference/package/)\> diff --git a/portal/typescript/content/docs/reference/loadParquetTable.md b/portal/typescript/content/docs/reference/loadParquetTable.md deleted file mode 100644 index 87c0706d..00000000 --- a/portal/typescript/content/docs/reference/loadParquetTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadParquetTable" ---- - -> **loadParquetTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/parquet/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/loadResourceDescriptor.md b/portal/typescript/content/docs/reference/loadResourceDescriptor.md deleted file mode 100644 index b1df29c0..00000000 --- a/portal/typescript/content/docs/reference/loadResourceDescriptor.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadResourceDescriptor" ---- - -> **loadResourceDescriptor**(`path`): `Promise`\<[`Resource`](/reference/resource/)\> - -Defined in: metadata/build/resource/load.d.ts:5 - -Load a Resource descriptor (JSON Object) from a file or URL -Ensures the descriptor is valid against its profile - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<[`Resource`](/reference/resource/)\> diff --git a/portal/typescript/content/docs/reference/loadSchema.md b/portal/typescript/content/docs/reference/loadSchema.md deleted file mode 100644 index 34d4ec6d..00000000 --- a/portal/typescript/content/docs/reference/loadSchema.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadSchema" ---- - -> **loadSchema**(`path`): `Promise`\<[`Schema`](/reference/schema/)\> - -Defined in: metadata/build/schema/load.d.ts:5 - -Load a Schema descriptor (JSON Object) from a file or URL -Ensures the descriptor is valid against its profile - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<[`Schema`](/reference/schema/)\> diff --git a/portal/typescript/content/docs/reference/loadTable.md b/portal/typescript/content/docs/reference/loadTable.md deleted file mode 100644 index 34b203c4..00000000 --- a/portal/typescript/content/docs/reference/loadTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadTable" ---- - -> **loadTable**(`resource`, `options?`): `Promise`\<`undefined` \| [`Table`](/reference/table/)\> - -Defined in: library/build/table/load.d.ts:3 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`undefined` \| [`Table`](/reference/table/)\> diff --git a/portal/typescript/content/docs/reference/loadXlsxTable.md b/portal/typescript/content/docs/reference/loadXlsxTable.md deleted file mode 100644 index 4bda5a18..00000000 --- a/portal/typescript/content/docs/reference/loadXlsxTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "loadXlsxTable" ---- - -> **loadXlsxTable**(`resource`, `options?`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/plugins/xlxs/table/load.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`LoadTableOptions`](/reference/loadtableoptions/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/modules.md b/portal/typescript/content/docs/reference/modules.md deleted file mode 100644 index a19f0f7f..00000000 --- a/portal/typescript/content/docs/reference/modules.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "dpkit" ---- - -## Classes - -- [ArrowPlugin](/reference/arrowplugin/) -- [CkanPlugin](/reference/ckanplugin/) -- [CsvPlugin](/reference/csvplugin/) -- [DatabasePlugin](/reference/databaseplugin/) -- [DatahubPlugin](/reference/datahubplugin/) -- [DescriptorPlugin](/reference/descriptorplugin/) -- [FolderPlugin](/reference/folderplugin/) -- [GithubPlugin](/reference/githubplugin/) -- [InlinePlugin](/reference/inlineplugin/) -- [JsonPlugin](/reference/jsonplugin/) -- [OdsPlugin](/reference/odsplugin/) -- [ParquetPlugin](/reference/parquetplugin/) -- [System](/reference/system/) -- [XlsxPlugin](/reference/xlsxplugin/) -- [ZenodoPlugin](/reference/zenodoplugin/) -- [ZipPlugin](/reference/zipplugin/) - -## Interfaces - -- [AnyConstraints](/reference/anyconstraints/) -- [AnyField](/reference/anyfield/) -- [ArrayConstraints](/reference/arrayconstraints/) -- [ArrayField](/reference/arrayfield/) -- [BaseCellError](/reference/basecellerror/) -- [BaseFieldError](/reference/basefielderror/) -- [BaseFieldsError](/reference/basefieldserror/) -- [BaseRowError](/reference/baserowerror/) -- [BooleanConstraints](/reference/booleanconstraints/) -- [BooleanField](/reference/booleanfield/) -- [BytesError](/reference/byteserror/) -- [CellEnumError](/reference/cellenumerror/) -- [CellExclusiveMaximumError](/reference/cellexclusivemaximumerror/) -- [CellExclusiveMinimumError](/reference/cellexclusiveminimumerror/) -- [CellJsonSchemaError](/reference/celljsonschemaerror/) -- [CellMaximumError](/reference/cellmaximumerror/) -- [CellMaxLengthError](/reference/cellmaxlengtherror/) -- [CellMinimumError](/reference/cellminimumerror/) -- [CellMinLengthError](/reference/cellminlengtherror/) -- [CellPatternError](/reference/cellpatternerror/) -- [CellRequiredError](/reference/cellrequirederror/) -- [CellTypeError](/reference/celltypeerror/) -- [CellUniqueError](/reference/celluniqueerror/) -- [Contributor](/reference/contributor/) -- [DataError](/reference/dataerror/) -- [DatasetPlugin](/reference/datasetplugin/) -- [DateConstraints](/reference/dateconstraints/) -- [DateField](/reference/datefield/) -- [DatetimeConstraints](/reference/datetimeconstraints/) -- [DatetimeField](/reference/datetimefield/) -- [Dialect](/reference/dialect/) -- [DialectOptions](/reference/dialectoptions/) -- [DurationConstraints](/reference/durationconstraints/) -- [DurationField](/reference/durationfield/) -- [EncodingError](/reference/encodingerror/) -- [FieldNameError](/reference/fieldnameerror/) -- [FieldsExtraError](/reference/fieldsextraerror/) -- [FieldsMissingError](/reference/fieldsmissingerror/) -- [FieldTypeError](/reference/fieldtypeerror/) -- [ForeignKeyError](/reference/foreignkeyerror/) -- [GeojsonConstraints](/reference/geojsonconstraints/) -- [GeojsonField](/reference/geojsonfield/) -- [GeopointConstraints](/reference/geopointconstraints/) -- [GeopointField](/reference/geopointfield/) -- [HashError](/reference/hasherror/) -- [InferDialectOptions](/reference/inferdialectoptions/) -- [InferSchemaOptions](/reference/inferschemaoptions/) -- [IntegerConstraints](/reference/integerconstraints/) -- [IntegerField](/reference/integerfield/) -- [JsonDocument](/reference/jsondocument/) -- [JsonDocumentError](/reference/jsondocumenterror/) -- [License](/reference/license/) -- [ListConstraints](/reference/listconstraints/) -- [ListField](/reference/listfield/) -- [MetadataError](/reference/metadataerror/) -- [NumberConstraints](/reference/numberconstraints/) -- [NumberField](/reference/numberfield/) -- [ObjectConstraints](/reference/objectconstraints/) -- [ObjectField](/reference/objectfield/) -- [Package](/reference/package/) -- [PolarsSchema](/reference/polarsschema/) -- [Report](/reference/report/) -- [Resource](/reference/resource/) -- [RowUniqueError](/reference/rowuniqueerror/) -- [Schema](/reference/schema/) -- [SchemaOptions](/reference/schemaoptions/) -- [Source](/reference/source/) -- [StringConstraints](/reference/stringconstraints/) -- [StringField](/reference/stringfield/) -- [TablePlugin](/reference/tableplugin/) -- [TimeConstraints](/reference/timeconstraints/) -- [TimeField](/reference/timefield/) -- [YearConstraints](/reference/yearconstraints/) -- [YearField](/reference/yearfield/) -- [YearmonthConstraints](/reference/yearmonthconstraints/) -- [YearmonthField](/reference/yearmonthfield/) - -## Type Aliases - -- [BoundError](/reference/bounderror/) -- [CellError](/reference/cellerror/) -- [DataRecord](/reference/datarecord/) -- [DataRow](/reference/datarow/) -- [DenormalizeFieldOptions](/reference/denormalizefieldoptions/) -- [Descriptor](/reference/descriptor/) -- [Document](/reference/document/) -- [DocumentError](/reference/documenterror/) -- [DpkitError](/reference/dpkiterror/) -- [Field](/reference/field/) -- [FieldError](/reference/fielderror/) -- [FieldsError](/reference/fieldserror/) -- [FieldType](/reference/fieldtype/) -- [FileError](/reference/fileerror/) -- [Frame](/reference/frame/) -- [LoadTableOptions](/reference/loadtableoptions/) -- [Plugin](/reference/plugin/) -- [PolarsField](/reference/polarsfield/) -- [RowError](/reference/rowerror/) -- [SavePackageOptions](/reference/savepackageoptions/) -- [SaveTableOptions](/reference/savetableoptions/) -- [Table](/reference/table/) -- [TableError](/reference/tableerror/) -- [UnboundError](/reference/unbounderror/) - -## Variables - -- [system](/reference/system-1/) - -## Functions - -- [assertLocalPathVacant](/reference/assertlocalpathvacant/) -- [convertPackageToDescriptor](/reference/convertpackagetodescriptor/) -- [convertResourceToDescriptor](/reference/convertresourcetodescriptor/) -- [convertSchemaFromJsonSchema](/reference/convertschemafromjsonschema/) -- [convertSchemaToHtml](/reference/convertschematohtml/) -- [convertSchemaToJsonSchema](/reference/convertschematojsonschema/) -- [convertSchemaToMarkdown](/reference/convertschematomarkdown/) -- [copyFile](/reference/copyfile/) -- [createReport](/reference/createreport/) -- [denormalizePath](/reference/denormalizepath/) -- [denormalizeTable](/reference/denormalizetable/) -- [describeFile](/reference/describefile/) -- [getBasepath](/reference/getbasepath/) -- [getFilename](/reference/getfilename/) -- [getFormat](/reference/getformat/) -- [getName](/reference/getname/) -- [getTempFilePath](/reference/gettempfilepath/) -- [getTempFolderPath](/reference/gettempfolderpath/) -- [inferBytes](/reference/inferbytes/) -- [inferDatabaseSchema](/reference/inferdatabaseschema/) -- [inferDialect](/reference/inferdialect/) -- [inferEncoding](/reference/inferencoding/) -- [inferFormat](/reference/inferformat/) -- [inferHash](/reference/inferhash/) -- [inferName](/reference/infername/) -- [inferPackage](/reference/inferpackage/) -- [inferResource](/reference/inferresource/) -- [inferSchema](/reference/inferschema/) -- [inferSchemaFromSample](/reference/inferschemafromsample/) -- [inferSchemaFromTable](/reference/inferschemafromtable/) -- [inspectJsonValue](/reference/inspectjsonvalue/) -- [inspectTable](/reference/inspecttable/) -- [isLocalPathExist](/reference/islocalpathexist/) -- [isRemotePath](/reference/isremotepath/) -- [isRemoteResource](/reference/isremoteresource/) -- [loadArrowTable](/reference/loadarrowtable/) -- [loadCsvTable](/reference/loadcsvtable/) -- [loadDatabaseTable](/reference/loaddatabasetable/) -- [loadDescriptor](/reference/loaddescriptor/) -- [loadDialect](/reference/loaddialect/) -- [loadFile](/reference/loadfile/) -- [loadFileStream](/reference/loadfilestream/) -- [loadInlineTable](/reference/loadinlinetable/) -- [loadJsonTable](/reference/loadjsontable/) -- [loadOdsTable](/reference/loadodstable/) -- [loadPackage](/reference/loadpackage/) -- [loadPackageDescriptor](/reference/loadpackagedescriptor/) -- [loadPackageFromCkan](/reference/loadpackagefromckan/) -- [loadPackageFromDatabase](/reference/loadpackagefromdatabase/) -- [loadPackageFromDatahub](/reference/loadpackagefromdatahub/) -- [loadPackageFromFolder](/reference/loadpackagefromfolder/) -- [loadPackageFromGithub](/reference/loadpackagefromgithub/) -- [loadPackageFromZenodo](/reference/loadpackagefromzenodo/) -- [loadPackageFromZip](/reference/loadpackagefromzip/) -- [loadParquetTable](/reference/loadparquettable/) -- [loadResourceDescriptor](/reference/loadresourcedescriptor/) -- [loadSchema](/reference/loadschema/) -- [loadTable](/reference/loadtable/) -- [loadXlsxTable](/reference/loadxlsxtable/) -- [normalizeTable](/reference/normalizetable/) -- [prefetchFile](/reference/prefetchfile/) -- [prefetchFiles](/reference/prefetchfiles/) -- [queryTable](/reference/querytable/) -- [resolveBasepath](/reference/resolvebasepath/) -- [resolveDialect](/reference/resolvedialect/) -- [resolveJsonSchema](/reference/resolvejsonschema/) -- [resolveSchema](/reference/resolveschema/) -- [saveArrowTable](/reference/savearrowtable/) -- [saveCsvTable](/reference/savecsvtable/) -- [saveDatabaseTable](/reference/savedatabasetable/) -- [saveDescriptor](/reference/savedescriptor/) -- [saveFile](/reference/savefile/) -- [saveJsonTable](/reference/savejsontable/) -- [saveOdsTable](/reference/saveodstable/) -- [savePackage](/reference/savepackage/) -- [savePackageDescriptor](/reference/savepackagedescriptor/) -- [savePackageToCkan](/reference/savepackagetockan/) -- [savePackageToDatabase](/reference/savepackagetodatabase/) -- [savePackageToFolder](/reference/savepackagetofolder/) -- [savePackageToGithub](/reference/savepackagetogithub/) -- [savePackageToZenodo](/reference/savepackagetozenodo/) -- [savePackageToZip](/reference/savepackagetozip/) -- [saveParquetTable](/reference/saveparquettable/) -- [saveTable](/reference/savetable/) -- [saveXlsxTable](/reference/savexlsxtable/) -- [stringifyDescriptor](/reference/stringifydescriptor/) -- [validateDialect](/reference/validatedialect/) -- [validateDocument](/reference/validatedocument/) -- [validateFile](/reference/validatefile/) -- [validatePackage](/reference/validatepackage/) -- [validatePackageMetadata](/reference/validatepackagemetadata/) -- [validateResource](/reference/validateresource/) -- [validateResourceMetadata](/reference/validateresourcemetadata/) -- [validateSchema](/reference/validateschema/) -- [validateTable](/reference/validatetable/) -- [writeTempFile](/reference/writetempfile/) diff --git a/portal/typescript/content/docs/reference/normalizeTable.md b/portal/typescript/content/docs/reference/normalizeTable.md deleted file mode 100644 index 04f0a105..00000000 --- a/portal/typescript/content/docs/reference/normalizeTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "normalizeTable" ---- - -> **normalizeTable**(`table`, `schema`): `Promise`\<`LazyDataFrame`\<`any`\>\> - -Defined in: table/build/table/normalize.d.ts:5 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### schema - -[`Schema`](/reference/schema/) - -## Returns - -`Promise`\<`LazyDataFrame`\<`any`\>\> diff --git a/portal/typescript/content/docs/reference/prefetchFile.md b/portal/typescript/content/docs/reference/prefetchFile.md deleted file mode 100644 index aefc2368..00000000 --- a/portal/typescript/content/docs/reference/prefetchFile.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "prefetchFile" ---- - -> **prefetchFile**(`path`): `Promise`\<`string`\> - -Defined in: dataset/build/file/fetch.d.ts:2 - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/prefetchFiles.md b/portal/typescript/content/docs/reference/prefetchFiles.md deleted file mode 100644 index df1b53fe..00000000 --- a/portal/typescript/content/docs/reference/prefetchFiles.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "prefetchFiles" ---- - -> **prefetchFiles**(`path?`): `Promise`\<`string`[]\> - -Defined in: dataset/build/file/fetch.d.ts:1 - -## Parameters - -### path? - -`string` | `string`[] - -## Returns - -`Promise`\<`string`[]\> diff --git a/portal/typescript/content/docs/reference/queryTable.md b/portal/typescript/content/docs/reference/queryTable.md deleted file mode 100644 index 9d8d92b1..00000000 --- a/portal/typescript/content/docs/reference/queryTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "queryTable" ---- - -> **queryTable**(`table`, `query`): `LazyDataFrame`\<`any`\> - -Defined in: table/build/table/query.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### query - -`string` - -## Returns - -`LazyDataFrame`\<`any`\> diff --git a/portal/typescript/content/docs/reference/resolveBasepath.md b/portal/typescript/content/docs/reference/resolveBasepath.md deleted file mode 100644 index 762b6237..00000000 --- a/portal/typescript/content/docs/reference/resolveBasepath.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "resolveBasepath" ---- - -> **resolveBasepath**(`path`): `Promise`\<`string`\> - -Defined in: metadata/build/path/basepath.d.ts:1 - -## Parameters - -### path - -`string` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/resolveDialect.md b/portal/typescript/content/docs/reference/resolveDialect.md deleted file mode 100644 index 5fb4af6d..00000000 --- a/portal/typescript/content/docs/reference/resolveDialect.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "resolveDialect" ---- - -> **resolveDialect**(`dialect?`): `Promise`\<`undefined` \| [`Dialect`](/reference/dialect/)\> - -Defined in: metadata/build/dialect/resolve.d.ts:2 - -## Parameters - -### dialect? - -`string` | [`Dialect`](/reference/dialect/) - -## Returns - -`Promise`\<`undefined` \| [`Dialect`](/reference/dialect/)\> diff --git a/portal/typescript/content/docs/reference/resolveJsonSchema.md b/portal/typescript/content/docs/reference/resolveJsonSchema.md deleted file mode 100644 index ee07ae2e..00000000 --- a/portal/typescript/content/docs/reference/resolveJsonSchema.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "resolveJsonSchema" ---- - -> **resolveJsonSchema**(`jsonSchema?`): `Promise`\<`undefined` \| [`Descriptor`](/reference/descriptor/)\> - -Defined in: metadata/build/json/resolve.d.ts:2 - -## Parameters - -### jsonSchema? - -`string` | [`Descriptor`](/reference/descriptor/) - -## Returns - -`Promise`\<`undefined` \| [`Descriptor`](/reference/descriptor/)\> diff --git a/portal/typescript/content/docs/reference/resolveSchema.md b/portal/typescript/content/docs/reference/resolveSchema.md deleted file mode 100644 index b42ba836..00000000 --- a/portal/typescript/content/docs/reference/resolveSchema.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "resolveSchema" ---- - -> **resolveSchema**(`schema?`): `Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> - -Defined in: metadata/build/schema/resolve.d.ts:2 - -## Parameters - -### schema? - -`string` | [`Schema`](/reference/schema/) - -## Returns - -`Promise`\<`undefined` \| [`Schema`](/reference/schema/)\> diff --git a/portal/typescript/content/docs/reference/saveArrowTable.md b/portal/typescript/content/docs/reference/saveArrowTable.md deleted file mode 100644 index 242596d0..00000000 --- a/portal/typescript/content/docs/reference/saveArrowTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveArrowTable" ---- - -> **saveArrowTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: table/build/plugins/arrow/table/save.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/saveCsvTable.md b/portal/typescript/content/docs/reference/saveCsvTable.md deleted file mode 100644 index b808058d..00000000 --- a/portal/typescript/content/docs/reference/saveCsvTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveCsvTable" ---- - -> **saveCsvTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: table/build/plugins/csv/table/save.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`DialectOptions`](/reference/dialectoptions/) & [`SchemaOptions`](/reference/schemaoptions/) & `object` & `object` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/saveDatabaseTable.md b/portal/typescript/content/docs/reference/saveDatabaseTable.md deleted file mode 100644 index 6e6727a0..00000000 --- a/portal/typescript/content/docs/reference/saveDatabaseTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveDatabaseTable" ---- - -> **saveDatabaseTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: database/build/table/save.d.ts:2 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`DialectOptions`](/reference/dialectoptions/) & [`SchemaOptions`](/reference/schemaoptions/) & `object` & `object` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/saveDescriptor.md b/portal/typescript/content/docs/reference/saveDescriptor.md deleted file mode 100644 index d8c7e519..00000000 --- a/portal/typescript/content/docs/reference/saveDescriptor.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveDescriptor" ---- - -> **saveDescriptor**(`descriptor`, `options`): `Promise`\<`void`\> - -Defined in: metadata/build/descriptor/save.d.ts:6 - -Save a descriptor (JSON Object) to a file path -Works in Node.js environments - -## Parameters - -### descriptor - -[`Descriptor`](/reference/descriptor/) - -### options - -#### overwrite? - -`boolean` - -#### path - -`string` - -## Returns - -`Promise`\<`void`\> diff --git a/portal/typescript/content/docs/reference/saveFile.md b/portal/typescript/content/docs/reference/saveFile.md deleted file mode 100644 index 5d9a6c1a..00000000 --- a/portal/typescript/content/docs/reference/saveFile.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveFile" ---- - -> **saveFile**(`path`, `buffer`, `options?`): `Promise`\<`void`\> - -Defined in: dataset/build/file/save.d.ts:1 - -## Parameters - -### path - -`string` - -### buffer - -`Buffer` - -### options? - -#### overwrite? - -`boolean` - -## Returns - -`Promise`\<`void`\> diff --git a/portal/typescript/content/docs/reference/saveJsonTable.md b/portal/typescript/content/docs/reference/saveJsonTable.md deleted file mode 100644 index 91fbd686..00000000 --- a/portal/typescript/content/docs/reference/saveJsonTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveJsonTable" ---- - -> **saveJsonTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: table/build/plugins/json/table/save.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`DialectOptions`](/reference/dialectoptions/) & [`SchemaOptions`](/reference/schemaoptions/) & `object` & `object` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/saveOdsTable.md b/portal/typescript/content/docs/reference/saveOdsTable.md deleted file mode 100644 index 66c2a210..00000000 --- a/portal/typescript/content/docs/reference/saveOdsTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveOdsTable" ---- - -> **saveOdsTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: table/build/plugins/ods/table/save.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/savePackage.md b/portal/typescript/content/docs/reference/savePackage.md deleted file mode 100644 index ebb01da4..00000000 --- a/portal/typescript/content/docs/reference/savePackage.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackage" ---- - -> **savePackage**(`dataPackage`, `options`): `Promise`\<\{ `path?`: `string`; \}\> - -Defined in: library/build/package/save.d.ts:3 - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -[`SavePackageOptions`](/reference/savepackageoptions/) - -## Returns - -`Promise`\<\{ `path?`: `string`; \}\> diff --git a/portal/typescript/content/docs/reference/savePackageDescriptor.md b/portal/typescript/content/docs/reference/savePackageDescriptor.md deleted file mode 100644 index 3b8d9b73..00000000 --- a/portal/typescript/content/docs/reference/savePackageDescriptor.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageDescriptor" ---- - -> **savePackageDescriptor**(`dataPackage`, `options`): `Promise`\<`void`\> - -Defined in: metadata/build/package/save.d.ts:6 - -Save a Package to a file path -Works in Node.js environments - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -#### overwrite? - -`boolean` - -#### path - -`string` - -## Returns - -`Promise`\<`void`\> diff --git a/portal/typescript/content/docs/reference/savePackageToCkan.md b/portal/typescript/content/docs/reference/savePackageToCkan.md deleted file mode 100644 index 7bea915a..00000000 --- a/portal/typescript/content/docs/reference/savePackageToCkan.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageToCkan" ---- - -> **savePackageToCkan**(`dataPackage`, `options`): `Promise`\<\{ `datasetUrl`: `string`; `path`: `unknown`; \}\> - -Defined in: dataset/build/plugins/ckan/package/save.d.ts:2 - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -#### apiKey - -`string` - -#### ckanUrl - -`string` - -#### datasetName - -`string` - -#### ownerOrg - -`string` - -## Returns - -`Promise`\<\{ `datasetUrl`: `string`; `path`: `unknown`; \}\> diff --git a/portal/typescript/content/docs/reference/savePackageToDatabase.md b/portal/typescript/content/docs/reference/savePackageToDatabase.md deleted file mode 100644 index e25ee6c7..00000000 --- a/portal/typescript/content/docs/reference/savePackageToDatabase.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageToDatabase" ---- - -> **savePackageToDatabase**(`dataPackage`, `options`): `Promise`\<\{ `path`: `string`; \}\> - -Defined in: database/build/package/save.d.ts:5 - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -[`SavePackageOptions`](/reference/savepackageoptions/) & `object` - -## Returns - -`Promise`\<\{ `path`: `string`; \}\> diff --git a/portal/typescript/content/docs/reference/savePackageToFolder.md b/portal/typescript/content/docs/reference/savePackageToFolder.md deleted file mode 100644 index 647a76ba..00000000 --- a/portal/typescript/content/docs/reference/savePackageToFolder.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageToFolder" ---- - -> **savePackageToFolder**(`dataPackage`, `options`): `Promise`\<\{ `resources`: [`Descriptor`](/reference/descriptor/)[]; \}\> - -Defined in: dataset/build/plugins/folder/package/save.d.ts:2 - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -#### folderPath - -`string` - -#### withRemote? - -`boolean` - -## Returns - -`Promise`\<\{ `resources`: [`Descriptor`](/reference/descriptor/)[]; \}\> diff --git a/portal/typescript/content/docs/reference/savePackageToGithub.md b/portal/typescript/content/docs/reference/savePackageToGithub.md deleted file mode 100644 index 8511b1f4..00000000 --- a/portal/typescript/content/docs/reference/savePackageToGithub.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageToGithub" ---- - -> **savePackageToGithub**(`dataPackage`, `options`): `Promise`\<\{ `path`: `string`; `repoUrl`: `string`; \}\> - -Defined in: dataset/build/plugins/github/package/save.d.ts:7 - -Save a package to a Github repository - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -Object containing the package to save and Github details - -#### apiKey - -`string` - -#### org? - -`string` - -#### repo - -`string` - -## Returns - -`Promise`\<\{ `path`: `string`; `repoUrl`: `string`; \}\> - -Object with the repository URL diff --git a/portal/typescript/content/docs/reference/savePackageToZenodo.md b/portal/typescript/content/docs/reference/savePackageToZenodo.md deleted file mode 100644 index e912d261..00000000 --- a/portal/typescript/content/docs/reference/savePackageToZenodo.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageToZenodo" ---- - -> **savePackageToZenodo**(`dataPackage`, `options`): `Promise`\<\{ `datasetUrl`: `string`; `path`: `string`; \}\> - -Defined in: dataset/build/plugins/zenodo/package/save.d.ts:7 - -Save a package to Zenodo - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -Object containing the package to save and Zenodo API details - -#### apiKey - -`string` - -#### sandbox? - -`boolean` - -## Returns - -`Promise`\<\{ `datasetUrl`: `string`; `path`: `string`; \}\> - -Object with the deposit URL and DOI diff --git a/portal/typescript/content/docs/reference/savePackageToZip.md b/portal/typescript/content/docs/reference/savePackageToZip.md deleted file mode 100644 index 9c1feed1..00000000 --- a/portal/typescript/content/docs/reference/savePackageToZip.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "savePackageToZip" ---- - -> **savePackageToZip**(`dataPackage`, `options`): `Promise`\<`void`\> - -Defined in: dataset/build/plugins/zip/package/save.d.ts:2 - -## Parameters - -### dataPackage - -[`Package`](/reference/package/) - -### options - -#### archivePath - -`string` - -#### withRemote? - -`boolean` - -## Returns - -`Promise`\<`void`\> diff --git a/portal/typescript/content/docs/reference/saveParquetTable.md b/portal/typescript/content/docs/reference/saveParquetTable.md deleted file mode 100644 index 326402ad..00000000 --- a/portal/typescript/content/docs/reference/saveParquetTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveParquetTable" ---- - -> **saveParquetTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: table/build/plugins/parquet/table/save.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/saveTable.md b/portal/typescript/content/docs/reference/saveTable.md deleted file mode 100644 index b2c8ffa4..00000000 --- a/portal/typescript/content/docs/reference/saveTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveTable" ---- - -> **saveTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: library/build/table/save.d.ts:2 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/saveXlsxTable.md b/portal/typescript/content/docs/reference/saveXlsxTable.md deleted file mode 100644 index c4d02f34..00000000 --- a/portal/typescript/content/docs/reference/saveXlsxTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "saveXlsxTable" ---- - -> **saveXlsxTable**(`table`, `options`): `Promise`\<`string`\> - -Defined in: table/build/plugins/xlxs/table/save.d.ts:3 - -## Parameters - -### table - -[`Table`](/reference/table/) - -### options - -[`SaveTableOptions`](/reference/savetableoptions/) - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/content/docs/reference/stringifyDescriptor.md b/portal/typescript/content/docs/reference/stringifyDescriptor.md deleted file mode 100644 index 9e6dbaf8..00000000 --- a/portal/typescript/content/docs/reference/stringifyDescriptor.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "stringifyDescriptor" ---- - -> **stringifyDescriptor**(`descriptor`): `string` - -Defined in: metadata/build/descriptor/process/stringify.d.ts:2 - -## Parameters - -### descriptor - -[`Descriptor`](/reference/descriptor/) - -## Returns - -`string` diff --git a/portal/typescript/content/docs/reference/system-1.md b/portal/typescript/content/docs/reference/system-1.md deleted file mode 100644 index 7ed526bf..00000000 --- a/portal/typescript/content/docs/reference/system-1.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "system" ---- - -> `const` **system**: [`System`](/reference/system/) - -Defined in: library/build/system.d.ts:6 diff --git a/portal/typescript/content/docs/reference/validateDialect.md b/portal/typescript/content/docs/reference/validateDialect.md deleted file mode 100644 index b34d178c..00000000 --- a/portal/typescript/content/docs/reference/validateDialect.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateDialect" ---- - -> **validateDialect**(`source`): `Promise`\<\{ `dialect`: `undefined` \| [`Dialect`](/reference/dialect/); `errors`: [`MetadataError`](/reference/metadataerror/)[]; `valid`: `boolean`; \}\> - -Defined in: metadata/build/dialect/validate.d.ts:6 - -Validate a Dialect descriptor (JSON Object) against its profile - -## Parameters - -### source - -`string` | [`Dialect`](/reference/dialect/) | [`Descriptor`](/reference/descriptor/) - -## Returns - -`Promise`\<\{ `dialect`: `undefined` \| [`Dialect`](/reference/dialect/); `errors`: [`MetadataError`](/reference/metadataerror/)[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validateDocument.md b/portal/typescript/content/docs/reference/validateDocument.md deleted file mode 100644 index e08297da..00000000 --- a/portal/typescript/content/docs/reference/validateDocument.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateDocument" ---- - -> **validateDocument**(`resource`): `Promise`\<\{ `errors`: [`DpkitError`](/reference/dpkiterror/)[]; `valid`: `boolean`; \}\> - -Defined in: document/build/document/validate.d.ts:2 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -## Returns - -`Promise`\<\{ `errors`: [`DpkitError`](/reference/dpkiterror/)[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validateFile.md b/portal/typescript/content/docs/reference/validateFile.md deleted file mode 100644 index f4c9265c..00000000 --- a/portal/typescript/content/docs/reference/validateFile.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateFile" ---- - -> **validateFile**(`resource`): `Promise`\<\{ `errors`: [`FileError`](/reference/fileerror/)[]; `valid`: `boolean`; \}\> - -Defined in: dataset/build/file/validate.d.ts:3 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -## Returns - -`Promise`\<\{ `errors`: [`FileError`](/reference/fileerror/)[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validatePackage.md b/portal/typescript/content/docs/reference/validatePackage.md deleted file mode 100644 index 91a4860e..00000000 --- a/portal/typescript/content/docs/reference/validatePackage.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validatePackage" ---- - -> **validatePackage**(`source`, `options?`): `Promise`\<\{ `errors`: [`BoundError`](/reference/bounderror/)[]; `valid`: `boolean`; \} \| \{ `errors`: `object`[]; `valid`: `boolean`; \}\> - -Defined in: library/build/package/validate.d.ts:3 - -## Parameters - -### source - -`string` | [`Descriptor`](/reference/descriptor/) | `Partial`\<[`Package`](/reference/package/)\> - -### options? - -#### basepath? - -`string` - -## Returns - -`Promise`\<\{ `errors`: [`BoundError`](/reference/bounderror/)[]; `valid`: `boolean`; \} \| \{ `errors`: `object`[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validatePackageMetadata.md b/portal/typescript/content/docs/reference/validatePackageMetadata.md deleted file mode 100644 index 0c9f28b1..00000000 --- a/portal/typescript/content/docs/reference/validatePackageMetadata.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validatePackageMetadata" ---- - -> **validatePackageMetadata**(`source`, `options?`): `Promise`\<\{ `dataPackage`: `undefined` \| [`Package`](/reference/package/); `errors`: [`MetadataError`](/reference/metadataerror/)[]; `valid`: `boolean`; \}\> - -Defined in: metadata/build/package/validate.d.ts:6 - -Validate a Package descriptor (JSON Object) against its profile - -## Parameters - -### source - -`string` | [`Descriptor`](/reference/descriptor/) | [`Package`](/reference/package/) - -### options? - -#### basepath? - -`string` - -## Returns - -`Promise`\<\{ `dataPackage`: `undefined` \| [`Package`](/reference/package/); `errors`: [`MetadataError`](/reference/metadataerror/)[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validateResource.md b/portal/typescript/content/docs/reference/validateResource.md deleted file mode 100644 index fd67d47a..00000000 --- a/portal/typescript/content/docs/reference/validateResource.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateResource" ---- - -> **validateResource**(`source`, `options?`): `Promise`\<\{ `errors`: [`UnboundError`](/reference/unbounderror/)[]; `valid`: `boolean`; \}\> - -Defined in: library/build/resource/validate.d.ts:3 - -## Parameters - -### source - -`string` | `Partial`\<[`Resource`](/reference/resource/)\> | [`Descriptor`](/reference/descriptor/) - -### options? - -[`InferSchemaOptions`](/reference/inferschemaoptions/) & `object` - -## Returns - -`Promise`\<\{ `errors`: [`UnboundError`](/reference/unbounderror/)[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validateResourceMetadata.md b/portal/typescript/content/docs/reference/validateResourceMetadata.md deleted file mode 100644 index 6311ea3d..00000000 --- a/portal/typescript/content/docs/reference/validateResourceMetadata.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateResourceMetadata" ---- - -> **validateResourceMetadata**(`source`, `options?`): `Promise`\<\{ `errors`: [`MetadataError`](/reference/metadataerror/)[]; `resource`: `undefined` \| [`Resource`](/reference/resource/); `valid`: `boolean`; \}\> - -Defined in: metadata/build/resource/validate.d.ts:7 - -Validate a Resource descriptor (JSON Object) against its profile - -## Parameters - -### source - -`string` | [`Resource`](/reference/resource/) | [`Descriptor`](/reference/descriptor/) - -### options? - -#### basepath? - -`string` - -## Returns - -`Promise`\<\{ `errors`: [`MetadataError`](/reference/metadataerror/)[]; `resource`: `undefined` \| [`Resource`](/reference/resource/); `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validateSchema.md b/portal/typescript/content/docs/reference/validateSchema.md deleted file mode 100644 index 255190b1..00000000 --- a/portal/typescript/content/docs/reference/validateSchema.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateSchema" ---- - -> **validateSchema**(`source`): `Promise`\<\{ `errors`: [`MetadataError`](/reference/metadataerror/)[]; `schema`: `undefined` \| [`Schema`](/reference/schema/); `valid`: `boolean`; \}\> - -Defined in: metadata/build/schema/validate.d.ts:6 - -Validate a Schema descriptor (JSON Object) against its profile - -## Parameters - -### source - -`string` | [`Descriptor`](/reference/descriptor/) | [`Schema`](/reference/schema/) - -## Returns - -`Promise`\<\{ `errors`: [`MetadataError`](/reference/metadataerror/)[]; `schema`: `undefined` \| [`Schema`](/reference/schema/); `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/validateTable.md b/portal/typescript/content/docs/reference/validateTable.md deleted file mode 100644 index 4245ac54..00000000 --- a/portal/typescript/content/docs/reference/validateTable.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "validateTable" ---- - -> **validateTable**(`resource`, `options?`): `Promise`\<\{ `errors`: [`UnboundError`](/reference/unbounderror/)[]; `valid`: `boolean`; \}\> - -Defined in: library/build/table/validate.d.ts:4 - -## Parameters - -### resource - -`Partial`\<[`Resource`](/reference/resource/)\> - -### options? - -[`InferDialectOptions`](/reference/inferdialectoptions/) & [`InferSchemaOptions`](/reference/inferschemaoptions/) & `object` & `object` - -## Returns - -`Promise`\<\{ `errors`: [`UnboundError`](/reference/unbounderror/)[]; `valid`: `boolean`; \}\> diff --git a/portal/typescript/content/docs/reference/writeTempFile.md b/portal/typescript/content/docs/reference/writeTempFile.md deleted file mode 100644 index 6af6ae91..00000000 --- a/portal/typescript/content/docs/reference/writeTempFile.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -editUrl: false -next: false -prev: false -title: "writeTempFile" ---- - -> **writeTempFile**(`content`, `options?`): `Promise`\<`string`\> - -Defined in: dataset/build/file/temp.d.ts:2 - -## Parameters - -### content - -`string` | `Buffer`\<`ArrayBufferLike`\> - -### options? - -#### filename? - -`string` - -#### format? - -`string` - -#### persist? - -`boolean` - -## Returns - -`Promise`\<`string`\> diff --git a/portal/typescript/public/favicon.png b/portal/typescript/public/favicon.png deleted file mode 100644 index 384b92dd..00000000 Binary files a/portal/typescript/public/favicon.png and /dev/null differ diff --git a/portal/typescript/styles/custom.css b/portal/typescript/styles/custom.css deleted file mode 100644 index eed9a576..00000000 --- a/portal/typescript/styles/custom.css +++ /dev/null @@ -1,52 +0,0 @@ -:root { - --purple-hsl: 209, 60%, 60%; - --overlay-blurple: hsla(var(--purple-hsl), 0.2); - --scrollbar-color: #ddd; -} - -:root[data-theme="light"] { - --purple-hsl: 209, 85%, 65%; - --sl-color-text-accent: #228be6; - --sl-color-banner-bg: #228be6; -} - -:root[data-theme="dark"] { - --sl-color-text-accent: #228be6; - --sl-color-banner-bg: #228be6; -} - -/* logo */ - -.site-title { - gap: 5px; - font-size: 22px; - font-weight: bold; - text-decoration: underline; -} - -.site-title img { - width: 32px; - height: 32px; -} - -html[data-theme="light"] .site-title { - color: black; -} - -html[data-theme="dark"] .site-title { - color: white; -} - -/* scrollbar */ - -* { - scrollbar-color: var(--scrollbar-color) transparent; - scrollbar-gutter: stable; -} - -/* scroll-to-top */ - -#scroll-to-top-button { - accent-color: var(--sl-color-text-accent); - background-color: var(--sl-color-text-accent); -} diff --git a/portal/typescript/tsconfig.json b/portal/typescript/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/portal/typescript/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/portal/typescript/wrangler.jsonc b/portal/typescript/wrangler.jsonc deleted file mode 100644 index a62b3060..00000000 --- a/portal/typescript/wrangler.jsonc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "dpkit-typescript", - // "preview_urls": true, - "compatibility_date": "2025-08-15", - "assets": { - "directory": "./build" - } -} diff --git a/scanner/README.md b/scanner/README.md deleted file mode 100644 index 1f3a676f..00000000 --- a/scanner/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/scanner - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/scanner/index.ts b/scanner/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/scanner/package.json b/scanner/package.json deleted file mode 100644 index fc9866b2..00000000 --- a/scanner/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "@dpkit/scanner", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "scanner", - "ocr" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "@dpkit/dataset": "workspace:*" - } -} diff --git a/scanner/tsconfig.json b/scanner/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/scanner/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/scanner/typedoc.json b/scanner/typedoc.json deleted file mode 100644 index f8e49f3a..00000000 --- a/scanner/typedoc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "entryPoints": ["index.ts"], - "skipErrorChecking": true -} diff --git a/table/README.md b/table/README.md deleted file mode 100644 index 21ef3358..00000000 --- a/table/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/table - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/table/assets/geojson.json b/table/assets/geojson.json deleted file mode 100644 index 4351e4b2..00000000 --- a/table/assets/geojson.json +++ /dev/null @@ -1,216 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "Geo JSON object", - "description": "Schema for a Geo JSON object", - "type": "object", - "required": ["type"], - "properties": { - "crs": { "$ref": "#/definitions/crs" }, - "bbox": { "$ref": "#/definitions/bbox" } - }, - "oneOf": [ - { "$ref": "#/definitions/geometry" }, - { "$ref": "#/definitions/geometryCollection" }, - { "$ref": "#/definitions/feature" }, - { "$ref": "#/definitions/featureCollection" } - ], - "definitions": { - "geometryCollection": { - "title": "GeometryCollection", - "description": "A collection of geometry objects", - "required": ["geometries"], - "properties": { - "type": { "enum": ["GeometryCollection"] }, - "geometries": { - "type": "array", - "items": { "$ref": "#/definitions/geometry" } - } - } - }, - "feature": { - "title": "Feature", - "description": "A Geo JSON feature object", - "required": ["geometry", "properties"], - "properties": { - "type": { "enum": ["Feature"] }, - "geometry": { - "oneOf": [{ "type": "null" }, { "$ref": "#/definitions/geometry" }] - }, - "properties": { "type": ["object", "null"] }, - "id": { "FIXME": "may be there, type not known (string? number?)" } - } - }, - "featureCollection": { - "title": "FeatureCollection", - "description": "A Geo JSON feature collection", - "required": ["features"], - "properties": { - "type": { "enum": ["FeatureCollection"] }, - "features": { - "type": "array", - "items": { "$ref": "#/definitions/feature" } - } - } - }, - "geometry": { - "title": "geometry", - "description": "One geometry as defined by GeoJSON", - "type": "object", - "required": ["type", "coordinates"], - "oneOf": [ - { - "title": "Point", - "properties": { - "type": { "enum": ["Point"] }, - "coordinates": { - "$ref": "#/definitions/geometry/definitions/position" - } - } - }, - { - "title": "MultiPoint", - "properties": { - "type": { "enum": ["MultiPoint"] }, - "coordinates": { - "$ref": "#/definitions/geometry/definitions/positionArray" - } - } - }, - { - "title": "LineString", - "properties": { - "type": { "enum": ["LineString"] }, - "coordinates": { - "$ref": "#/definitions/geometry/definitions/lineString" - } - } - }, - { - "title": "MultiLineString", - "properties": { - "type": { "enum": ["MultiLineString"] }, - "coordinates": { - "type": "array", - "items": { - "$ref": "#/definitions/geometry/definitions/lineString" - } - } - } - }, - { - "title": "Polygon", - "properties": { - "type": { "enum": ["Polygon"] }, - "coordinates": { - "$ref": "#/definitions/geometry/definitions/polygon" - } - } - }, - { - "title": "MultiPolygon", - "properties": { - "type": { "enum": ["MultiPolygon"] }, - "coordinates": { - "type": "array", - "items": { "$ref": "#/definitions/geometry/definitions/polygon" } - } - } - } - ], - "definitions": { - "position": { - "description": "A single position", - "type": "array", - "minItems": 2, - "items": [{ "type": "number" }, { "type": "number" }], - "additionalItems": false - }, - "positionArray": { - "description": "An array of positions", - "type": "array", - "items": { "$ref": "#/definitions/geometry/definitions/position" } - }, - "lineString": { - "description": "An array of two or more positions", - "allOf": [ - { "$ref": "#/definitions/geometry/definitions/positionArray" }, - { "minItems": 2 } - ] - }, - "linearRing": { - "description": "An array of four positions where the first equals the last", - "allOf": [ - { "$ref": "#/definitions/geometry/definitions/positionArray" }, - { "minItems": 4 } - ] - }, - "polygon": { - "description": "An array of linear rings", - "type": "array", - "items": { "$ref": "#/definitions/geometry/definitions/linearRing" } - } - } - }, - "crs": { - "title": "crs", - "description": "a Coordinate Reference System object", - "type": ["object", "null"], - "required": ["type", "properties"], - "properties": { - "type": { "type": "string" }, - "properties": { "type": "object" } - }, - "additionalProperties": false, - "oneOf": [ - { "$ref": "#/definitions/crs/definitions/namedCrs" }, - { "$ref": "#/definitions/crs/definitions/linkedCrs" } - ], - "definitions": { - "namedCrs": { - "properties": { - "type": { "enum": ["name"] }, - "properties": { - "required": ["name"], - "additionalProperties": false, - "properties": { - "name": { - "type": "string", - "FIXME": "semantic validation necessary" - } - } - } - } - }, - "linkedObject": { - "type": "object", - "required": ["href"], - "properties": { - "href": { - "type": "string", - "format": "uri", - "FIXME": "spec says \"dereferenceable\", cannot enforce that" - }, - "type": { - "type": "string", - "description": "Suggested values: proj4, ogjwkt, esriwkt" - } - } - }, - "linkedCrs": { - "properties": { - "type": { "enum": ["link"] }, - "properties": { - "$ref": "#/definitions/crs/definitions/linkedObject" - } - } - } - } - }, - "bbox": { - "description": "A bounding box as defined by GeoJSON", - "FIXME": "unenforceable constraint: even number of elements in array", - "type": "array", - "items": { "type": "number" } - } - } -} diff --git a/table/assets/topojson.json b/table/assets/topojson.json deleted file mode 100644 index a4e5b09c..00000000 --- a/table/assets/topojson.json +++ /dev/null @@ -1,259 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "TopoJSON object", - "description": "Schema for a TopoJSON object", - "type": "object", - "required": ["type"], - "properties": { - "bbox": { - "$ref": "#/definitions/bbox" - } - }, - "oneOf": [ - { - "$ref": "#/definitions/topology" - }, - { - "$ref": "#/definitions/geometry" - } - ], - "definitions": { - "bbox": { - "title": "TopoJSON bounding box", - "description": "A bounding box as defined by TopoJSON", - "type": "array", - "items": { - "$ref": "#/definitions/bbox/definitions/dimension" - }, - "minItems": 2, - "maxItems": 2, - "definitions": { - "dimension": { - "type": "array", - "description": "This array should have an entry per dimension in the geometries", - "items": { - "type": "number" - } - } - } - }, - "geometry": { - "title": "Geometry objects", - "description": "A Geometry object as defined by TopoJSON", - "type": "object", - "required": ["type"], - "properties": { - "id": { - "type": ["string", "integer"] - }, - "properties": { - "type": "object" - } - }, - "oneOf": [ - { - "title": "Point", - "description": "A Point Geometry object as defined by TopoJSON", - "required": ["type", "coordinates"], - "properties": { - "type": { - "enum": ["Point"] - }, - "coordinates": { - "$ref": "#/definitions/geometry/definitions/position" - } - } - }, - { - "title": "MultiPoint", - "description": "A MultiPoint Geometry object as defined by TopoJSON", - "required": ["type", "coordinates"], - "properties": { - "type": { - "enum": ["MultiPoint"] - }, - "coordinates": { - "type": "array", - "items": { - "$ref": "#/definitions/geometry/definitions/position" - } - } - } - }, - { - "title": "LineString", - "description": "A LineString Geometry object as defined by TopoJSON", - "required": ["type", "arcs"], - "properties": { - "type": { - "enum": ["LineString"] - }, - "arcs": { - "type": "array", - "items": { - "type": "integer" - } - } - } - }, - { - "title": "MultiLineString", - "description": "A MultiLineString Geometry object as defined by TopoJSON", - "required": ["type", "arcs"], - "properties": { - "type": { - "enum": ["MultiLineString"] - }, - "arcs": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "integer" - } - } - } - } - }, - { - "title": "Polygon", - "description": "A Polygon Geometry object as defined by TopoJSON", - "required": ["type", "arcs"], - "properties": { - "type": { - "enum": ["Polygon"] - }, - "arcs": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "integer" - } - } - } - } - }, - { - "title": "MultiPolygon", - "description": "A MultiPolygon Geometry object as defined by TopoJSON", - "required": ["type", "arcs"], - "properties": { - "type": { - "enum": ["MultiPolygon"] - }, - "arcs": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "integer" - } - } - } - } - } - }, - { - "title": "GeometryCollection", - "description": "A MultiPolygon Geometry object as defined by TopoJSON", - "required": ["type", "geometries"], - "properties": { - "type": { - "enum": ["GeometryCollection"] - }, - "geometries": { - "type": "array", - "items": { - "$ref": "#/definitions/geometry" - } - } - } - } - ], - "definitions": { - "position": { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 2 - } - } - }, - "topology": { - "title": "Topology", - "description": "A Topology object as defined by TopoJSON", - "type": "object", - "required": ["objects", "arcs"], - "properties": { - "type": { - "enum": ["Topology"] - }, - "objects": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/geometry" - } - }, - "arcs": { - "$ref": "#/definitions/topology/definitions/arcs" - }, - "transform": { - "$ref": "#/definitions/topology/definitions/transform" - }, - "bbox": { - "$ref": "#/definitions/bbox" - } - }, - "definitions": { - "transform": { - "type": "object", - "required": ["scale", "translate"], - "properties": { - "scale": { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 2 - }, - "translate": { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 2 - } - } - }, - "arcs": { - "type": "array", - "items": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/definitions/topology/definitions/position" - }, - { - "type": "null" - } - ] - }, - "minItems": 2 - } - }, - "position": { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 2 - } - } - } - } -} diff --git a/table/data/index.ts b/table/data/index.ts deleted file mode 100644 index d2f09ca2..00000000 --- a/table/data/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { getRecordsFromRows } from "./record.ts" -export type { DataRow, DataRecord } from "./record.ts" diff --git a/table/data/record.spec.ts b/table/data/record.spec.ts deleted file mode 100644 index d79e83d0..00000000 --- a/table/data/record.spec.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { describe, expect, it } from "vitest" -import { getRecordsFromRows } from "./record.ts" - -describe("getRecordsFromRows", () => { - it("should convert rows to records with default header", () => { - const rows = [ - ["name", "age", "city"], - ["Alice", 30, "NYC"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should handle single row with header", () => { - const rows = [["name", "age", "city"]] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([]) - }) - - it("should handle empty rows", () => { - const rows: unknown[][] = [] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([]) - }) - - it("should handle rows without header when header is false", () => { - const rows = [ - ["Alice", 30, "NYC"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows, { header: false }) - - expect(result).toEqual([ - { field1: "Alice", field2: 30, field3: "NYC" }, - { field1: "Bob", field2: 25, field3: "LA" }, - ]) - }) - - it("should handle custom header rows", () => { - const rows = [ - ["skip this row"], - ["name", "age", "city"], - ["Alice", 30, "NYC"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows, { headerRows: [2] }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should handle multiple header rows with default join", () => { - const rows = [ - ["first", "last", "contact"], - ["name", "name", "email"], - ["Alice", "Smith", "alice@example.com"], - ["Bob", "Jones", "bob@example.com"], - ] - - const result = getRecordsFromRows(rows, { headerRows: [1, 2] }) - - expect(result).toEqual([ - { - "first name": "Alice", - "last name": "Smith", - "contact email": "alice@example.com", - }, - { - "first name": "Bob", - "last name": "Jones", - "contact email": "bob@example.com", - }, - ]) - }) - - it("should handle multiple header rows with custom join", () => { - const rows = [ - ["user", "user", "meta"], - ["first", "last", "created"], - ["Alice", "Smith", "2023-01-01"], - ["Bob", "Jones", "2023-01-02"], - ] - - const result = getRecordsFromRows(rows, { - headerRows: [1, 2], - headerJoin: "_", - }) - - expect(result).toEqual([ - { user_first: "Alice", user_last: "Smith", meta_created: "2023-01-01" }, - { user_first: "Bob", user_last: "Jones", meta_created: "2023-01-02" }, - ]) - }) - - it("should skip comment rows by row number", () => { - const rows = [ - ["name", "age", "city"], - ["Alice", 30, "NYC"], - ["# Comment row", "ignored", "data"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows, { commentRows: [3] }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should skip rows with comment character", () => { - const rows = [ - ["name", "age", "city"], - ["Alice", 30, "NYC"], - ["# Comment", "ignored", "data"], - ["Bob", 25, "LA"], - ["Regular row", "data", "value"], - ] - - const result = getRecordsFromRows(rows, { commentChar: "#" }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - { name: "Regular row", age: "data", city: "value" }, - ]) - }) - - it("should skip rows with multiple comment characters", () => { - const rows = [ - ["name", "age", "city"], - ["Alice", 30, "NYC"], - ["# Comment 1", "ignored", "data"], - ["Bob", 25, "LA"], - ["## Comment 2", "ignored", "data"], - ] - - const result = getRecordsFromRows(rows, { commentChar: "#" }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should not skip rows when first cell is not string", () => { - const rows = [ - ["name", "age", "city"], - ["Alice", 30, "NYC"], - [123, "data", "test"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows, { commentChar: "#" }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: 123, age: "data", city: "test" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should handle rows with different lengths", () => { - const rows = [ - ["name", "age", "city", "country"], - ["Alice", 30, "NYC"], - ["Bob", 25, "LA", "USA"], - ["Charlie"], - ] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC", country: undefined }, - { name: "Bob", age: 25, city: "LA", country: "USA" }, - { - name: "Charlie", - age: undefined, - city: undefined, - country: undefined, - }, - ]) - }) - - it("should handle null and undefined values", () => { - const rows = [ - ["name", "age", "city"], - ["Alice", null, undefined], - [null, 25, "LA"], - ] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([ - { name: "Alice", age: null, city: undefined }, - { name: null, age: 25, city: "LA" }, - ]) - }) - - it("should handle boolean and number types", () => { - const rows = [ - ["name", "active", "count"], - ["Alice", true, 100], - ["Bob", false, 0], - ] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([ - { name: "Alice", active: true, count: 100 }, - { name: "Bob", active: false, count: 0 }, - ]) - }) - - it("should convert header values to strings", () => { - const rows = [ - [1, 2, 3], - ["Alice", 30, "NYC"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([ - { "1": "Alice", "2": 30, "3": "NYC" }, - { "1": "Bob", "2": 25, "3": "LA" }, - ]) - }) - - it("should handle empty header cells", () => { - const rows = [ - ["name", "", "city"], - ["Alice", 30, "NYC"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows) - - expect(result).toEqual([ - { name: "Alice", "": 30, city: "NYC" }, - { name: "Bob", "": 25, city: "LA" }, - ]) - }) - - it("should handle multi-row headers with empty cells", () => { - const rows = [ - ["person", "", "location"], - ["first", "last", "city"], - ["Alice", "Smith", "NYC"], - ] - - const result = getRecordsFromRows(rows, { headerRows: [1, 2] }) - - expect(result).toEqual([ - { "person first": "Alice", last: "Smith", "location city": "NYC" }, - ]) - }) - - it("should handle combination of headerRows and commentRows", () => { - const rows = [ - ["skip row 1"], - ["name", "age", "city"], - ["# Comment", "data", "data"], - ["Alice", 30, "NYC"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows, { - headerRows: [2], - commentRows: [3], - }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should handle combination of commentRows and commentChar", () => { - const rows = [ - ["name", "age", "city"], - ["# Inline comment", "data", "data"], - ["Alice", 30, "NYC"], - ["Comment by row number", "data", "data"], - ["Bob", 25, "LA"], - ] - - const result = getRecordsFromRows(rows, { - commentRows: [4], - commentChar: "#", - }) - - expect(result).toEqual([ - { name: "Alice", age: 30, city: "NYC" }, - { name: "Bob", age: 25, city: "LA" }, - ]) - }) - - it("should generate field names based on longest row when no header", () => { - const rows = [ - ["Alice", 30], - ["Bob", 25, "LA", "USA"], - ["Charlie", 35, "SF"], - ] - - const result = getRecordsFromRows(rows, { header: false }) - - expect(result).toEqual([ - { field1: "Alice", field2: 30, field3: undefined, field4: undefined }, - { field1: "Bob", field2: 25, field3: "LA", field4: "USA" }, - { field1: "Charlie", field2: 35, field3: "SF", field4: undefined }, - ]) - }) -}) diff --git a/table/data/record.ts b/table/data/record.ts deleted file mode 100644 index 570c1c3f..00000000 --- a/table/data/record.ts +++ /dev/null @@ -1,116 +0,0 @@ -import type { Dialect } from "@dpkit/metadata" - -export type DataRow = unknown[] -export type DataRecord = Record - -export function getRecordsFromRows(rows: DataRow[], dialect?: Dialect) { - const records: DataRecord[] = [] - - const header = getHeaderFromRows(rows, dialect) - const content = getContentFromRows(rows, dialect) - - const labels = getLabelsFromHeader(header, dialect) - if (!labels) { - return records - } - - for (const row of content) { - const isCommentedRow = getIsCommentedRow(row, dialect) - if (isCommentedRow) { - continue - } - - records.push( - Object.fromEntries(labels.map((label, index) => [label, row[index]])), - ) - } - - return records -} - -function getHeaderFromRows(rows: DataRow[], dialect?: Dialect) { - const hasHeader = dialect?.header !== false - const headerRows = dialect?.headerRows ?? [1] - - if (!hasHeader) { - const length = Math.max(...rows.map(row => row.length)) - const labels = Array.from({ length }, (_, idx) => `field${idx + 1}`) - - return [labels] - } - - const header: DataRow[] = [] - for (const number of headerRows) { - const row = rows[number - 1] - if (row) { - header.push(row) - } - } - - return header -} - -function getContentFromRows(rows: DataRow[], dialect?: Dialect) { - const hasHeader = dialect?.header !== false - const headerRows = dialect?.headerRows ?? (hasHeader ? [1] : []) - const commentRows = dialect?.commentRows ?? [] - const skipRows = headerRows[0] ? headerRows[0] - 1 : 0 - - const content: DataRow[] = [] - for (const [index, row] of rows.entries()) { - const number = index + 1 - - if (number <= skipRows) { - continue - } - - if (headerRows.includes(number)) { - continue - } - - if (commentRows.includes(number)) { - continue - } - - const isCommentedRow = getIsCommentedRow(row, dialect) - if (isCommentedRow) { - continue - } - - content.push(row) - } - - return content -} - -function getLabelsFromHeader(header: DataRow[], dialect?: Dialect) { - if (!header[0]) { - return undefined - } - - const labels = header[0].map(String) - const headerJoin = dialect?.headerJoin ?? " " - - for (const row of header.slice(1)) { - for (const [index, label] of row.entries()) { - const prefix = labels[index] ?? "" - labels[index] = [prefix, label].filter(Boolean).join(headerJoin) - } - } - - return labels -} - -function getIsCommentedRow(row: unknown[], dialect?: Dialect) { - const commentChar = dialect?.commentChar - - if (!commentChar) { - return false - } - - if (typeof row[0] !== "string") { - return false - } - - return row[0].startsWith(commentChar) -} diff --git a/table/dialect/Options.ts b/table/dialect/Options.ts deleted file mode 100644 index 4adcfb8a..00000000 --- a/table/dialect/Options.ts +++ /dev/null @@ -1,5 +0,0 @@ -// TODO: Implement - -export interface DialectOptions { - delimiter?: string -} diff --git a/table/dialect/index.ts b/table/dialect/index.ts deleted file mode 100644 index d68efff9..00000000 --- a/table/dialect/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { DialectOptions } from "./Options.ts" -export type { InferDialectOptions } from "./infer.ts" diff --git a/table/dialect/infer.ts b/table/dialect/infer.ts deleted file mode 100644 index 59344d57..00000000 --- a/table/dialect/infer.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { DialectOptions } from "./Options.ts" - -// TODO: Have some shared inferDialect* function? - -export interface InferDialectOptions extends DialectOptions { - sampleBytes?: number -} diff --git a/table/field/Field.ts b/table/field/Field.ts deleted file mode 100644 index e756a955..00000000 --- a/table/field/Field.ts +++ /dev/null @@ -1,6 +0,0 @@ -import * as pl from "nodejs-polars" - -export type PolarsField = { - name: string - type: pl.DataType -} diff --git a/table/field/Mapping.ts b/table/field/Mapping.ts deleted file mode 100644 index 9fc6c97c..00000000 --- a/table/field/Mapping.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type * as pl from "nodejs-polars" -import type { PolarsField } from "./Field.ts" - -export interface FieldMapping { - source: PolarsField - target: Field -} - -export interface CellMapping { - source: pl.Expr - target: pl.Expr -} diff --git a/table/field/checks/enum.spec.ts b/table/field/checks/enum.spec.ts deleted file mode 100644 index 887dbc0d..00000000 --- a/table/field/checks/enum.spec.ts +++ /dev/null @@ -1,376 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/enum)", () => { - it("should not errors for string values that are in the enum", async () => { - const table = pl - .DataFrame({ - status: ["pending", "approved", "rejected", "pending"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "status", - type: "string", - constraints: { - enum: ["pending", "approved", "rejected"], - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for values not in the enum", async () => { - const allowedValues = ["pending", "approved", "rejected"] - - const table = pl - .DataFrame({ - status: ["pending", "approved", "unknown", "cancelled", "rejected"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "status", - type: "string", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/enum")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/enum", - fieldName: "status", - enum: allowedValues, - rowNumber: 3, - cell: "unknown", - }) - expect(errors).toContainEqual({ - type: "cell/enum", - fieldName: "status", - enum: allowedValues, - rowNumber: 4, - cell: "cancelled", - }) - }) - - it("should handle null values correctly", async () => { - const table = pl - .DataFrame({ - status: ["pending", null, "approved", null], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "status", - type: "string", - constraints: { - enum: ["pending", "approved", "rejected"], - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/enum")).toHaveLength(0) - }) - - it("should handle case sensitivity correctly", async () => { - const allowedValues = ["pending", "approved", "rejected"] - - const table = pl - .DataFrame({ - status: ["Pending", "APPROVED", "rejected"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "status", - type: "string", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/enum")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/enum", - fieldName: "status", - enum: allowedValues, - rowNumber: 1, - cell: "Pending", - }) - expect(errors).toContainEqual({ - type: "cell/enum", - fieldName: "status", - enum: allowedValues, - rowNumber: 2, - cell: "APPROVED", - }) - }) - - it("should handle integer enum with string values", async () => { - const allowedValues = ["1", "2", "3"] - - const table = pl - .DataFrame({ - priority: ["1", "2", "5"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "priority", - type: "integer", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "priority", - enum: allowedValues, - rowNumber: 3, - cell: "5", - }, - ]) - }) - - it("should handle number enum with string values", async () => { - const allowedValues = ["1.5", "2.5", "3.5"] - - const table = pl - .DataFrame({ - rating: ["1.5", "2.5", "4.5"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "rating", - type: "number", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "rating", - enum: allowedValues, - rowNumber: 3, - cell: "4.5", - }, - ]) - }) - - it.skip("should handle date enum with string values", async () => { - const allowedValues = ["2024-01-01", "2024-02-01", "2024-03-01"] - - const table = pl - .DataFrame({ - date: ["2024-01-01", "2024-02-01", "2024-05-01"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "date", - type: "date", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "date", - enum: allowedValues, - rowNumber: 3, - cell: "2024-05-01", - }, - ]) - }) - - it.skip("should handle datetime enum with string values", async () => { - const allowedValues = [ - "2024-01-01T10:00:00", - "2024-01-01T14:00:00", - "2024-01-01T18:00:00", - ] - - const table = pl - .DataFrame({ - timestamp: [ - "2024-01-01T10:00:00", - "2024-01-01T14:00:00", - "2024-01-01T20:00:00", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "timestamp", - type: "datetime", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "timestamp", - enum: allowedValues, - rowNumber: 3, - cell: "2024-01-01T20:00:00", - }, - ]) - }) - - it("should handle year enum with string values", async () => { - const allowedValues = ["2020", "2021", "2022"] - - const table = pl - .DataFrame({ - year: ["2020", "2021", "2023"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "year", - type: "year", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "year", - enum: allowedValues, - rowNumber: 3, - cell: "2023", - }, - ]) - }) - - it.skip("should handle time enum with string values", async () => { - const allowedValues = ["10:00:00", "14:00:00", "18:00:00"] - - const table = pl - .DataFrame({ - time: ["10:00:00", "14:00:00", "20:00:00"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "time", - type: "time", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "time", - enum: allowedValues, - rowNumber: 3, - cell: "20:00:00", - }, - ]) - }) - - it.skip("should handle yearmonth enum with string values", async () => { - const allowedValues = ["2024-01", "2024-02", "2024-03"] - - const table = pl - .DataFrame({ - yearmonth: ["2024-01", "2024-02", "2024-05"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "yearmonth", - type: "yearmonth", - constraints: { - enum: allowedValues, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/enum", - fieldName: "yearmonth", - enum: allowedValues, - rowNumber: 3, - cell: "2024-05", - }, - ]) - }) -}) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts deleted file mode 100644 index 7806f36d..00000000 --- a/table/field/checks/enum.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellEnumError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { evaluateExpression } from "../../helpers.ts" -import type { CellMapping } from "../Mapping.ts" -import { parseDateField } from "../types/date.ts" -import { parseDatetimeField } from "../types/datetime.ts" -import { parseIntegerField } from "../types/integer.ts" -import { parseNumberField } from "../types/number.ts" -import { parseTimeField } from "../types/time.ts" -import { parseYearField } from "../types/year.ts" -import { parseYearmonthField } from "../types/yearmonth.ts" - -export function checkCellEnum(field: Field, mapping: CellMapping) { - if ( - field.type !== "string" && - field.type !== "integer" && - field.type !== "number" && - field.type !== "date" && - field.type !== "time" && - field.type !== "datetime" && - field.type !== "year" && - field.type !== "yearmonth" - ) { - return undefined - } - - const rawEnum = field.constraints?.enum - if (!rawEnum) return undefined - - let isErrorExpr: pl.Expr - try { - const parsedEnum = parseConstraint(field, rawEnum) - isErrorExpr = mapping.target.isIn(parsedEnum).not() - } catch (error) { - isErrorExpr = pl.pl.lit(true) - } - - const errorTemplate: CellEnumError = { - type: "cell/enum", - fieldName: field.name, - enum: rawEnum.map(String), - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} - -function parseConstraint(field: Field, value: number[] | string[]) { - return value.map(it => parseConstraintItem(field, it)) -} - -function parseConstraintItem(field: Field, value: number | string) { - if (typeof value !== "string") return value - - let expr = pl.pl.lit(value) - if (field.type === "integer") { - expr = parseIntegerField(field, expr) - } else if (field.type === "number") { - expr = parseNumberField(field, expr) - } else if (field.type === "date") { - expr = parseDateField(field, expr) - } else if (field.type === "time") { - expr = parseTimeField(field, expr) - } else if (field.type === "datetime") { - expr = parseDatetimeField(field, expr) - } else if (field.type === "year") { - expr = parseYearField(field, expr) - } else if (field.type === "yearmonth") { - expr = parseYearmonthField(field, expr) - } - - return evaluateExpression(expr) -} diff --git a/table/field/checks/maxLength.spec.ts b/table/field/checks/maxLength.spec.ts deleted file mode 100644 index d3a9de09..00000000 --- a/table/field/checks/maxLength.spec.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/maxLength)", () => { - it("should not errors for string values that meet the maxLength constraint", async () => { - const table = pl - .DataFrame({ - code: ["A123", "B456", "C789"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "code", - type: "string", - constraints: { maxLength: 4 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report an error for strings that are too long", async () => { - const table = pl - .DataFrame({ - username: ["bob", "alice", "christopher", "david"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "username", - type: "string", - constraints: { maxLength: 8 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/maxLength")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/maxLength", - fieldName: "username", - maxLength: 8, - rowNumber: 3, - cell: "christopher", - }) - }) -}) diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts deleted file mode 100644 index 704f0a6e..00000000 --- a/table/field/checks/maxLength.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellMaxLengthError } from "@dpkit/metadata" -import type { CellMapping } from "../Mapping.ts" - -export function checkCellMaxLength(field: Field, mapping: CellMapping) { - if (field.type !== "string") return undefined - - const maxLength = field.constraints?.maxLength - if (!maxLength) return undefined - - const isErrorExpr = mapping.target.str.lengths().gt(maxLength) - - const errorTemplate: CellMaxLengthError = { - type: "cell/maxLength", - fieldName: field.name, - maxLength: maxLength, - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts deleted file mode 100644 index b486aeef..00000000 --- a/table/field/checks/maximum.spec.ts +++ /dev/null @@ -1,543 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/maximum)", () => { - it("should not errors for valid values", async () => { - const table = pl - .DataFrame({ - price: [10.5, 20.75, 30.0], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - constraints: { maximum: 50 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report an error for invalid values", async () => { - const table = pl - .DataFrame({ - temperature: [20.5, 30.0, 40, 50.5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "temperature", - type: "number", - constraints: { maximum: 40 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/maximum")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/maximum", - fieldName: "temperature", - maximum: "40", - rowNumber: 4, - cell: "50.5", - }) - }) - - it("should report an error for invalid values (exclusive)", async () => { - const table = pl - .DataFrame({ - temperature: [20.5, 30.0, 40.0, 50.5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "temperature", - type: "number", - constraints: { exclusiveMaximum: 40 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/exclusiveMaximum")).toHaveLength( - 2, - ) - expect(errors).toContainEqual({ - type: "cell/exclusiveMaximum", - fieldName: "temperature", - maximum: "40", - rowNumber: 3, - cell: "40", - }) - expect(errors).toContainEqual({ - type: "cell/exclusiveMaximum", - fieldName: "temperature", - maximum: "40", - rowNumber: 4, - cell: "50.5", - }) - }) - - it("should handle maximum as string", async () => { - const table = pl - .DataFrame({ - price: [10.5, 20.75, 55.0], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - constraints: { maximum: "50" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "price", - maximum: "50", - rowNumber: 3, - cell: "55", - }, - ]) - }) - - it("should handle exclusiveMaximum as string", async () => { - const table = pl - .DataFrame({ - temperature: [20.5, 40.0, 50.5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "temperature", - type: "number", - constraints: { exclusiveMaximum: "40" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/exclusiveMaximum", - fieldName: "temperature", - maximum: "40", - rowNumber: 2, - cell: "40", - }, - { - type: "cell/exclusiveMaximum", - fieldName: "temperature", - maximum: "40", - rowNumber: 3, - cell: "50.5", - }, - ]) - }) - - it("should handle maximum as string with groupChar", async () => { - const table = pl - .DataFrame({ - price: ["5,000", "10,500", "15,000"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "integer", - groupChar: ",", - constraints: { maximum: "12,000" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "price", - maximum: "12,000", - rowNumber: 3, - cell: "15,000", - }, - ]) - }) - - it("should handle maximum as string with decimalChar", async () => { - const table = pl - .DataFrame({ - price: ["5,5", "10,75", "15,3"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - decimalChar: ",", - constraints: { maximum: "12,0" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "price", - maximum: "12,0", - rowNumber: 3, - cell: "15,3", - }, - ]) - }) - - it("should handle maximum as string with groupChar and decimalChar", async () => { - const table = pl - .DataFrame({ - price: ["5.000,50", "10.500,75", "15.000,30"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - groupChar: ".", - decimalChar: ",", - constraints: { maximum: "12.000,00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "price", - maximum: "12.000,00", - rowNumber: 3, - cell: "15.000,30", - }, - ]) - }) - - it("should handle maximum as string with bareNumber false", async () => { - const table = pl - .DataFrame({ - price: ["$5.00", "$10.50", "$15.50"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - bareNumber: false, - constraints: { maximum: "$12.00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "price", - maximum: "$12.00", - rowNumber: 3, - cell: "$15.50", - }, - ]) - }) - - it("should handle maximum for date fields", async () => { - const table = pl - .DataFrame({ - date: ["2024-01-15", "2024-02-20", "2024-03-25"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "date", - type: "date", - constraints: { maximum: "2024-02-28" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "date", - maximum: "2024-02-28", - rowNumber: 3, - cell: "2024-03-25", - }, - ]) - }) - - it.skip("should handle maximum for time fields", async () => { - const table = pl - .DataFrame({ - time: ["14:30:00", "16:45:00", "18:00:00"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "time", - type: "time", - constraints: { maximum: "17:00:00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "time", - maximum: "17:00:00", - rowNumber: 3, - cell: "18:00:00", - }, - ]) - }) - - it("should handle maximum for datetime fields", async () => { - const table = pl - .DataFrame({ - timestamp: [ - "2024-01-15T14:30:00", - "2024-02-20T08:15:00", - "2024-03-25T10:00:00", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "timestamp", - type: "datetime", - constraints: { maximum: "2024-02-28T23:59:59" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "timestamp", - maximum: "2024-02-28T23:59:59", - rowNumber: 3, - cell: "2024-03-25T10:00:00", - }, - ]) - }) - - it("should handle maximum for date fields with custom format", async () => { - const table = pl - .DataFrame({ - date: ["15/01/2024", "20/02/2024", "25/03/2024"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "date", - type: "date", - format: "%d/%m/%Y", - constraints: { maximum: "28/02/2024" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "date", - maximum: "28/02/2024", - rowNumber: 3, - cell: "25/03/2024", - }, - ]) - }) - - it("should handle maximum for year fields", async () => { - const table = pl - .DataFrame({ - year: ["2020", "2021", "2023"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "year", - type: "year", - constraints: { maximum: "2022" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "year", - maximum: "2022", - rowNumber: 3, - cell: "2023", - }, - ]) - }) - - it("should handle exclusiveMaximum for year fields", async () => { - const table = pl - .DataFrame({ - year: ["2020", "2021", "2022", "2023"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "year", - type: "year", - constraints: { exclusiveMaximum: "2022" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/exclusiveMaximum", - fieldName: "year", - maximum: "2022", - rowNumber: 3, - cell: "2022", - }, - { - type: "cell/exclusiveMaximum", - fieldName: "year", - maximum: "2022", - rowNumber: 4, - cell: "2023", - }, - ]) - }) - - it.skip("should handle maximum for yearmonth fields", async () => { - const table = pl - .DataFrame({ - yearmonth: ["2024-01", "2024-03", "2024-06"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "yearmonth", - type: "yearmonth", - constraints: { maximum: "2024-05" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/maximum", - fieldName: "yearmonth", - maximum: "2024-05", - rowNumber: 3, - cell: "2024-06", - }, - ]) - }) - - it.skip("should handle exclusiveMaximum for yearmonth fields", async () => { - const table = pl - .DataFrame({ - yearmonth: ["2024-01", "2024-03", "2024-05", "2024-06"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "yearmonth", - type: "yearmonth", - constraints: { exclusiveMaximum: "2024-05" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/exclusiveMaximum", - fieldName: "yearmonth", - maximum: "2024-05", - rowNumber: 3, - cell: "2024-05", - }, - { - type: "cell/exclusiveMaximum", - fieldName: "yearmonth", - maximum: "2024-05", - rowNumber: 4, - cell: "2024-06", - }, - ]) - }) -}) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts deleted file mode 100644 index 40df1df4..00000000 --- a/table/field/checks/maximum.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellExclusiveMaximumError } from "@dpkit/metadata" -import type { CellMaximumError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { evaluateExpression } from "../../helpers.ts" -import type { CellMapping } from "../Mapping.ts" -import { parseDateField } from "../types/date.ts" -import { parseDatetimeField } from "../types/datetime.ts" -import { parseIntegerField } from "../types/integer.ts" -import { parseNumberField } from "../types/number.ts" -import { parseTimeField } from "../types/time.ts" -import { parseYearField } from "../types/year.ts" -import { parseYearmonthField } from "../types/yearmonth.ts" - -export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { - return (field: Field, mapping: CellMapping) => { - if ( - field.type !== "integer" && - field.type !== "number" && - field.type !== "date" && - field.type !== "time" && - field.type !== "datetime" && - field.type !== "year" && - field.type !== "yearmonth" - ) { - return undefined - } - - const maximum = options?.isExclusive - ? field.constraints?.exclusiveMaximum - : field.constraints?.maximum - if (maximum === undefined) return undefined - - let isErrorExpr: pl.Expr - try { - const parsedMaximum = parseConstraint(field, maximum) - isErrorExpr = options?.isExclusive - ? mapping.target.gtEq(parsedMaximum) - : mapping.target.gt(parsedMaximum) - } catch (error) { - isErrorExpr = pl.pl.lit(true) - } - - const errorTemplate: CellMaximumError | CellExclusiveMaximumError = { - type: options?.isExclusive ? "cell/exclusiveMaximum" : "cell/maximum", - fieldName: field.name, - maximum: String(maximum), - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } - } -} - -function parseConstraint(field: Field, value: number | string) { - if (typeof value !== "string") return value - - let expr = pl.pl.lit(value) - if (field.type === "integer") { - expr = parseIntegerField(field, expr) - } else if (field.type === "number") { - expr = parseNumberField(field, expr) - } else if (field.type === "date") { - expr = parseDateField(field, expr) - } else if (field.type === "time") { - expr = parseTimeField(field, expr) - } else if (field.type === "datetime") { - expr = parseDatetimeField(field, expr) - } else if (field.type === "year") { - expr = parseYearField(field, expr) - } else if (field.type === "yearmonth") { - expr = parseYearmonthField(field, expr) - } - - return evaluateExpression(expr) -} diff --git a/table/field/checks/minLength.spec.ts b/table/field/checks/minLength.spec.ts deleted file mode 100644 index 42619bea..00000000 --- a/table/field/checks/minLength.spec.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/minLength)", () => { - it("should not errors for string values that meet the minLength constraint", async () => { - const table = pl - .DataFrame({ - code: ["A123", "B456", "C789"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "code", - type: "string", - constraints: { minLength: 3 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report an error for strings that are too short", async () => { - const table = pl - .DataFrame({ - username: ["bob", "a", "christopher", "ab"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "username", - type: "string", - constraints: { minLength: 3 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/minLength")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/minLength", - fieldName: "username", - minLength: 3, - rowNumber: 2, - cell: "a", - }) - expect(errors).toContainEqual({ - type: "cell/minLength", - fieldName: "username", - minLength: 3, - rowNumber: 4, - cell: "ab", - }) - }) -}) diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts deleted file mode 100644 index 86529b81..00000000 --- a/table/field/checks/minLength.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellMinLengthError } from "@dpkit/metadata" -import type { CellMapping } from "../Mapping.ts" - -export function checkCellMinLength(field: Field, mapping: CellMapping) { - if (field.type !== "string") return undefined - - const minLength = field.constraints?.minLength - if (!minLength) return undefined - - const isErrorExpr = mapping.target.str.lengths().lt(minLength) - - const errorTemplate: CellMinLengthError = { - type: "cell/minLength", - fieldName: field.name, - minLength: minLength, - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts deleted file mode 100644 index 15dba0e5..00000000 --- a/table/field/checks/minimum.spec.ts +++ /dev/null @@ -1,543 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/minimum)", () => { - it("should not errors for valid values", async () => { - const table = pl - .DataFrame({ - price: [10.5, 20.75, 30.0], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - constraints: { minimum: 5 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report an error for invalid values", async () => { - const table = pl - .DataFrame({ - temperature: [20.5, 30.0, 40, 3.5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "temperature", - type: "number", - constraints: { minimum: 10 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/minimum")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/minimum", - fieldName: "temperature", - minimum: "10", - rowNumber: 4, - cell: "3.5", - }) - }) - - it("should report an error for invalid values (exclusive)", async () => { - const table = pl - .DataFrame({ - temperature: [20.5, 30.0, 10.0, 5.5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "temperature", - type: "number", - constraints: { exclusiveMinimum: 10 }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/exclusiveMinimum")).toHaveLength( - 2, - ) - expect(errors).toContainEqual({ - type: "cell/exclusiveMinimum", - fieldName: "temperature", - minimum: "10", - rowNumber: 3, - cell: "10", - }) - expect(errors).toContainEqual({ - type: "cell/exclusiveMinimum", - fieldName: "temperature", - minimum: "10", - rowNumber: 4, - cell: "5.5", - }) - }) - - it("should handle minimum as string", async () => { - const table = pl - .DataFrame({ - price: [10.5, 20.75, 3.0], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - constraints: { minimum: "5" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "price", - minimum: "5", - rowNumber: 3, - cell: "3", - }, - ]) - }) - - it("should handle exclusiveMinimum as string", async () => { - const table = pl - .DataFrame({ - temperature: [20.5, 10.0, 5.5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "temperature", - type: "number", - constraints: { exclusiveMinimum: "10" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/exclusiveMinimum", - fieldName: "temperature", - minimum: "10", - rowNumber: 2, - cell: "10", - }, - { - type: "cell/exclusiveMinimum", - fieldName: "temperature", - minimum: "10", - rowNumber: 3, - cell: "5.5", - }, - ]) - }) - - it("should handle minimum as string with groupChar", async () => { - const table = pl - .DataFrame({ - price: ["5,000", "10,500", "2,500"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "integer", - groupChar: ",", - constraints: { minimum: "3,000" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "price", - minimum: "3,000", - rowNumber: 3, - cell: "2,500", - }, - ]) - }) - - it("should handle minimum as string with decimalChar", async () => { - const table = pl - .DataFrame({ - price: ["5,5", "10,75", "2,3"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - decimalChar: ",", - constraints: { minimum: "3,0" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "price", - minimum: "3,0", - rowNumber: 3, - cell: "2,3", - }, - ]) - }) - - it("should handle minimum as string with groupChar and decimalChar", async () => { - const table = pl - .DataFrame({ - price: ["5.000,50", "10.500,75", "2.500,30"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - groupChar: ".", - decimalChar: ",", - constraints: { minimum: "3.000,00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "price", - minimum: "3.000,00", - rowNumber: 3, - cell: "2.500,30", - }, - ]) - }) - - it("should handle minimum as string with bareNumber false", async () => { - const table = pl - .DataFrame({ - price: ["$5.00", "$10.50", "$2.50"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - bareNumber: false, - constraints: { minimum: "$3.00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "price", - minimum: "$3.00", - rowNumber: 3, - cell: "$2.50", - }, - ]) - }) - - it("should handle minimum for date fields", async () => { - const table = pl - .DataFrame({ - date: ["2024-01-15", "2024-02-20", "2024-01-05"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "date", - type: "date", - constraints: { minimum: "2024-01-10" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "date", - minimum: "2024-01-10", - rowNumber: 3, - cell: "2024-01-05", - }, - ]) - }) - - it.skip("should handle minimum for time fields", async () => { - const table = pl - .DataFrame({ - time: ["14:30:00", "16:45:00", "12:15:00"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "time", - type: "time", - constraints: { minimum: "13:00:00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "time", - minimum: "13:00:00", - rowNumber: 3, - cell: "12:15:00", - }, - ]) - }) - - it("should handle minimum for datetime fields", async () => { - const table = pl - .DataFrame({ - timestamp: [ - "2024-01-15T14:30:00", - "2024-02-20T08:15:00", - "2024-01-10T10:00:00", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "timestamp", - type: "datetime", - constraints: { minimum: "2024-01-15T00:00:00" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "timestamp", - minimum: "2024-01-15T00:00:00", - rowNumber: 3, - cell: "2024-01-10T10:00:00", - }, - ]) - }) - - it("should handle minimum for date fields with custom format", async () => { - const table = pl - .DataFrame({ - date: ["15/01/2024", "20/02/2024", "05/01/2024"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "date", - type: "date", - format: "%d/%m/%Y", - constraints: { minimum: "10/01/2024" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "date", - minimum: "10/01/2024", - rowNumber: 3, - cell: "05/01/2024", - }, - ]) - }) - - it("should handle minimum for year fields", async () => { - const table = pl - .DataFrame({ - year: ["2020", "2021", "2018"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "year", - type: "year", - constraints: { minimum: "2019" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "year", - minimum: "2019", - rowNumber: 3, - cell: "2018", - }, - ]) - }) - - it("should handle exclusiveMinimum for year fields", async () => { - const table = pl - .DataFrame({ - year: ["2020", "2021", "2019", "2018"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "year", - type: "year", - constraints: { exclusiveMinimum: "2019" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/exclusiveMinimum", - fieldName: "year", - minimum: "2019", - rowNumber: 3, - cell: "2019", - }, - { - type: "cell/exclusiveMinimum", - fieldName: "year", - minimum: "2019", - rowNumber: 4, - cell: "2018", - }, - ]) - }) - - it.skip("should handle minimum for yearmonth fields", async () => { - const table = pl - .DataFrame({ - yearmonth: ["2024-03", "2024-05", "2024-01"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "yearmonth", - type: "yearmonth", - constraints: { minimum: "2024-02" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/minimum", - fieldName: "yearmonth", - minimum: "2024-02", - rowNumber: 3, - cell: "2024-01", - }, - ]) - }) - - it.skip("should handle exclusiveMinimum for yearmonth fields", async () => { - const table = pl - .DataFrame({ - yearmonth: ["2024-03", "2024-05", "2024-02", "2024-01"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "yearmonth", - type: "yearmonth", - constraints: { exclusiveMinimum: "2024-02" }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/exclusiveMinimum", - fieldName: "yearmonth", - minimum: "2024-02", - rowNumber: 3, - cell: "2024-02", - }, - { - type: "cell/exclusiveMinimum", - fieldName: "yearmonth", - minimum: "2024-02", - rowNumber: 4, - cell: "2024-01", - }, - ]) - }) -}) diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts deleted file mode 100644 index 290c87f6..00000000 --- a/table/field/checks/minimum.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellExclusiveMinimumError } from "@dpkit/metadata" -import type { CellMinimumError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { evaluateExpression } from "../../helpers.ts" -import type { CellMapping } from "../Mapping.ts" -import { parseDateField } from "../types/date.ts" -import { parseDatetimeField } from "../types/datetime.ts" -import { parseIntegerField } from "../types/integer.ts" -import { parseNumberField } from "../types/number.ts" -import { parseTimeField } from "../types/time.ts" -import { parseYearField } from "../types/year.ts" -import { parseYearmonthField } from "../types/yearmonth.ts" - -export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { - return (field: Field, mapping: CellMapping) => { - if ( - field.type !== "integer" && - field.type !== "number" && - field.type !== "date" && - field.type !== "time" && - field.type !== "datetime" && - field.type !== "year" && - field.type !== "yearmonth" - ) { - return undefined - } - - const minimum = options?.isExclusive - ? field.constraints?.exclusiveMinimum - : field.constraints?.minimum - if (minimum === undefined) return undefined - - let isErrorExpr: pl.Expr - try { - const parsedMinimum = parseConstraint(field, minimum) - isErrorExpr = options?.isExclusive - ? mapping.target.ltEq(parsedMinimum) - : mapping.target.lt(parsedMinimum) - } catch (error) { - isErrorExpr = pl.pl.lit(true) - } - - const errorTemplate: CellMinimumError | CellExclusiveMinimumError = { - type: options?.isExclusive ? "cell/exclusiveMinimum" : "cell/minimum", - fieldName: field.name, - minimum: String(minimum), - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } - } -} - -function parseConstraint(field: Field, value: number | string) { - if (typeof value !== "string") return value - - let expr = pl.pl.lit(value) - if (field.type === "integer") { - expr = parseIntegerField(field, expr) - } else if (field.type === "number") { - expr = parseNumberField(field, expr) - } else if (field.type === "date") { - expr = parseDateField(field, expr) - } else if (field.type === "time") { - expr = parseTimeField(field, expr) - } else if (field.type === "datetime") { - expr = parseDatetimeField(field, expr) - } else if (field.type === "year") { - expr = parseYearField(field, expr) - } else if (field.type === "yearmonth") { - expr = parseYearmonthField(field, expr) - } - - return evaluateExpression(expr) -} diff --git a/table/field/checks/pattern.spec.ts b/table/field/checks/pattern.spec.ts deleted file mode 100644 index 52c052d5..00000000 --- a/table/field/checks/pattern.spec.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/pattern)", () => { - it("should not errors for string values that match the pattern", async () => { - const table = pl - .DataFrame({ - email: ["john@example.com", "alice@domain.org", "test@test.io"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "email", - type: "string", - constraints: { - pattern: "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report an error for strings that don't match the pattern", async () => { - const pattern = "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$" - - const table = pl - .DataFrame({ - email: [ - "john@example.com", - "alice@domain", - "test.io", - "valid@email.com", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "email", - type: "string", - constraints: { - pattern, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/pattern")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/pattern", - fieldName: "email", - pattern, - rowNumber: 2, - cell: "alice@domain", - }) - expect(errors).toContainEqual({ - type: "cell/pattern", - fieldName: "email", - pattern, - rowNumber: 3, - cell: "test.io", - }) - }) -}) diff --git a/table/field/checks/pattern.ts b/table/field/checks/pattern.ts deleted file mode 100644 index 2a106918..00000000 --- a/table/field/checks/pattern.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellPatternError } from "@dpkit/metadata" -import type { CellMapping } from "../Mapping.ts" - -export function checkCellPattern(field: Field, mapping: CellMapping) { - if (field.type !== "string") return undefined - - const pattern = field.constraints?.pattern - if (!pattern) return undefined - - const isErrorExpr = mapping.target.str.contains(pattern).not() - - const errorTemplate: CellPatternError = { - type: "cell/pattern", - fieldName: field.name, - pattern: pattern, - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/field/checks/required.spec.ts b/table/field/checks/required.spec.ts deleted file mode 100644 index 554b23ca..00000000 --- a/table/field/checks/required.spec.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (cell/required)", () => { - it("should report a cell/required error", async () => { - const table = pl - .DataFrame({ - id: [1, null, 3], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "id", type: "number", constraints: { required: true } }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/required", - fieldName: "id", - rowNumber: 2, - cell: "", - }) - }) -}) diff --git a/table/field/checks/required.ts b/table/field/checks/required.ts deleted file mode 100644 index 6196fc65..00000000 --- a/table/field/checks/required.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellRequiredError } from "@dpkit/metadata" -import type { CellMapping } from "../Mapping.ts" - -export function checkCellRequired(field: Field, mapping: CellMapping) { - const required = field.constraints?.required - if (!required) return undefined - - const isErrorExpr = mapping.target.isNull() - - const errorTemplate: CellRequiredError = { - type: "cell/required", - fieldName: field.name, - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/field/checks/type.spec.ts b/table/field/checks/type.spec.ts deleted file mode 100644 index ecadeb5e..00000000 --- a/table/field/checks/type.spec.ts +++ /dev/null @@ -1,298 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable", () => { - it("should validate string to integer convertions errors", async () => { - const table = pl - .DataFrame({ - id: ["1", "bad", "3", "4x"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "id", type: "integer" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "bad", - fieldName: "id", - fieldType: "integer", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "4x", - fieldName: "id", - fieldType: "integer", - rowNumber: 4, - }) - }) - - it("should validate string to number convertions errors", async () => { - const table = pl - .DataFrame({ - price: ["10.5", "twenty", "30.75", "$40"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "price", type: "number" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "twenty", - fieldName: "price", - fieldType: "number", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "$40", - fieldName: "price", - fieldType: "number", - rowNumber: 4, - }) - }) - - it("should validate string to boolean convertions errors", async () => { - const table = pl - .DataFrame({ - active: ["true", "yes", "false", "0", "1"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "active", type: "boolean" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "yes", - fieldName: "active", - fieldType: "boolean", - rowNumber: 2, - }) - }) - - it("should validate string to date convertions errors", async () => { - const table = pl - .DataFrame({ - created: ["2023-01-15", "Jan 15, 2023", "20230115", "not-a-date"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "created", type: "date" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(3) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "Jan 15, 2023", - fieldName: "created", - fieldType: "date", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "20230115", - fieldName: "created", - fieldType: "date", - rowNumber: 3, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "not-a-date", - fieldName: "created", - fieldType: "date", - rowNumber: 4, - }) - }) - - it("should validate string to time convertions errors", async () => { - const table = pl - .DataFrame({ - time: ["14:30:00", "2:30pm", "invalid", "14h30"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "time", type: "time" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(3) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "2:30pm", - fieldName: "time", - fieldType: "time", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "invalid", - fieldName: "time", - fieldType: "time", - rowNumber: 3, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "14h30", - fieldName: "time", - fieldType: "time", - rowNumber: 4, - }) - }) - - it("should validate string to time convertions errors with custom format", async () => { - const table = pl - .DataFrame({ - time: ["14:30", "invalid"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "time", type: "time", format: "%H:%M" }], - } - - const errors = await inspectTable(table, { schema }) - - console.log(errors) - - expect(errors).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "invalid", - fieldName: "time", - fieldType: "time", - fieldFormat: "%H:%M", - rowNumber: 2, - }) - }) - - it("should validate string to year convertions errors", async () => { - const table = pl - .DataFrame({ - year: ["2023", "23", "MMXXIII", "two-thousand-twenty-three"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "year", type: "year" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(3) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "23", - fieldName: "year", - fieldType: "year", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "MMXXIII", - fieldName: "year", - fieldType: "year", - rowNumber: 3, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "two-thousand-twenty-three", - fieldName: "year", - fieldType: "year", - rowNumber: 4, - }) - }) - - it("should validate string to datetime convertions errors", async () => { - const table = pl - .DataFrame({ - timestamp: [ - "2023-01-15T14:30:00", - "January 15, 2023 2:30 PM", - "2023-01-15 14:30", - "not-a-datetime", - ], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "datetime", type: "datetime" }], - } - - const errors = await inspectTable(table, { schema }) - - // Adjust the expectations to match actual behavior - expect(errors.length).toBeGreaterThan(0) - - // Check for specific invalid values we expect to fail - expect(errors).toContainEqual({ - type: "cell/type", - cell: "January 15, 2023 2:30 PM", - fieldName: "datetime", - fieldType: "datetime", - rowNumber: 2, - }) - - expect(errors).toContainEqual({ - type: "cell/type", - cell: "not-a-datetime", - fieldName: "datetime", - fieldType: "datetime", - rowNumber: 4, - }) - }) - - it("should pass validation when all cells are valid", async () => { - const table = pl - .DataFrame({ - id: ["1", "2", "3", "4"], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "id", type: "integer" }], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(0) - }) - - it("should validate with non-string source data", async () => { - const table = pl - .DataFrame({ - is_active: [true, false, 1, 0], - }) - .lazy() - - const schema: Schema = { - fields: [{ name: "is_active", type: "boolean" }], - } - - const errors = await inspectTable(table, { schema }) - - // Since the column isn't string type, validateField will not normalize it - expect(errors).toHaveLength(0) - }) -}) diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts deleted file mode 100644 index 20a9f335..00000000 --- a/table/field/checks/type.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellTypeError } from "@dpkit/metadata" -import type { CellMapping } from "../Mapping.ts" - -export function checkCellType(field: Field, mapping: CellMapping) { - const isErrorExpr = mapping.source.isNotNull().and(mapping.target.isNull()) - - const errorTemplate: CellTypeError = { - type: "cell/type", - fieldName: field.name, - fieldType: field.type ?? "any", - // @ts-ignore - fieldFormat: field.format, - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/field/checks/unique.spec.ts b/table/field/checks/unique.spec.ts deleted file mode 100644 index 72bf4577..00000000 --- a/table/field/checks/unique.spec.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -// TODO: recover -describe("inspectTable (cell/unique)", () => { - it("should not errors when all values are unique", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 4, 5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - constraints: { unique: true }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for duplicate values", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 2, 5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - constraints: { unique: true }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors.filter(e => e.type === "cell/unique")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/unique", - fieldName: "id", - rowNumber: 4, - cell: "2", - }) - }) - - it("should report multiple errors for string duplicates", async () => { - const table = pl - .DataFrame({ - code: ["A001", "B002", "A001", "C003", "B002"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "code", - type: "string", - constraints: { unique: true }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/unique")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/unique", - fieldName: "code", - rowNumber: 3, - cell: "A001", - }) - expect(errors).toContainEqual({ - type: "cell/unique", - fieldName: "code", - rowNumber: 5, - cell: "B002", - }) - }) - - it("should handle null values correctly", async () => { - const table = pl - .DataFrame({ - id: [1, null, 3, null, 5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - constraints: { unique: true }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) -}) diff --git a/table/field/checks/unique.ts b/table/field/checks/unique.ts deleted file mode 100644 index 419e60ad..00000000 --- a/table/field/checks/unique.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellUniqueError } from "@dpkit/metadata" -import type { CellMapping } from "../Mapping.ts" - -// TODO: Support schema.primaryKey and schema.uniqueKeys -export function checkCellUnique(field: Field, mapping: CellMapping) { - const unique = field.constraints?.unique - if (!unique) return undefined - - const isErrorExpr = mapping.target - .isNotNull() - .and(mapping.target.isFirstDistinct().not()) - - const errorTemplate: CellUniqueError = { - type: "cell/unique", - fieldName: field.name, - rowNumber: 0, - cell: "", - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/field/denormalize.ts b/table/field/denormalize.ts deleted file mode 100644 index 28efd055..00000000 --- a/table/field/denormalize.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { desubstituteField } from "./desubstitute.ts" -import { stringifyField } from "./stringify.ts" - -export type DenormalizeFieldOptions = { - nativeTypes?: Exclude[] -} - -export function denormalizeField( - field: Field, - options?: DenormalizeFieldOptions, -) { - let expr = pl.col(field.name) - const { nativeTypes } = options ?? {} - - if (!nativeTypes?.includes(field.type ?? "any")) { - expr = stringifyField(field, expr) - expr = desubstituteField(field, expr) - } - - return expr -} diff --git a/table/field/desubstitute.ts b/table/field/desubstitute.ts deleted file mode 100644 index 023b5194..00000000 --- a/table/field/desubstitute.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -const DEFAULT_MISSING_VALUE = "" - -export function desubstituteField(field: Field, fieldExpr: pl.Expr) { - const flattenMissingValues = field.missingValues?.map(it => - typeof it === "string" ? it : it.value, - ) - - const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE - fieldExpr = pl - .when(fieldExpr.isNull()) - .then(pl.lit(missingValue)) - .otherwise(fieldExpr) - .alias(field.name) - - return fieldExpr -} diff --git a/table/field/index.ts b/table/field/index.ts deleted file mode 100644 index 944e877a..00000000 --- a/table/field/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { denormalizeField } from "./denormalize.ts" -export { parseField } from "./parse.ts" -export { inspectField } from "./inspect.ts" -export { normalizeField } from "./normalize.ts" -export { stringifyField } from "./stringify.ts" -export type { PolarsField } from "./Field.ts" -export type { FieldMapping } from "./Mapping.ts" -export type { DenormalizeFieldOptions } from "./denormalize.ts" diff --git a/table/field/inspect.spec.ts b/table/field/inspect.spec.ts deleted file mode 100644 index 992c10e2..00000000 --- a/table/field/inspect.spec.ts +++ /dev/null @@ -1,441 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../table/inspect.ts" - -describe("inspectField", () => { - describe("field name validation", () => { - it("should report an error when field names don't match", async () => { - const table = pl - .DataFrame({ - actual_id: [1, 2, 3], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toContainEqual({ - type: "field/name", - fieldName: "id", - actualFieldName: "actual_id", - }) - }) - - it("should not errors when field names match", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should be case-sensitive when comparing field names", async () => { - const table = pl - .DataFrame({ - ID: [1, 2, 3], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(1) - expect(errors).toContainEqual({ - type: "field/name", - fieldName: "id", - actualFieldName: "ID", - }) - }) - }) - - describe("field type validation", () => { - it("should report an error when field types don't match", async () => { - const table = pl - .DataFrame({ - id: [true, false, true], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "integer", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(1) - expect(errors).toContainEqual({ - type: "field/type", - fieldName: "id", - fieldType: "integer", - actualFieldType: "boolean", - }) - }) - - it("should not errors when field types match", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "number", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - }) - - describe("cell types validation", () => { - it("should validate string to integer conversion errors", async () => { - const table = pl - .DataFrame({ - id: ["1", "bad", "3", "4x"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "integer", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "bad", - fieldName: "id", - fieldType: "integer", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "4x", - fieldName: "id", - fieldType: "integer", - rowNumber: 4, - }) - }) - - it("should validate string to number conversion errors", async () => { - const table = pl - .DataFrame({ - price: ["10.5", "twenty", "30.75", "$40"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "price", - type: "number", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "twenty", - fieldName: "price", - fieldType: "number", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "$40", - fieldName: "price", - fieldType: "number", - rowNumber: 4, - }) - }) - - it("should validate string to boolean conversion errors", async () => { - const table = pl - .DataFrame({ - active: ["true", "yes", "false", "0", "1"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "active", - type: "boolean", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(1) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "yes", - fieldName: "active", - fieldType: "boolean", - rowNumber: 2, - }) - }) - - it("should validate string to date conversion errors", async () => { - const table = pl - .DataFrame({ - created: ["2023-01-15", "Jan 15, 2023", "20230115", "not-a-date"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "created", - type: "date", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(3) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "Jan 15, 2023", - fieldName: "created", - fieldType: "date", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "20230115", - fieldName: "created", - fieldType: "date", - rowNumber: 3, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "not-a-date", - fieldName: "created", - fieldType: "date", - rowNumber: 4, - }) - }) - - it("should validate string to time conversion errors", async () => { - const table = pl - .DataFrame({ - time: ["14:30:00", "2:30pm", "invalid", "14h30"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "time", - type: "time", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(3) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "2:30pm", - fieldName: "time", - fieldType: "time", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "invalid", - fieldName: "time", - fieldType: "time", - rowNumber: 3, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "14h30", - fieldName: "time", - fieldType: "time", - rowNumber: 4, - }) - }) - - it("should validate string to year conversion errors", async () => { - const table = pl - .DataFrame({ - year: ["2023", "23", "MMXXIII", "two-thousand-twenty-three"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "year", - type: "year", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(3) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "23", - fieldName: "year", - fieldType: "year", - rowNumber: 2, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "MMXXIII", - fieldName: "year", - fieldType: "year", - rowNumber: 3, - }) - expect(errors).toContainEqual({ - type: "cell/type", - cell: "two-thousand-twenty-three", - fieldName: "year", - fieldType: "year", - rowNumber: 4, - }) - }) - - it("should validate string to datetime conversion errors", async () => { - const table = pl - .DataFrame({ - timestamp: [ - "2023-01-15T14:30:00", - "January 15, 2023 2:30 PM", - "2023-01-15 14:30", - "not-a-datetime", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "timestamp", - type: "datetime", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - // Adjust the expectations to match actual behavior - expect(errors.length).toBeGreaterThan(0) - - // Check for specific invalid values we expect to fail - expect(errors).toContainEqual({ - type: "cell/type", - cell: "January 15, 2023 2:30 PM", - fieldName: "timestamp", - fieldType: "datetime", - rowNumber: 2, - }) - - expect(errors).toContainEqual({ - type: "cell/type", - cell: "not-a-datetime", - fieldName: "timestamp", - fieldType: "datetime", - rowNumber: 4, - }) - }) - - it("should pass validation when all cells are valid", async () => { - const table = pl - .DataFrame({ - id: ["1", "2", "3", "4"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "id", - type: "integer", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toHaveLength(0) - }) - - it("should validate with non-string source data", async () => { - const table = pl - .DataFrame({ - is_active: [true, false, true, false], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "is_active", - type: "boolean", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - - // Since the column matches the expected type, validation passes - expect(errors).toHaveLength(0) - }) - }) -}) diff --git a/table/field/inspect.ts b/table/field/inspect.ts deleted file mode 100644 index b6b33107..00000000 --- a/table/field/inspect.ts +++ /dev/null @@ -1,176 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { CellError, FieldError, TableError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import type { Table } from "../table/index.ts" -import type { FieldMapping } from "./Mapping.ts" -import { checkCellEnum } from "./checks/enum.ts" -import { checkCellMaxLength } from "./checks/maxLength.ts" -import { createCheckCellMaximum } from "./checks/maximum.ts" -import { checkCellMinLength } from "./checks/minLength.ts" -import { createCheckCellMinimum } from "./checks/minimum.ts" -import { checkCellPattern } from "./checks/pattern.ts" -import { checkCellRequired } from "./checks/required.ts" -import { checkCellType } from "./checks/type.ts" -import { checkCellUnique } from "./checks/unique.ts" -import { normalizeField } from "./normalize.ts" -import { inspectArrayField } from "./types/array.ts" -import { inspectGeojsonField } from "./types/geojson.ts" -import { inspectObjectField } from "./types/object.ts" - -export async function inspectField( - mapping: FieldMapping, - table: Table, - options: { - maxErrors: number - }, -) { - const { maxErrors } = options - const errors: TableError[] = [] - - const nameErrors = inspectName(mapping) - errors.push(...nameErrors) - - const typeErrors = inspectType(mapping) - errors.push(...typeErrors) - - if (!typeErrors.length) { - const dataErorrs = await inspectCells(mapping, table, { maxErrors }) - errors.push(...dataErorrs) - } - - return errors -} - -function inspectName(mapping: FieldMapping) { - const errors: FieldError[] = [] - - if (mapping.source.name !== mapping.target.name) { - errors.push({ - type: "field/name", - fieldName: mapping.target.name, - actualFieldName: mapping.source.name, - }) - } - - return errors -} - -function inspectType(mapping: FieldMapping) { - const errors: FieldError[] = [] - const variant = mapping.source.type.variant - - // TODO: Rebase on proper polars type definition when available - // https://github.com/pola-rs/nodejs-polars/issues/372 - const compatMapping: Record = { - Bool: ["boolean"], - Categorical: ["string"], - Date: ["date"], - Datetime: ["datetime"], - Float32: ["number", "integer"], - Float64: ["number", "integer"], - Int16: ["integer"], - Int32: ["integer"], - Int64: ["integer"], - Int8: ["integer"], - List: ["list"], - String: ["any"], - Time: ["time"], - UInt16: ["integer"], - UInt32: ["integer"], - UInt64: ["integer"], - UInt8: ["integer"], - Utf8: ["any"], - } - - const compatTypes = compatMapping[variant] ?? [] - const isCompat = !!new Set(compatTypes).intersection( - new Set([mapping.target.type, "any"]), - ).size - - if (!isCompat) { - errors.push({ - type: "field/type", - fieldName: mapping.target.name, - fieldType: mapping.target.type ?? "any", - actualFieldType: compatTypes[0] ?? "any", - }) - } - - return errors -} - -async function inspectCells( - mapping: FieldMapping, - table: Table, - options: { - maxErrors: number - }, -) { - const { maxErrors } = options - const errors: CellError[] = [] - - // Types that require non-polars validation - switch (mapping.target.type) { - case "array": - return await inspectArrayField(mapping.target, table) - case "geojson": - return await inspectGeojsonField(mapping.target, table) - case "object": - return await inspectObjectField(mapping.target, table) - } - - let fieldCheckTable = table - .withRowCount() - .select( - pl.col("row_nr").add(1).alias("number"), - normalizeField(mapping).alias("target"), - normalizeField(mapping, { keepType: true }).alias("source"), - pl.lit(null).alias("error"), - ) - - for (const checkCell of [ - checkCellType, - checkCellRequired, - checkCellPattern, - checkCellEnum, - createCheckCellMinimum(), - createCheckCellMaximum(), - createCheckCellMinimum({ isExclusive: true }), - createCheckCellMaximum({ isExclusive: true }), - checkCellMinLength, - checkCellMaxLength, - checkCellUnique, - ]) { - const cellMapping = { source: pl.col("source"), target: pl.col("target") } - - const check = checkCell(mapping.target, cellMapping) - if (!check) continue - - fieldCheckTable = fieldCheckTable.withColumn( - pl - .when(pl.col("error").isNotNull()) - .then(pl.col("error")) - .when(check.isErrorExpr) - .then(pl.lit(JSON.stringify(check.errorTemplate))) - .otherwise(pl.lit(null)) - .alias("error"), - ) - } - - const fieldCheckFrame = await fieldCheckTable - .filter(pl.col("error").isNotNull()) - .drop(["target"]) - .head(maxErrors) - .collect() - - for (const row of fieldCheckFrame.toRecords() as any[]) { - const errorTemplate = JSON.parse(row.error) as CellError - errors.push({ - ...errorTemplate, - rowNumber: row.number, - cell: String(row.source ?? ""), - }) - } - - return errors -} diff --git a/table/field/narrow.spec.ts b/table/field/narrow.spec.ts deleted file mode 100644 index 9fc190c7..00000000 --- a/table/field/narrow.spec.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../table/inspect.ts" -import { normalizeTable } from "../table/normalize.ts" - -describe("narrowField", () => { - it("should narrow float to integer", async () => { - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["a", "b", "c"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()).toEqual([ - { id: 1, name: "a" }, - { id: 2, name: "b" }, - { id: 3, name: "c" }, - ]) - }) - - it("should detect error when float cannot be narrowed to integer", async () => { - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.5], - name: ["a", "b", "c"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "id", - fieldType: "integer", - rowNumber: 3, - cell: "3.5", - }, - ]) - }) -}) diff --git a/table/field/narrow.ts b/table/field/narrow.ts deleted file mode 100644 index adb7570a..00000000 --- a/table/field/narrow.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as pl from "nodejs-polars" -import type { FieldMapping } from "./Mapping.ts" - -export function narrowField(mapping: FieldMapping, fieldExpr: pl.Expr) { - const variant = mapping.source.type.variant - - if (mapping.target.type === "integer") { - if (["Float32", "Float64"].includes(variant)) { - fieldExpr = pl - .when(fieldExpr.eq(fieldExpr.round(0))) - .then(fieldExpr.cast(pl.Int64)) - .otherwise(pl.lit(null)) - } - } - - return fieldExpr -} diff --git a/table/field/normalize.ts b/table/field/normalize.ts deleted file mode 100644 index f795aa18..00000000 --- a/table/field/normalize.ts +++ /dev/null @@ -1,20 +0,0 @@ -import * as pl from "nodejs-polars" -import type { FieldMapping } from "./Mapping.ts" -import { narrowField } from "./narrow.ts" -import { parseField } from "./parse.ts" -import { substituteField } from "./substitute.ts" - -export function normalizeField( - mapping: FieldMapping, - options?: { keepType?: boolean }, -) { - let fieldExpr = pl.col(mapping.source.name) - fieldExpr = substituteField(mapping, fieldExpr) - - if (!options?.keepType) { - fieldExpr = parseField(mapping, fieldExpr) - fieldExpr = narrowField(mapping, fieldExpr) - } - - return fieldExpr.alias(mapping.target.name) -} diff --git a/table/field/parse.spec.ts b/table/field/parse.spec.ts deleted file mode 100644 index a28f9f87..00000000 --- a/table/field/parse.spec.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../table/index.ts" - -describe("parseField", () => { - describe("missing values", () => { - it.each([ - // Schema-level - ["", null, {}], - ["", "", { schemaLevel: [] }], - ["-", null, { schemaLevel: ["-"] }], - ["x", null, { schemaLevel: ["x"] }], - - // Field-level - ["", null, {}], - ["-", null, { fieldLevel: ["-"] }], - ["-", "-", { fieldLevel: [""] }], - ["n/a", null, { fieldLevel: ["n/a"] }], - - // Schema-level and field-level - ["-", null, { schemaLevel: ["x"], fieldLevel: ["-"] }], - ["-", "-", { schemaLevel: ["-"], fieldLevel: ["x"] }], - // @ts-ignore - ])("$0 -> $1 $2", async (cell, value, { fieldLevel, schemaLevel }) => { - const table = pl.DataFrame({ name: [cell] }).lazy() - const schema: Schema = { - missingValues: schemaLevel, - fields: [{ name: "name", type: "string", missingValues: fieldLevel }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").get(0)).toEqual(value) - }) - }) -}) diff --git a/table/field/parse.ts b/table/field/parse.ts deleted file mode 100644 index bfcd7abe..00000000 --- a/table/field/parse.ts +++ /dev/null @@ -1,48 +0,0 @@ -import * as pl from "nodejs-polars" -import type { FieldMapping } from "./Mapping.ts" -import { parseBooleanField } from "./types/boolean.ts" -import { parseDateField } from "./types/date.ts" -import { parseDatetimeField } from "./types/datetime.ts" -import { parseDurationField } from "./types/duration.ts" -import { parseGeopointField } from "./types/geopoint.ts" -import { parseIntegerField } from "./types/integer.ts" -import { parseListField } from "./types/list.ts" -import { parseNumberField } from "./types/number.ts" -import { parseStringField } from "./types/string.ts" -import { parseTimeField } from "./types/time.ts" -import { parseYearField } from "./types/year.ts" -import { parseYearmonthField } from "./types/yearmonth.ts" - -export function parseField(mapping: FieldMapping, fieldExpr: pl.Expr) { - if (!mapping.source.type.equals(pl.String)) return fieldExpr - - const field = mapping.target - switch (field.type) { - case "boolean": - return parseBooleanField(field, fieldExpr) - case "date": - return parseDateField(field, fieldExpr) - case "datetime": - return parseDatetimeField(field, fieldExpr) - case "duration": - return parseDurationField(field, fieldExpr) - case "geopoint": - return parseGeopointField(field, fieldExpr) - case "integer": - return parseIntegerField(field, fieldExpr) - case "list": - return parseListField(field, fieldExpr) - case "number": - return parseNumberField(field, fieldExpr) - case "string": - return parseStringField(field, fieldExpr) - case "time": - return parseTimeField(field, fieldExpr) - case "year": - return parseYearField(field, fieldExpr) - case "yearmonth": - return parseYearmonthField(field, fieldExpr) - default: - return fieldExpr - } -} diff --git a/table/field/stringify.spec.ts b/table/field/stringify.spec.ts deleted file mode 100644 index 826c25b1..00000000 --- a/table/field/stringify.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable } from "../table/index.ts" - -describe("stringifyField", () => { - describe("missing values", () => { - it.each([ - // Schema-level - null values should be converted to first missing value - [null, "", {}], - [null, "", { schemaLevel: [] }], // defaults to "" - [null, "-", { schemaLevel: ["-"] }], - [null, "x", { schemaLevel: ["x"] }], - - // Regular values should remain unchanged - ["hello", "hello", {}], - ["world", "world", { schemaLevel: ["-"] }], - - // Field-level missing values take precedence - [null, "", {}], // default field-level missing value - [null, "-", { fieldLevel: ["-"] }], - [null, "n/a", { fieldLevel: ["n/a"] }], - - // Regular values with field-level settings - ["test", "test", { fieldLevel: ["-"] }], - ["value", "value", { fieldLevel: ["n/a"] }], - - // Field-level overrides schema-level - [null, "-", { schemaLevel: ["x"], fieldLevel: ["-"] }], - [null, "x", { schemaLevel: ["-"], fieldLevel: ["x"] }], - - // Multiple missing values - should use first one - [null, "-", { fieldLevel: ["-", "n/a", "null"] }], - [null, "n/a", { schemaLevel: ["n/a", "NULL", ""] }], - - // @ts-ignore - ])("%s -> %s %s", async (value, expected, { fieldLevel, schemaLevel }) => { - const table = pl.DataFrame({ name: [value] }).lazy() - const schema: Schema = { - missingValues: schemaLevel, - fields: [{ name: "name", type: "string", missingValues: fieldLevel }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) -}) diff --git a/table/field/stringify.ts b/table/field/stringify.ts deleted file mode 100644 index 6b56d505..00000000 --- a/table/field/stringify.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type * as pl from "nodejs-polars" -import { stringifyBooleanField } from "./types/boolean.ts" -import { stringifyDateField } from "./types/date.ts" -import { stringifyDatetimeField } from "./types/datetime.ts" -import { stringifyDurationField } from "./types/duration.ts" -import { stringifyGeopointField } from "./types/geopoint.ts" -import { stringifyIntegerField } from "./types/integer.ts" -import { stringifyListField } from "./types/list.ts" -import { stringifyNumberField } from "./types/number.ts" -import { stringifyStringField } from "./types/string.ts" -import { stringifyTimeField } from "./types/time.ts" -import { stringifyYearField } from "./types/year.ts" -import { stringifyYearmonthField } from "./types/yearmonth.ts" - -export function stringifyField(field: Field, fieldExpr: pl.Expr) { - switch (field.type) { - case "boolean": - return stringifyBooleanField(field, fieldExpr) - case "date": - return stringifyDateField(field, fieldExpr) - case "datetime": - return stringifyDatetimeField(field, fieldExpr) - case "duration": - return stringifyDurationField(field, fieldExpr) - case "geopoint": - return stringifyGeopointField(field, fieldExpr) - case "integer": - return stringifyIntegerField(field, fieldExpr) - case "list": - return stringifyListField(field, fieldExpr) - case "number": - return stringifyNumberField(field, fieldExpr) - case "string": - return stringifyStringField(field, fieldExpr) - case "time": - return stringifyTimeField(field, fieldExpr) - case "year": - return stringifyYearField(field, fieldExpr) - case "yearmonth": - return stringifyYearmonthField(field, fieldExpr) - default: - return fieldExpr - } -} diff --git a/table/field/substitute.ts b/table/field/substitute.ts deleted file mode 100644 index 0bbac3fa..00000000 --- a/table/field/substitute.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as pl from "nodejs-polars" -import type { FieldMapping } from "./Mapping.ts" - -const DEFAULT_MISSING_VALUES = [""] - -export function substituteField(mapping: FieldMapping, fieldExpr: pl.Expr) { - if (!mapping.source.type.equals(pl.String)) return fieldExpr - - const flattenMissingValues = - mapping.target.missingValues?.map(it => - typeof it === "string" ? it : it.value, - ) ?? DEFAULT_MISSING_VALUES - - if (flattenMissingValues.length) { - fieldExpr = pl - .when(fieldExpr.isIn(flattenMissingValues)) - .then(pl.lit(null)) - .otherwise(fieldExpr) - .alias(mapping.target.name) - } - - return fieldExpr -} diff --git a/table/field/types/array.spec.ts b/table/field/types/array.spec.ts deleted file mode 100644 index e328f598..00000000 --- a/table/field/types/array.spec.ts +++ /dev/null @@ -1,396 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("validateArrayField", () => { - it("should not errors for valid JSON arrays", async () => { - const table = pl - .DataFrame({ - tags: ['["tag1","tag2"]', "[1,2,3]", '["a","b","c"]'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "tags", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for empty arrays", async () => { - const table = pl - .DataFrame({ - items: ["[]", "[]", "[]"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "items", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for null values", async () => { - const table = pl - .DataFrame({ - data: ['["value"]', null, "[1,2,3]"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for JSON objects", async () => { - const table = pl - .DataFrame({ - data: ["[1,2,3]", '{"key":"value"}', '["a","b"]'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 2, - cell: '{"key":"value"}', - }, - ]) - }) - - it("should errors for invalid JSON", async () => { - const table = pl - .DataFrame({ - data: ['["valid"]', "invalid json", "[1,2,3]", "[broken"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 2, - cell: "invalid json", - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 4, - cell: "[broken", - }) - }) - - it("should handle nested arrays", async () => { - const table = pl - .DataFrame({ - matrix: ["[[1,2],[3,4]]", "[[5,6],[7,8]]", '[["a","b"],["c","d"]]'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "matrix", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for empty strings", async () => { - const table = pl - .DataFrame({ - data: ['["valid"]', "", "[1,2,3]"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 2, - cell: "", - }, - ]) - }) - - it("should errors for JSON primitives", async () => { - const table = pl - .DataFrame({ - data: ['"string"', "123", "true", "false", "null"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "array", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 1, - cell: '"string"', - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 2, - cell: "123", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 3, - cell: "true", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 4, - cell: "false", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "array", - rowNumber: 5, - cell: "null", - }, - ]) - }) - - it("should not errors for arrays matching jsonSchema", async () => { - const table = pl - .DataFrame({ - scores: ["[80,90,100]", "[75,85,95]", "[90,95,100]"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "scores", - type: "array", - constraints: { - jsonSchema: { - type: "array", - items: { type: "number" }, - minItems: 3, - maxItems: 3, - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for arrays not matching jsonSchema", async () => { - const jsonSchema = { - type: "array", - items: { type: "number" }, - minItems: 2, - } - - const table = pl - .DataFrame({ - numbers: ["[1,2,3]", '["not","numbers"]', "[1]", "[4,5,6]"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "numbers", - type: "array", - constraints: { - jsonSchema, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/jsonSchema")).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "numbers", - rowNumber: 2, - cell: '["not","numbers"]', - pointer: "/0", - message: "must be number", - }, - { - type: "cell/jsonSchema", - fieldName: "numbers", - rowNumber: 2, - cell: '["not","numbers"]', - pointer: "/1", - message: "must be number", - }, - { - type: "cell/jsonSchema", - fieldName: "numbers", - rowNumber: 3, - cell: "[1]", - pointer: "", - message: "must NOT have fewer than 2 items", - }, - ]) - }) - - it("should validate complex jsonSchema with array of objects", async () => { - const table = pl - .DataFrame({ - users: [ - '[{"name":"John","age":30},{"name":"Jane","age":25}]', - '[{"name":"Bob","age":"invalid"}]', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "users", - type: "array", - constraints: { - jsonSchema: { - type: "array", - items: { - type: "object", - properties: { - name: { type: "string" }, - age: { type: "number" }, - }, - required: ["name", "age"], - }, - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "users", - rowNumber: 2, - cell: '[{"name":"Bob","age":"invalid"}]', - pointer: "/0/age", - message: "must be number", - }, - ]) - }) - - it("should validate jsonSchema with unique items constraint", async () => { - const table = pl - .DataFrame({ - tags: ['["unique","values"]', '["duplicate","duplicate"]'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "tags", - type: "array", - constraints: { - jsonSchema: { - type: "array", - items: { type: "string" }, - uniqueItems: true, - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "tags", - rowNumber: 2, - cell: '["duplicate","duplicate"]', - pointer: "", - message: - "must NOT have duplicate items (items ## 1 and 0 are identical)", - }, - ]) - }) -}) diff --git a/table/field/types/array.ts b/table/field/types/array.ts deleted file mode 100644 index e1d9a10e..00000000 --- a/table/field/types/array.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { ArrayField } from "@dpkit/metadata" -import type { Table } from "../../table/index.ts" -import { inspectJsonField } from "./json.ts" - -export async function inspectArrayField(field: ArrayField, table: Table) { - return inspectJsonField(field, table) -} diff --git a/table/field/types/boolean.spec.ts b/table/field/types/boolean.spec.ts deleted file mode 100644 index 51fcf6a4..00000000 --- a/table/field/types/boolean.spec.ts +++ /dev/null @@ -1,91 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../../table/index.ts" -import { denormalizeTable } from "../../table/index.ts" - -describe("parseBooleanField", () => { - it.each([ - // Default true values - ["true", true, {}], - ["True", true, {}], - ["TRUE", true, {}], - ["1", true, {}], - - // Default false values - ["false", false, {}], - ["False", false, {}], - ["FALSE", false, {}], - ["0", false, {}], - - // Invalid values - ["", null, {}], - ["invalid", null, {}], - ["truthy", null, {}], - ["falsy", null, {}], - ["2", null, {}], - ["-100", null, {}], - ["t", null, {}], - ["f", null, {}], - ["3.14", null, {}], - - // Custom true values - ["Y", true, { trueValues: ["Y", "y", "yes"] }], - ["y", true, { trueValues: ["Y", "y", "yes"] }], - ["yes", true, { trueValues: ["Y", "y", "yes"] }], - ["true", null, { trueValues: ["Y", "y", "yes"] }], - - // Custom false values - ["N", false, { falseValues: ["N", "n", "no"] }], - ["n", false, { falseValues: ["N", "n", "no"] }], - ["no", false, { falseValues: ["N", "n", "no"] }], - ["false", null, { falseValues: ["N", "n", "no"] }], - - // Custom true and false values - ["oui", true, { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], - ["si", true, { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], - ["non", false, { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], - ["no", false, { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], - ])("%s -> %s %o", async (cell, value, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "boolean" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) -}) - -describe("stringifyBooleanField", () => { - it.each([ - // Default values - [true, "true", {}], - [false, "false", {}], - - // Custom true values - [true, "Y", { trueValues: ["Y", "y", "yes"] }], - [false, "false", { trueValues: ["Y", "y", "yes"] }], - - // Custom false values - [true, "true", { falseValues: ["N", "n", "no"] }], - [false, "N", { falseValues: ["N", "n", "no"] }], - - // Custom true and false values - [true, "oui", { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], - [false, "non", { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], - ])("%s -> %s %o", async (value, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Bool)]).lazy() - - const schema = { - fields: [{ name: "name", type: "boolean" as const, ...options }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/boolean.ts b/table/field/types/boolean.ts deleted file mode 100644 index 82c24c9f..00000000 --- a/table/field/types/boolean.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { BooleanField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -const DEFAULT_TRUE_VALUES = ["true", "True", "TRUE", "1"] -const DEFAULT_FALSE_VALUES = ["false", "False", "FALSE", "0"] - -export function parseBooleanField(field: BooleanField, fieldExpr: pl.Expr) { - const trueValues = field.trueValues ?? DEFAULT_TRUE_VALUES - const falseValues = field.falseValues ?? DEFAULT_FALSE_VALUES - - for (const value of trueValues) fieldExpr = fieldExpr.replace(value, "1") - for (const value of falseValues) fieldExpr = fieldExpr.replace(value, "0") - - fieldExpr = fieldExpr.cast(pl.Int8) - - return pl - .when(fieldExpr.eq(1)) - .then(pl.lit(true)) - .when(fieldExpr.eq(0)) - .then(pl.lit(false)) - .otherwise(pl.lit(null)) - .alias(field.name) -} - -const DEFAULT_TRUE_VALUE = "true" -const DEFAULT_FALSE_VALUE = "false" - -export function stringifyBooleanField(field: BooleanField, fieldExpr: pl.Expr) { - const trueValue = field.trueValues?.[0] ?? DEFAULT_TRUE_VALUE - const falseValue = field.falseValues?.[0] ?? DEFAULT_FALSE_VALUE - - return pl - .when(fieldExpr.eq(pl.lit(true))) - .then(pl.lit(trueValue)) - .otherwise(pl.lit(falseValue)) - .alias(field.name) -} diff --git a/table/field/types/date.spec.ts b/table/field/types/date.spec.ts deleted file mode 100644 index 0f6d2d32..00000000 --- a/table/field/types/date.spec.ts +++ /dev/null @@ -1,62 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../../table/index.ts" -import { denormalizeTable } from "../../table/index.ts" - -describe("parseDateField", () => { - it.each([ - // Default format - ["2019-01-01", new Date(Date.UTC(2019, 0, 1)), {}], - ["10th Jan 1969", null, {}], - ["invalid", null, {}], - ["", null, {}], - - // Custom format - ["21/11/2006", new Date(Date.UTC(2006, 10, 21)), { format: "%d/%m/%Y" }], - [ - "21/11/06 16:30", - new Date(Date.UTC(2006, 10, 21)), - { format: "%d/%m/%y" }, - ], - ["invalid", null, { format: "%d/%m/%y" }], - ["", null, { format: "%d/%m/%y" }], - ["2006/11/21", new Date(Date.UTC(2006, 10, 21)), { format: "%Y/%m/%d" }], - - // Invalid format - ["21/11/06", null, { format: "invalid" }], - ])("%s -> %s %o", async (cell, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "date" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) - -describe("stringifyDateField", () => { - it.each([ - // Default format - [new Date(Date.UTC(2019, 0, 1)), "2019-01-01", {}], - [new Date(Date.UTC(2006, 10, 21)), "2006-11-21", {}], - - // Custom format - [new Date(Date.UTC(2006, 10, 21)), "21/11/2006", { format: "%d/%m/%Y" }], - [new Date(Date.UTC(2006, 10, 21)), "2006/11/21", { format: "%Y/%m/%d" }], - ])("%s -> %s %o", async (value, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Date)]).lazy() - - const schema = { - fields: [{ name: "name", type: "date" as const, ...options }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/date.ts b/table/field/types/date.ts deleted file mode 100644 index ef07c9f5..00000000 --- a/table/field/types/date.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { DateField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -const DEFAULT_FORMAT = "%Y-%m-%d" - -export function parseDateField(field: DateField, fieldExpr: pl.Expr) { - let format = DEFAULT_FORMAT - if (field.format && field.format !== "default" && field.format !== "any") { - format = field.format - } - - return fieldExpr.str.strptime(pl.Date, format) -} - -export function stringifyDateField(field: DateField, fieldExpr: pl.Expr) { - const format = field.format ?? DEFAULT_FORMAT - - return fieldExpr.date.strftime(format) -} diff --git a/table/field/types/datetime.spec.ts b/table/field/types/datetime.spec.ts deleted file mode 100644 index 516b0143..00000000 --- a/table/field/types/datetime.spec.ts +++ /dev/null @@ -1,72 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../../table/index.ts" -import { denormalizeTable } from "../../table/index.ts" - -// TODO: Enable this test suite after this issue is fixed: -// https://github.com/pola-rs/nodejs-polars/issues/365 -describe.skip("parseDatetimeField", () => { - it.each([ - // Default format - ["2014-01-01T06:00:00", new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), {}], - ["2014-01-01T06:00:00Z", new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), {}], - ["Mon 1st Jan 2014 9 am", null, {}], - ["invalid", null, {}], - ["", null, {}], - - // Custom formats - [ - "21/11/2006 16:30", - new Date(2006, 10, 21, 16, 30), - { format: "%d/%m/%Y %H:%M" }, - ], - ["16:30 21/11/06", null, { format: "%H:%M %d/%m/%y" }], // Incorrect format - ["invalid", null, { format: "%d/%m/%y %H:%M" }], - ["", null, { format: "%d/%m/%y %H:%M" }], - - // Invalid format - ["21/11/06 16:30", null, { format: "invalid" }], - ])("%s -> %s %o", async (cell, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "datetime" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) - -describe("stringifyDatetimeField", () => { - it.each([ - // Default format - [new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), "2014-01-01T06:00:00", {}], - [new Date(Date.UTC(2006, 10, 21, 16, 30, 0)), "2006-11-21T16:30:00", {}], - - // Custom format - [ - new Date(Date.UTC(2006, 10, 21, 16, 30, 0)), - "21/11/2006 16:30", - { format: "%d/%m/%Y %H:%M" }, - ], - [ - new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), - "2014/01/01T06:00:00", - { format: "%Y/%m/%dT%H:%M:%S" }, - ], - ])("%s -> %s %o", async (value, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Datetime)]).lazy() - - const schema = { - fields: [{ name: "name", type: "datetime" as const, ...options }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/datetime.ts b/table/field/types/datetime.ts deleted file mode 100644 index 8ba6da0e..00000000 --- a/table/field/types/datetime.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { DatetimeField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -const DEFAULT_FORMAT = "%Y-%m-%dT%H:%M:%S" - -// TODO: Add support for timezone handling -export function parseDatetimeField(field: DatetimeField, fieldExpr: pl.Expr) { - let format = DEFAULT_FORMAT - if (field.format && field.format !== "default" && field.format !== "any") { - format = field.format - } - - return fieldExpr.str.strptime(pl.Datetime, format) -} - -export function stringifyDatetimeField( - field: DatetimeField, - fieldExpr: pl.Expr, -) { - const format = field.format ?? DEFAULT_FORMAT - - return fieldExpr.date.strftime(format) -} diff --git a/table/field/types/duration.spec.ts b/table/field/types/duration.spec.ts deleted file mode 100644 index e8535ceb..00000000 --- a/table/field/types/duration.spec.ts +++ /dev/null @@ -1,47 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseDurationField", () => { - it.each([["P23DT23H", "P23DT23H", {}]])( - "$0 -> $1 $2", - async (cell, value, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - const schema = { - fields: [{ name: "name", type: "duration" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").get(0)).toEqual(value) - }, - ) -}) - -describe("stringifyDurationField", () => { - it.each([ - // ISO 8601 duration strings should be returned as-is - ["P23DT23H", "P23DT23H"], - ["P1Y2M3DT4H5M6S", "P1Y2M3DT4H5M6S"], - ["PT30M", "PT30M"], - ["P1D", "P1D"], - ["PT1H", "PT1H"], - ["P1W", "P1W"], - ["PT0S", "PT0S"], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "duration" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/duration.ts b/table/field/types/duration.ts deleted file mode 100644 index 47bbae3e..00000000 --- a/table/field/types/duration.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { DurationField } from "@dpkit/metadata" -import type * as pl from "nodejs-polars" - -// TODO: raise an issue on nodejs-polars repo as this is not supported yet -// So we do nothing on this column type for now -export function parseDurationField(_field: DurationField, fieldExpr: pl.Expr) { - return fieldExpr -} - -export function stringifyDurationField( - _field: DurationField, - fieldExpr: pl.Expr, -) { - return fieldExpr -} diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts deleted file mode 100644 index 86fcd3a2..00000000 --- a/table/field/types/geojson.spec.ts +++ /dev/null @@ -1,576 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("validateGeojsonField", () => { - it("should not errors for valid GeoJSON Point", async () => { - const table = pl - .DataFrame({ - location: [ - '{"type":"Point","coordinates":[0,0]}', - '{"type":"Point","coordinates":[12.5,41.9]}', - '{"type":"Point","coordinates":[-73.9,40.7]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "location", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for valid GeoJSON geometries", async () => { - const table = pl - .DataFrame({ - geometry: [ - '{"type":"LineString","coordinates":[[0,0],[1,1]]}', - '{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,1],[0,0]]]}', - '{"type":"MultiPoint","coordinates":[[0,0],[1,1]]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "geometry", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for valid GeoJSON Feature", async () => { - const table = pl - .DataFrame({ - feature: [ - '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Test"}}', - '{"type":"Feature","geometry":{"type":"LineString","coordinates":[[0,0],[1,1]]},"properties":{"id":1}}', - '{"type":"Feature","geometry":null,"properties":{}}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "feature", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for valid GeoJSON FeatureCollection", async () => { - const table = pl - .DataFrame({ - collection: [ - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', - '{"type":"FeatureCollection","features":[]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "collection", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for null values", async () => { - const table = pl - .DataFrame({ - location: [ - '{"type":"Point","coordinates":[0,0]}', - null, - '{"type":"Feature","geometry":null,"properties":{}}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "location", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for JSON arrays", async () => { - const table = pl - .DataFrame({ - data: [ - '{"type":"Point","coordinates":[0,0]}', - "[[0,0],[1,1]]", - '{"type":"Feature","geometry":null,"properties":{}}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 2, - cell: "[[0,0],[1,1]]", - }, - ]) - }) - - it("should errors for invalid JSON", async () => { - const table = pl - .DataFrame({ - data: [ - '{"type":"Point","coordinates":[0,0]}', - "invalid json", - "{broken}", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 2, - cell: "invalid json", - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 3, - cell: "{broken}", - }) - }) - - it("should errors for empty strings", async () => { - const table = pl - .DataFrame({ - data: [ - '{"type":"Point","coordinates":[0,0]}', - "", - '{"type":"Feature","geometry":null,"properties":{}}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 2, - cell: "", - }, - ]) - }) - - it("should errors for JSON primitives", async () => { - const table = pl - .DataFrame({ - data: ['"string"', "123", "true", "false", "null"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 1, - cell: '"string"', - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 2, - cell: "123", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 3, - cell: "true", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 4, - cell: "false", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "geojson", - rowNumber: 5, - cell: "null", - }, - ]) - }) - - it("should errors for invalid GeoJSON Point coordinates", async () => { - const table = pl - .DataFrame({ - location: [ - '{"type":"Point","coordinates":[0,0]}', - '{"type":"Point","coordinates":[0]}', - '{"type":"Point","coordinates":[0,0,0,0]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "location", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "location", - fieldType: "geojson", - rowNumber: 2, - cell: '{"type":"Point","coordinates":[0]}', - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "location", - fieldType: "geojson", - rowNumber: 3, - cell: '{"type":"Point","coordinates":[0,0,0,0]}', - }) - }) - - it("should errors for invalid GeoJSON LineString", async () => { - const table = pl - .DataFrame({ - line: [ - '{"type":"LineString","coordinates":[[0,0],[1,1]]}', - '{"type":"LineString","coordinates":[[0,0]]}', - '{"type":"LineString","coordinates":[0,0]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "line", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "line", - fieldType: "geojson", - rowNumber: 2, - cell: '{"type":"LineString","coordinates":[[0,0]]}', - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "line", - fieldType: "geojson", - rowNumber: 3, - cell: '{"type":"LineString","coordinates":[0,0]}', - }) - }) - - it("should errors for incomplete GeoJSON Feature", async () => { - const table = pl - .DataFrame({ - feature: [ - '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}', - '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]}}', - '{"type":"Feature","properties":{}}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "feature", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "feature", - fieldType: "geojson", - rowNumber: 2, - cell: '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]}}', - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "feature", - fieldType: "geojson", - rowNumber: 3, - cell: '{"type":"Feature","properties":{}}', - }) - }) - - it("should errors for invalid GeoJSON FeatureCollection", async () => { - const table = pl - .DataFrame({ - collection: [ - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', - '{"type":"FeatureCollection"}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "collection", - type: "geojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "collection", - fieldType: "geojson", - rowNumber: 2, - cell: '{"type":"FeatureCollection"}', - }, - ]) - }) - - it("should not validate jsonSchema constraints for geojson fields", async () => { - const table = pl - .DataFrame({ - location: [ - '{"type":"Point","coordinates":[0,0]}', - '{"type":"Point","coordinates":[100,200]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "location", - type: "geojson", - constraints: { - jsonSchema: { - type: "object", - properties: { - coordinates: { - type: "array", - items: { type: "number", minimum: -50, maximum: 50 }, - }, - }, - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should not errors for valid TopoJSON", async () => { - const table = pl - .DataFrame({ - topology: [ - '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[{"type":"Point","coordinates":[0,0]}]}},"arcs":[]}', - '{"type":"Topology","objects":{"collection":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "topology", - type: "geojson", - format: "topojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for invalid TopoJSON structure", async () => { - const table = pl - .DataFrame({ - topology: [ - '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', - '{"type":"Topology","objects":{}}', - '{"type":"Topology"}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "topology", - type: "geojson", - format: "topojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "topology", - fieldType: "geojson", - fieldFormat: "topojson", - rowNumber: 2, - cell: '{"type":"Topology","objects":{}}', - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "topology", - fieldType: "geojson", - fieldFormat: "topojson", - rowNumber: 3, - cell: '{"type":"Topology"}', - }) - }) - - it("should accept TopoJSON geometry objects", async () => { - const table = pl - .DataFrame({ - geometry: [ - '{"type":"Point","coordinates":[0,0]}', - '{"type":"LineString","arcs":[0,1]}', - '{"type":"Polygon","arcs":[[0,1,2]]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "geometry", - type: "geojson", - format: "topojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should handle null values for topojson format", async () => { - const table = pl - .DataFrame({ - topology: [ - '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', - null, - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "topology", - type: "geojson", - format: "topojson", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) -}) diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts deleted file mode 100644 index 98fc4300..00000000 --- a/table/field/types/geojson.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { GeojsonField } from "@dpkit/metadata" -import geojson from "../../assets/geojson.json" with { type: "json" } -import topojson from "../../assets/topojson.json" with { type: "json" } -import type { Table } from "../../table/index.ts" -import { inspectJsonField } from "./json.ts" - -export async function inspectGeojsonField(field: GeojsonField, table: Table) { - return inspectJsonField(field, table, { - formatJsonSchema: field.format === "topojson" ? topojson : geojson, - }) -} diff --git a/table/field/types/geopoint.spec.ts b/table/field/types/geopoint.spec.ts deleted file mode 100644 index c3ff7ded..00000000 --- a/table/field/types/geopoint.spec.ts +++ /dev/null @@ -1,206 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseGeopointField", () => { - describe("default format", () => { - it.each([ - // Valid geopoints in default format (lon,lat) - ["90.50,45.50", [90.5, 45.5]], - ["0,0", [0, 0]], - //["-122.40, 37.78", [-122.4, 37.78]], - //["-180.0,-90.0", [-180.0, -90.0]], - //["180.0, 90.0", [180.0, 90.0]], - - // With whitespace - //[" 90.50, 45.50 ", [90.5, 45.5]], - - // Invalid formats - //["not a geopoint", null], - //["", null], - //["90.50", null], - //["90.50,lat", null], - //["lon,45.50", null], - //["90.50,45.50,0", null], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "geopoint" as const }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("array format", () => { - it.each([ - // Valid geopoints in array format - ["[90.50, 45.50]", [90.5, 45.5]], - ["[0, 0]", [0, 0]], - ["[-122.40, 37.78]", [-122.4, 37.78]], - ["[-180.0, -90.0]", [-180.0, -90.0]], - ["[180.0, 90.0]", [180.0, 90.0]], - - // With whitespace - [" [90.50, 45.50] ", [90.5, 45.5]], - - // Invalid formats - // TODO: fix this - //["not a geopoint", null], - //["", null], - //["[90.50]", null], - //["[90.50, 45.50, 0]", null], - //["['lon', 'lat']", null], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "geopoint" as const, format: "array" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("object format", () => { - it.each([ - // Valid geopoints in object format - ['{"lon": 90.50, "lat": 45.50}', [90.5, 45.5]], - ['{"lon": 0, "lat": 0}', [0, 0]], - ['{"lon": -122.40, "lat": 37.78}', [-122.4, 37.78]], - ['{"lon": -180.0, "lat": -90.0}', [-180.0, -90.0]], - ['{"lon": 180.0, "lat": 90.0}', [180.0, 90.0]], - - // With whitespace - [' {"lon": 90.50, "lat": 45.50} ', [90.5, 45.5]], - - // TODO: fix this - // Invalid formats - //["not a geopoint", null], - //["", null], - //['{"longitude": 90.50, "latitude": 45.50}', null], - //['{"lon": 90.50}', null], - //['{"lat": 45.50}', null], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { - name: "name", - type: "geopoint" as const, - format: "object" as const, - }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) -}) - -describe("stringifyGeopointField", () => { - describe("default format", () => { - it.each([ - // Coordinate arrays to default format (lon,lat) - [[90.5, 45.5], "90.5,45.5"], - [[0, 0], "0.0,0.0"], - [[-122.4, 37.78], "-122.4,37.78"], - [[-180.0, -90.0], "-180.0,-90.0"], - [[180.0, 90.0], "180.0,90.0"], - - // With precise decimals - [[125.6789, 10.1234], "125.6789,10.1234"], - - // Null handling - //[null, null], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.Float64))]) - .lazy() - - const schema = { - fields: [{ name: "name", type: "geopoint" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) - - describe("array format", () => { - it.each([ - // Coordinate arrays to array format string - [[90.5, 45.5], "[90.5,45.5]"], - [[0, 0], "[0.0,0.0]"], - [[-122.4, 37.78], "[-122.4,37.78]"], - [[-180.0, -90.0], "[-180.0,-90.0]"], - [[180.0, 90.0], "[180.0,90.0]"], - - // Null handling - //[null, null], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.Float64))]) - .lazy() - - const schema = { - fields: [ - { name: "name", type: "geopoint" as const, format: "array" as const }, - ], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) - - describe("object format", () => { - it.each([ - // Coordinate arrays to object format string - [[90.5, 45.5], '{"lon":90.5,"lat":45.5}'], - [[0, 0], '{"lon":0.0,"lat":0.0}'], - [[-122.4, 37.78], '{"lon":-122.4,"lat":37.78}'], - [[-180.0, -90.0], '{"lon":-180.0,"lat":-90.0}'], - [[180.0, 90.0], '{"lon":180.0,"lat":90.0}'], - - // Null handling - //[null, null], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.Float64))]) - .lazy() - - const schema = { - fields: [ - { - name: "name", - type: "geopoint" as const, - format: "object" as const, - }, - ], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) -}) diff --git a/table/field/types/geopoint.ts b/table/field/types/geopoint.ts deleted file mode 100644 index 1e71f5c5..00000000 --- a/table/field/types/geopoint.ts +++ /dev/null @@ -1,79 +0,0 @@ -import type { GeopointField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -// TODO: -// Add more validation: -// - Check the length of the list is 2 (no list.lenghts in polars currently) -// - Check the values are within -180..180 and -90..90 -// - Return null instead of list if any of the values are out of range - -export function parseGeopointField(field: GeopointField, fieldExpr: pl.Expr) { - // Default format is "lon,lat" string - const format = field.format ?? "default" - - if (format === "default") { - fieldExpr = fieldExpr.str.split(",").cast(pl.List(pl.Float64)) - } - - if (format === "array") { - fieldExpr = fieldExpr.str - .replaceAll("[\\[\\]\\s]", "") - .str.split(",") - .cast(pl.List(pl.Float64)) - } - - if (format === "object") { - fieldExpr = pl - .concatList([ - fieldExpr.str.jsonPathMatch("$.lon").cast(pl.Float64), - fieldExpr.str.jsonPathMatch("$.lat").cast(pl.Float64), - ]) - .alias(field.name) - } - - return fieldExpr -} - -export function stringifyGeopointField( - field: GeopointField, - fieldExpr: pl.Expr, -) { - // Default format is "lon,lat" string - const format = field.format ?? "default" - - if (format === "default") { - return fieldExpr.cast(pl.List(pl.String)).lst.join(",") - } - - if (format === "array") { - return pl - .concatString( - [ - pl.lit("["), - fieldExpr.lst.get(0).cast(pl.String), - pl.lit(","), - fieldExpr.lst.get(1).cast(pl.String), - pl.lit("]"), - ], - "", - ) - .alias(field.name) as pl.Expr - } - - if (format === "object") { - return pl - .concatString( - [ - pl.lit('{"lon":'), - fieldExpr.lst.get(0).cast(pl.String), - pl.lit(',"lat":'), - fieldExpr.lst.get(1).cast(pl.String), - pl.lit("}"), - ], - "", - ) - .alias(field.name) as pl.Expr - } - - return fieldExpr -} diff --git a/table/field/types/integer.spec.ts b/table/field/types/integer.spec.ts deleted file mode 100644 index d45e7f3d..00000000 --- a/table/field/types/integer.spec.ts +++ /dev/null @@ -1,121 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseIntegerField", () => { - it.each([ - // Basic integer parsing - ["1", 1, {}], - ["2", 2, {}], - ["1000", 1000, {}], - - // Empty or invalid values - ["", null, {}], - ["2.1", null, {}], - ["bad", null, {}], - ["0.0003", null, {}], - ["3.14", null, {}], - ["1/2", null, {}], - - // Group character handling - ["1", 1, { groupChar: "," }], - ["1,000", 1000, { groupChar: "," }], - ["1,000,000", 1000000, { groupChar: "," }], - ["1 000", 1000, { groupChar: " " }], - ["1'000'000", 1000000, { groupChar: "'" }], - ["1.000.000", 1000000, { groupChar: "." }], - - // Bare number handling - ["1", 1, { bareNumber: false }], - ["1000", 1000, { bareNumber: false }], - ["$1000", 1000, { bareNumber: false }], - ["1000$", 1000, { bareNumber: false }], - ["€1000", 1000, { bareNumber: false }], - ["1000€", 1000, { bareNumber: false }], - ["1,000", null, { bareNumber: false }], - ["-12€", -12, { bareNumber: false }], - ["€-12", -12, { bareNumber: false }], - - // Leading zeros and whitespace - ["000835", 835, {}], - ["0", 0, {}], - ["00", 0, {}], - ["01", 1, {}], - //[" 01 ", 1, {}], - //[" 42 ", 42, {}], - - // Combined cases - ["$1,000,000", 1000000, { bareNumber: false, groupChar: "," }], - ["1,000,000$", 1000000, { bareNumber: false, groupChar: "," }], - ["€ 1.000.000", 1000000, { bareNumber: false, groupChar: "." }], - //[" -1,000 ", -1000, { groupChar: "," }], - ["000,001", 1, { groupChar: "," }], - ])("$0 -> $1 $2", async (cell, value, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "integer" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").get(0)).toEqual(value) - expect(frame.getColumn("name").get(0)).toEqual(value) - }) - - describe("categories", () => { - it.each([ - // Flat categories - ["1", 1, { categories: [1, 2] }], - ["2", 2, { categories: [1, 2] }], - ["3", null, { categories: [1, 2] }], - - // Object categories - ["1", 1, { categories: [{ value: 1, label: "One" }] }], - ["2", null, { categories: [{ value: 1, label: "One" }] }], - ])("$0 -> $1 $2", async (cell, value, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "integer" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) -}) - -describe("stringifyIntegerField", () => { - it.each([ - // Basic integer to string conversion - [1, "1"], - [2, "2"], - [1000, "1000"], - [42, "42"], - [-1, "-1"], - [-100, "-100"], - [0, "0"], - - // Large integers - [1234567890, "1234567890"], - [-1234567890, "-1234567890"], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Int64)]).lazy() - - const schema = { - fields: [{ name: "name", type: "integer" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/integer.ts b/table/field/types/integer.ts deleted file mode 100644 index 05722a58..00000000 --- a/table/field/types/integer.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { IntegerField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -// TODO: support categories -// TODO: support categoriesOrder -export function parseIntegerField(field: IntegerField, fieldExpr: pl.Expr) { - const groupChar = field.groupChar - const bareNumber = field.bareNumber - const flattenCategories = field.categories?.map(it => - typeof it === "number" ? it : it.value, - ) - - // Handle non-bare numbers (with currency symbols, percent signs, etc.) - if (bareNumber === false) { - // Preserve the minus sign when removing leading characters - fieldExpr = fieldExpr.str.replaceAll("^[^\\d\\-]+", "") - fieldExpr = fieldExpr.str.replaceAll("[^\\d\\-]+$", "") - } - - // Handle group character (thousands separator) - if (groupChar) { - // Escape special characters for regex - const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - fieldExpr = fieldExpr.str.replaceAll(escapedGroupChar, "") - } - - // Cast to int64 (will handle values up to 2^63-1) - fieldExpr = fieldExpr.cast(pl.Int64) - - // Currently, only string categories are supported - if (flattenCategories) { - return pl - .when(fieldExpr.isIn(flattenCategories)) - .then(fieldExpr) - .otherwise(pl.lit(null)) - .alias(field.name) - } - - return fieldExpr -} - -export function stringifyIntegerField( - _field: IntegerField, - fieldExpr: pl.Expr, -) { - // Convert to string - fieldExpr = fieldExpr.cast(pl.String) - - //const groupChar = field.groupChar - //const bareNumber = field.bareNumber - - // TODO: Add group character formatting (thousands separator) when needed - // TODO: Add non-bare number formatting (currency symbols, etc.) when needed - - return fieldExpr -} diff --git a/table/field/types/json.ts b/table/field/types/json.ts deleted file mode 100644 index 465d959c..00000000 --- a/table/field/types/json.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { ArrayField, GeojsonField, ObjectField } from "@dpkit/metadata" -import { inspectJsonValue } from "@dpkit/metadata" -import type { CellError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { isObject } from "../../helpers.ts" -import type { Table } from "../../table/index.ts" - -// TODO: Improve the implementation -// Make unblocking / handle large data / process in parallel / move processing to Rust? - -export async function inspectJsonField( - field: ArrayField | GeojsonField | ObjectField, - table: Table, - options?: { - formatJsonSchema?: Record - }, -) { - const errors: CellError[] = [] - - const formatJsonSchema = options?.formatJsonSchema - const constraintJsonSchema = field.constraints?.jsonSchema - - const frame = await table - .withRowCount() - .select( - pl.pl.col("row_nr").add(1).alias("number"), - pl.pl.col(field.name).alias("source"), - ) - .collect() - - for (const row of frame.toRecords() as any[]) { - if (row.source === null) continue - - let target: Record | undefined - const checkCompat = field.type === "array" ? Array.isArray : isObject - - try { - target = JSON.parse(row.source) - } catch (error) {} - - if (!target || !checkCompat(target)) { - errors.push({ - type: "cell/type", - cell: String(row.source), - fieldName: field.name, - fieldType: field.type, - fieldFormat: field.format, - rowNumber: row.number, - }) - - continue - } - - if (formatJsonSchema) { - const formatErrors = await inspectJsonValue(target, { - jsonSchema: formatJsonSchema, - }) - - if (formatErrors.length) { - errors.push({ - type: "cell/type", - cell: String(row.source), - fieldName: field.name, - fieldType: field.type, - fieldFormat: field.format, - rowNumber: row.number, - }) - } - - continue - } - - if (constraintJsonSchema) { - const constraintErrors = await inspectJsonValue(target, { - jsonSchema: constraintJsonSchema, - }) - - for (const error of constraintErrors) { - errors.push({ - type: "cell/jsonSchema", - cell: String(row.source), - fieldName: field.name, - rowNumber: row.number, - pointer: error.pointer, - message: error.message, - }) - } - } - } - - return errors -} diff --git a/table/field/types/list.spec.ts b/table/field/types/list.spec.ts deleted file mode 100644 index 12dd51d8..00000000 --- a/table/field/types/list.spec.ts +++ /dev/null @@ -1,294 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseListField", () => { - describe("default settings (string items, comma delimiter)", () => { - it.each([ - // Basic list parsing - ["a,b,c", ["a", "b", "c"]], - ["1,2,3", ["1", "2", "3"]], - ["foo,bar,baz", ["foo", "bar", "baz"]], - - // Empty list - //["", null], - - // Single item - ["single", ["single"]], - - // Whitespace handling - //[" a, b, c ", ["a", "b", "c"]], - //["\ta,b,c\n", ["a", "b", "c"]], - - // Empty items in list - ["a,,c", ["a", "", "c"]], - [",b,", ["", "b", ""]], - [",,,", ["", "", "", ""]], - - // Null handling - //[null, null], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "list" as const }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("integer item type", () => { - it.each([ - // Valid integers - ["1,2,3", [1, 2, 3]], - ["0,-1,42", [0, -1, 42]], - ["-10,0,10", [-10, 0, 10]], - - // Empty list - //["", null], - - // Single item - ["42", [42]], - - // Whitespace handling - //[" 1, 2, 3 ", [1, 2, 3]], - //["\t-5,0,5\n", [-5, 0, 5]], - - // Empty items in list (become nulls when converted to integers) - ["1,,3", [1, null, 3]], - [",2,", [null, 2, null]], - - // Invalid integers become null - ["1,a,3", [1, null, 3]], - ["1.5,2,3", [null, 2, 3]], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "list" as const, itemType: "integer" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("number item type", () => { - it.each([ - // Valid numbers - ["1.5,2.1,3.7", [1.5, 2.1, 3.7]], - ["0,-1.1,42", [0, -1.1, 42]], - ["-10.5,0,10", [-10.5, 0, 10]], - - // Empty list - //["", null], - - // Single item - ["3.14", [3.14]], - - // Whitespace handling - //[" 1.1, 2.2, 3.3 ", [1.1, 2.2, 3.3]], - //["\t-5.5,0,5.5\n", [-5.5, 0, 5.5]], - - // Empty items in list (become nulls when converted to numbers) - ["1.1,,3.3", [1.1, null, 3.3]], - [",2.2,", [null, 2.2, null]], - - // Invalid numbers become null - ["1.1,a,3.3", [1.1, null, 3.3]], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "list" as const, itemType: "number" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("custom delimiter", () => { - it.each([ - // Semicolon delimiter - ["a;b;c", ["a", "b", "c"]], - ["1;2;3", ["1", "2", "3"]], - - // Empty list - //["", null], - - // Single item - ["single", ["single"]], - - // Whitespace handling - //[" a; b; c ", ["a", "b", "c"]], - - // Empty items in list - ["a;;c", ["a", "", "c"]], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "list" as const, delimiter: ";" }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) -}) - -describe("stringifyListField", () => { - describe("default settings (string items, comma delimiter)", () => { - it.each([ - // Basic list stringifying - [["a", "b", "c"], "a,b,c"], - [["foo", "bar", "baz"], "foo,bar,baz"], - [["1", "2", "3"], "1,2,3"], - - // Single item - [["single"], "single"], - - // Empty items in list - [["a", "", "c"], "a,,c"], - [["", "b", ""], ",b,"], - [["", "", "", ""], ",,,"], - - // Null handling - [[null, "b", null], "b"], - [["a", null, "c"], "a,c"], - - // Empty array - [[], ""], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.String))]) - .lazy() - - const schema = { - fields: [{ name: "name", type: "list" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) - - describe("integer item type", () => { - it.each([ - // Integer lists to string - [[1, 2, 3], "1,2,3"], - [[0, -1, 42], "0,-1,42"], - [[-10, 0, 10], "-10,0,10"], - - // Single item - [[42], "42"], - - // With nulls (nulls are filtered out) - [[1, null, 3], "1,3"], - [[null, 2, null], "2"], - - // Empty array - [[], ""], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.Int16))]) - .lazy() - - const schema = { - fields: [ - { name: "name", type: "list" as const, itemType: "integer" as const }, - ], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) - - describe("number item type", () => { - it.each([ - // Number lists to string - [[1.5, 2.1, 3.7], "1.5,2.1,3.7"], - [[0, -1.1, 42], "0.0,-1.1,42.0"], - [[-10.5, 0, 10], "-10.5,0.0,10.0"], - - // Single item - [[3.14], "3.14"], - - // With nulls - [[1.1, null, 3.3], "1.1,3.3"], - [[null, 2.2, null], "2.2"], - - // Empty array - [[], ""], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.Float64))]) - .lazy() - - const schema = { - fields: [ - { name: "name", type: "list" as const, itemType: "number" as const }, - ], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) - - describe("custom delimiter", () => { - it.each([ - // Semicolon delimiter - [["a", "b", "c"], "a;b;c"], - [["1", "2", "3"], "1;2;3"], - - // Single item - [["single"], "single"], - - // Empty items in list - [["a", "", "c"], "a;;c"], - [["", "b", ""], ";b;"], - - // Numeric items - [[1.0, 2.0, 3.0], "1.0;2.0;3.0"], - - // Empty array - [[], ""], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.String))]) - .lazy() - - const schema = { - fields: [{ name: "name", type: "list" as const, delimiter: ";" }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) - }) -}) diff --git a/table/field/types/list.ts b/table/field/types/list.ts deleted file mode 100644 index 389d75bc..00000000 --- a/table/field/types/list.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { ListField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -// TODO: -// Add more validation: -// - Return null instead of list if all array values are nulls? -export function parseListField(field: ListField, fieldExpr: pl.Expr) { - const delimiter = field.delimiter ?? "," - const itemType = field.itemType - - let dtype: any = pl.String - if (itemType === "integer") dtype = pl.Int64 - if (itemType === "number") dtype = pl.Float64 - if (itemType === "boolean") dtype = pl.Bool - if (itemType === "datetime") dtype = pl.Datetime - if (itemType === "date") dtype = pl.Date - if (itemType === "time") dtype = pl.Time - - fieldExpr = fieldExpr.str.split(delimiter).cast(pl.List(dtype)) - - return fieldExpr -} - -export function stringifyListField(field: ListField, fieldExpr: pl.Expr) { - const delimiter = field.delimiter ?? "," - - return fieldExpr - .cast(pl.List(pl.String)) - .lst.join({ separator: delimiter, ignoreNulls: true }) -} diff --git a/table/field/types/number.spec.ts b/table/field/types/number.spec.ts deleted file mode 100644 index 7d292cb9..00000000 --- a/table/field/types/number.spec.ts +++ /dev/null @@ -1,117 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseNumberField", () => { - it.each([ - // Basic number parsing - ["1", 1.0, {}], - ["2", 2.0, {}], - ["1000", 1000.0, {}], - ["1.5", 1.5, {}], - // biome-ignore lint/suspicious: tests - ["3.14159", 3.14159, {}], - ["-42", -42.0, {}], - ["-3.14", -3.14, {}], - - // Empty or invalid values - ["", null, {}], - ["bad", null, {}], - ["text", null, {}], - - // Group character handling - ["1", 1.0, { groupChar: "," }], - ["1,000", 1000.0, { groupChar: "," }], - ["1,000,000", 1000000.0, { groupChar: "," }], - ["1 000", 1000.0, { groupChar: " " }], - ["1#000#000", 1000000.0, { groupChar: "#" }], - - // Decimal character handling - ["1.5", 1.5, { decimalChar: "." }], - ["1,5", 1.5, { decimalChar: "," }], - ["3,14", 3.14, { decimalChar: "," }], - ["3.14", 3.14, { decimalChar: "." }], - - // Bare number handling - ["1.5", 1.5, { bareNumber: true }], - ["$1.5", null, { bareNumber: true }], - ["1.5%", null, { bareNumber: true }], - ["$1.5", 1.5, { bareNumber: false }], - ["1.5%", 1.5, { bareNumber: false }], - ["$1,000.00", null, { bareNumber: true }], - ["$1,000.00", 1000.0, { bareNumber: false, groupChar: "," }], - [ - "€ 1.000,00", - 1000.0, - { bareNumber: false, groupChar: ".", decimalChar: "," }, - ], - [ - "1.000,00 €", - 1000.0, - { bareNumber: false, groupChar: ".", decimalChar: "," }, - ], - - // Complex cases with multiple options - ["1,234.56", 1234.56, { groupChar: "," }], - ["1.234,56", 1234.56, { groupChar: ".", decimalChar: "," }], - ["$1,234.56", null, { bareNumber: true, groupChar: "," }], - ["$1,234.56", 1234.56, { bareNumber: false, groupChar: "," }], - ["1,234.56$", 1234.56, { bareNumber: false, groupChar: "," }], - [ - "1.234,56 €", - 1234.56, - { bareNumber: false, groupChar: ".", decimalChar: "," }, - ], - ])("$0 -> $1 $2", async (cell, value, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "number" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").get(0)).toEqual(value) - }) -}) - -describe("stringifyNumberField", () => { - it.each([ - // Basic number to string conversion - [1.0, "1.0"], - [2.0, "2.0"], - [1000.0, "1000.0"], - [3.14, "3.14"], - [42.5, "42.5"], - [-1.0, "-1.0"], - [-100.5, "-100.5"], - [0.0, "0.0"], - - // Numbers with many decimal places - //[3.141592653589793, "3.141592653589793"], - [-123.456789, "-123.456789"], - - // Large numbers - [1234567890.123, "1234567890.123"], - [-9876543210.987, "-9876543210.987"], - - // Small numbers - [0.001, "0.001"], - [-0.0001, "-0.0001"], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Float64)]).lazy() - - const schema = { - fields: [{ name: "name", type: "number" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/number.ts b/table/field/types/number.ts deleted file mode 100644 index 8fa709a7..00000000 --- a/table/field/types/number.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { NumberField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -export function parseNumberField(field: NumberField, fieldExpr: pl.Expr) { - // Extract the decimal and group characters - const decimalChar = field.decimalChar ?? "." - const groupChar = field.groupChar ?? "" - const bareNumber = field.bareNumber ?? true - - // Handle non-bare numbers (with currency symbols, percent signs, etc.) - if (bareNumber === false) { - // Remove leading non-digit characters (except minus sign and allowed decimal points) - const allowedDecimalChars = - decimalChar === "." ? "\\." : `\\.${decimalChar}` - fieldExpr = fieldExpr.str.replaceAll( - `^[^\\d\\-${allowedDecimalChars}]+`, - "", - ) - // Remove trailing non-digit characters - fieldExpr = fieldExpr.str.replaceAll(`[^\\d${allowedDecimalChars}]+$`, "") - } - - // Special case handling for European number format where "." is group and "," is decimal - if (groupChar === "." && decimalChar === ",") { - // First temporarily replace the decimal comma with a placeholder - fieldExpr = fieldExpr.str.replaceAll(",", "###DECIMAL###") - // Remove the group dots - fieldExpr = fieldExpr.str.replaceAll("\\.", "") - // Replace the placeholder with an actual decimal point - fieldExpr = fieldExpr.str.replaceAll("###DECIMAL###", ".") - } else { - // Standard case: first remove group characters - if (groupChar) { - // Escape special characters for regex - const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - fieldExpr = fieldExpr.str.replaceAll(escapedGroupChar, "") - } - - // Then handle decimal character - if (decimalChar && decimalChar !== ".") { - fieldExpr = fieldExpr.str.replaceAll(decimalChar, ".") - } - } - - // Cast to float64 - fieldExpr = fieldExpr.cast(pl.Float64) - return fieldExpr -} - -export function stringifyNumberField(_field: NumberField, fieldExpr: pl.Expr) { - // Convert to string - fieldExpr = fieldExpr.cast(pl.String) - - //const decimalChar = field.decimalChar ?? "." - //const groupChar = field.groupChar ?? "" - - // TODO: Add decimal character formatting when needed - // TODO: Add group character formatting (thousands separator) when needed - // TODO: Add non-bare number formatting (currency symbols, etc.) when needed - - return fieldExpr -} diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts deleted file mode 100644 index c314911e..00000000 --- a/table/field/types/object.spec.ts +++ /dev/null @@ -1,427 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("validateObjectField", () => { - it("should not errors for valid JSON objects", async () => { - const table = pl - .DataFrame({ - metadata: ['{"key":"value"}', '{"num":123}', '{"arr":[1,2,3]}'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "metadata", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for JSON arrays", async () => { - const table = pl - .DataFrame({ - data: ["[1,2,3]", '{"key":"value"}', '["a","b","c"]'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 1, - cell: "[1,2,3]", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 3, - cell: '["a","b","c"]', - }, - ]) - }) - - it("should not errors for null values", async () => { - const table = pl - .DataFrame({ - config: ['{"key":"value"}', null, '{"num":123}'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "config", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for invalid JSON", async () => { - const table = pl - .DataFrame({ - data: ['{"valid":true}', "invalid json", '{"key":"value"}', "{broken}"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 2, - cell: "invalid json", - }) - expect(errors).toContainEqual({ - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 4, - cell: "{broken}", - }) - }) - - it("should handle complex nested JSON structures", async () => { - const table = pl - .DataFrame({ - complex: [ - '{"user":{"name":"John","age":30,"tags":["admin","user"]}}', - '{"nested":{"deep":{"value":true}}}', - '{"array":[{"id":1},{"id":2}]}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "complex", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for empty strings", async () => { - const table = pl - .DataFrame({ - data: ['{"valid":true}', "", '{"key":"value"}'], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 2, - cell: "", - }, - ]) - }) - - it("should errors for JSON primitives", async () => { - const table = pl - .DataFrame({ - data: ['"string"', "123", "true", "false", "null"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "object", - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 1, - cell: '"string"', - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 2, - cell: "123", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 3, - cell: "true", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 4, - cell: "false", - }, - { - type: "cell/type", - fieldName: "data", - fieldType: "object", - rowNumber: 5, - cell: "null", - }, - ]) - }) - - it("should not errors for objects matching jsonSchema", async () => { - const table = pl - .DataFrame({ - user: [ - '{"name":"John","age":30}', - '{"name":"Jane","age":25}', - '{"name":"Bob","age":35}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "user", - type: "object", - constraints: { - jsonSchema: { - type: "object", - properties: { - name: { type: "string" }, - age: { type: "number" }, - }, - required: ["name", "age"], - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for objects not matching jsonSchema", async () => { - const jsonSchema = { - type: "object", - properties: { - name: { type: "string" }, - age: { type: "number" }, - }, - required: ["name", "age"], - } - - const table = pl - .DataFrame({ - user: [ - '{"name":"John","age":30}', - '{"name":"Jane"}', - '{"age":25}', - '{"name":"Bob","age":"invalid"}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "user", - type: "object", - constraints: { - jsonSchema, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/jsonSchema")).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "user", - rowNumber: 2, - cell: '{"name":"Jane"}', - pointer: "", - message: "must have required property 'age'", - }, - { - type: "cell/jsonSchema", - fieldName: "user", - rowNumber: 3, - cell: '{"age":25}', - pointer: "", - message: "must have required property 'name'", - }, - { - type: "cell/jsonSchema", - fieldName: "user", - rowNumber: 4, - cell: '{"name":"Bob","age":"invalid"}', - pointer: "/age", - message: "must be number", - }, - ]) - }) - - it("should validate complex jsonSchema with nested objects", async () => { - const table = pl - .DataFrame({ - config: [ - '{"database":{"host":"localhost","port":5432},"cache":{"enabled":true}}', - '{"database":{"host":"localhost","port":"invalid"},"cache":{"enabled":true}}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "config", - type: "object", - constraints: { - jsonSchema: { - type: "object", - properties: { - database: { - type: "object", - properties: { - host: { type: "string" }, - port: { type: "number" }, - }, - required: ["host", "port"], - }, - cache: { - type: "object", - properties: { - enabled: { type: "boolean" }, - }, - required: ["enabled"], - }, - }, - required: ["database", "cache"], - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "config", - rowNumber: 2, - cell: '{"database":{"host":"localhost","port":"invalid"},"cache":{"enabled":true}}', - pointer: "/database/port", - message: "must be number", - }, - ]) - }) - - it("should validate jsonSchema with array properties", async () => { - const table = pl - .DataFrame({ - data: [ - '{"items":[1,2,3],"name":"test"}', - '{"items":["not","numbers"],"name":"test"}', - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { - name: "data", - type: "object", - constraints: { - jsonSchema: { - type: "object", - properties: { - items: { - type: "array", - items: { type: "number" }, - }, - name: { type: "string" }, - }, - required: ["items", "name"], - }, - }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/jsonSchema")).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "data", - rowNumber: 2, - cell: '{"items":["not","numbers"],"name":"test"}', - pointer: "/items/0", - message: "must be number", - }, - { - type: "cell/jsonSchema", - fieldName: "data", - rowNumber: 2, - cell: '{"items":["not","numbers"],"name":"test"}', - pointer: "/items/1", - message: "must be number", - }, - ]) - }) -}) diff --git a/table/field/types/object.ts b/table/field/types/object.ts deleted file mode 100644 index 881c5e50..00000000 --- a/table/field/types/object.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { ObjectField } from "@dpkit/metadata" -import type { Table } from "../../table/index.ts" -import { inspectJsonField } from "./json.ts" - -export async function inspectObjectField(field: ObjectField, table: Table) { - return inspectJsonField(field, table) -} diff --git a/table/field/types/string.spec.ts b/table/field/types/string.spec.ts deleted file mode 100644 index 781ca46f..00000000 --- a/table/field/types/string.spec.ts +++ /dev/null @@ -1,195 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../../table/index.ts" - -// TODO: Implement proper tests -// TODO: Currently, it fails on to JS conversion from Polars -describe("parseStringField", () => { - it.each([ - // Simplr string - ["string", "string"], - - // Null handling - ["", null], - ])("$0 -> $1", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "string" as const }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").dtype).toEqual(pl.String) - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - - describe("email format", () => { - it.each([ - // Valid emails - ["user@example.com", "user@example.com"], - ["test.email@domain.co.uk", "test.email@domain.co.uk"], - ["user+tag@example.org", "user+tag@example.org"], - ["first.last@subdomain.example.com", "first.last@subdomain.example.com"], - ["user123@test-domain.com", "user123@test-domain.com"], - - // Invalid emails - ["invalid-email", null], - ["@example.com", null], - ["user@", null], - ["user@@example.com", null], - ["user@example", null], - ["user name@example.com", null], - - // Null handling - ["", null], - ])("$0 -> $1", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "string" as const, format: "email" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").dtype).toEqual(pl.String) - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("uri format", () => { - it.each([ - // Valid URIs - ["https://example.com", "https://example.com"], - [ - "http://www.google.com/search?q=test", - "http://www.google.com/search?q=test", - ], - ["ftp://files.example.org/file.txt", "ftp://files.example.org/file.txt"], - ["mailto:user@example.com", "mailto:user@example.com"], - ["file:///path/to/file", "file:///path/to/file"], - ["ssh://user@host:22/path", "ssh://user@host:22/path"], - - // Invalid URIs - ["not-a-uri", null], - ["://missing-scheme", null], - ["http://", null], - ["example.com", null], - ["http:// space in uri", null], - - // Null handling - ["", null], - ])("$0 -> $1", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "string" as const, format: "uri" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").dtype).toEqual(pl.String) - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("binary format", () => { - it.each([ - // Valid base64 strings - ["SGVsbG8gV29ybGQ=", "SGVsbG8gV29ybGQ="], - ["YWJjZGVmZw==", "YWJjZGVmZw=="], - ["VGVzdA==", "VGVzdA=="], - ["QQ==", "QQ=="], - ["Zg==", "Zg=="], - ["Zm8=", "Zm8="], - ["Zm9v", "Zm9v"], - - // Invalid base64 strings - ["Hello World!", null], - ["SGVsbG8gV29ybGQ===", null], - ["Invalid@#$", null], - ["SGVsb(8gV29ybGQ=", null], - - // Null handling - ["", null], - ])("$0 -> $1", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "string" as const, format: "binary" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").dtype).toEqual(pl.String) - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("uuid format", () => { - it.each([ - // Valid UUIDs - [ - "f47ac10b-58cc-4372-a567-0e02b2c3d479", - "f47ac10b-58cc-4372-a567-0e02b2c3d479", - ], - - // Invalid UUIDs - ["f47ac10b-58cc-4372-a567-0e02b2c3d47X", null], - ["f47ac10b", null], - ["X", null], - - // Null handling - ["", null], - ])("$0 -> $1 $2", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [ - { name: "name", type: "string" as const, format: "uuid" as const }, - ], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").dtype).toEqual(pl.Categorical) - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) - - describe("categories", () => { - it.each([ - // Flat categories - ["apple", "apple", { categories: ["apple", "banana"] }], - ["banana", "banana", { categories: ["apple", "banana"] }], - ["orange", null, { categories: ["apple", "banana"] }], - - // Object categories - ["apple", "apple", { categories: [{ value: "apple", label: "Apple" }] }], - ["orange", null, { categories: [{ value: "apple", label: "Apple" }] }], - ])("$0 -> $1 $2", async (cell, value, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "string" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").dtype).toEqual(pl.Categorical) - expect(frame.toRecords()[0]?.name).toEqual(value) - }) - }) -}) diff --git a/table/field/types/string.ts b/table/field/types/string.ts deleted file mode 100644 index 872a99e3..00000000 --- a/table/field/types/string.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { StringField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -const FORMAT_REGEX = { - email: - "^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$", - uri: "^[a-zA-Z][a-zA-Z0-9+.-]*:(//([^\\s/]+[^\\s]*|/[^\\s]*)|[^\\s/][^\\s]*)$", - binary: "^[A-Za-z0-9+/]*={0,2}$", - uuid: "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", -} as const - -// TODO: support categoriesOrder? -export function parseStringField(field: StringField, fieldExpr: pl.Expr) { - const format = field.format - const flattenCategories = field.categories?.map(it => - typeof it === "string" ? it : it.value, - ) - - if (flattenCategories) { - return pl - .when(fieldExpr.isIn(flattenCategories)) - .then(fieldExpr.cast(pl.Categorical)) - .otherwise(pl.lit(null)) - .alias(field.name) - } - - if (format) { - const regex = FORMAT_REGEX[format] - return pl - .when(fieldExpr.str.contains(regex)) - .then(fieldExpr) - .otherwise(pl.lit(null)) - .alias(field.name) - } - - return fieldExpr -} - -export function stringifyStringField(_field: StringField, fieldExpr: pl.Expr) { - return fieldExpr -} diff --git a/table/field/types/time.spec.ts b/table/field/types/time.spec.ts deleted file mode 100644 index 0822860c..00000000 --- a/table/field/types/time.spec.ts +++ /dev/null @@ -1,63 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../../table/index.ts" -import { denormalizeTable } from "../../table/index.ts" - -describe("parseTimeField", () => { - it.each([ - // Default format tests - ["06:00:00", "06:00:00", {}], - // #TODO: Clarify the behavior on the Standard level first - //["06:00:00Z", 6 * 60 * 60 * 10 ** 9, {}], - ["09:00", null, {}], // Incomplete time - ["3 am", null, {}], // Wrong format - ["3.00", null, {}], // Wrong format - ["invalid", null, {}], - ["", null, {}], - - // Custom format tests - ["06:00", "06:00:00", { format: "%H:%M" }], - ["06:50", null, { format: "%M:%H" }], // Invalid format - ["3:00 am", "03:00:00", { format: "%H:%M" }], - ["some night", null, { format: "%H:%M" }], - ["invalid", null, { format: "%H:%M" }], - ["", null, { format: "%H:%M" }], - - // Invalid format - //["06:00", null, { format: "invalid" }], - ])("$0 -> $1 $2", async (cell, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "time" as const, ...options }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) - -describe("stringifyTimeField", () => { - it.each([ - // Default format - [new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), "06:00:00", {}], - [new Date(Date.UTC(2014, 0, 1, 16, 30, 0)), "16:30:00", {}], - - // Custom format - [new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), "06:00", { format: "%H:%M" }], - [new Date(Date.UTC(2014, 0, 1, 16, 30, 0)), "16:30", { format: "%H:%M" }], - ])("%s -> %s %o", async (value, expected, options) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Time)]).lazy() - - const schema = { - fields: [{ name: "name", type: "time" as const, ...options }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/time.ts b/table/field/types/time.ts deleted file mode 100644 index be3a9cc1..00000000 --- a/table/field/types/time.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { TimeField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -const DEFAULT_FORMAT = "%H:%M:%S" - -export function parseTimeField(field: TimeField, fieldExpr: pl.Expr) { - let format = DEFAULT_FORMAT - if (field.format && field.format !== "default" && field.format !== "any") { - format = field.format - } - - return pl.pl - .concatString([pl.pl.lit("1970-01-01T"), fieldExpr], "") - .str.strptime(pl.Datetime, `%Y-%m-%dT${format}`) - .cast(pl.Time) - .alias(field.name) -} - -export function stringifyTimeField(field: TimeField, fieldExpr: pl.Expr) { - const format = field.format ?? DEFAULT_FORMAT - - return fieldExpr.date.strftime(format) -} diff --git a/table/field/types/year.spec.ts b/table/field/types/year.spec.ts deleted file mode 100644 index 428b0907..00000000 --- a/table/field/types/year.spec.ts +++ /dev/null @@ -1,59 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseYearField", () => { - it.each([ - // Tests ported from frictionless-py (string values only) - ["2000", 2000], - ["-2000", null], - ["20000", null], - ["3.14", null], - ["", null], - - // Additional tests for completeness - ["0000", 0], - ["9999", 9999], - //[" 2023 ", 2023], - //[" 1984 ", 1984], - ["bad", null], - ["12345", null], - ["123", null], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "year" as const }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.getColumn("name").get(0)).toEqual(value) - }) -}) - -describe("stringifyYearField", () => { - it.each([ - // Basic integer years to string conversion - [2000, "2000"], - [2023, "2023"], - [1999, "1999"], - [0, "0000"], - [9999, "9999"], - - // Edge cases with null values - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = pl.DataFrame([pl.Series("name", [value], pl.Int16)]).lazy() - - const schema = { - fields: [{ name: "name", type: "year" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/year.ts b/table/field/types/year.ts deleted file mode 100644 index 4c87046a..00000000 --- a/table/field/types/year.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { YearField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -export function parseYearField(_field: YearField, fieldExpr: pl.Expr) { - fieldExpr = pl - .when(fieldExpr.str.lengths().eq(4)) - .then(fieldExpr) - .otherwise(pl.lit(null)) - .cast(pl.Int16) - - return pl - .when(fieldExpr.gtEq(0).and(fieldExpr.ltEq(9999))) - .then(fieldExpr) - .otherwise(pl.lit(null)) -} - -export function stringifyYearField(_field: YearField, fieldExpr: pl.Expr) { - return fieldExpr.cast(pl.String).str.zFill(4) -} diff --git a/table/field/types/yearmonth.spec.ts b/table/field/types/yearmonth.spec.ts deleted file mode 100644 index 1bac9884..00000000 --- a/table/field/types/yearmonth.spec.ts +++ /dev/null @@ -1,43 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "../../table/index.ts" -import { denormalizeTable } from "../../table/index.ts" - -describe("parseYearmonthField", () => { - it.each([ - ["2000-01", [2000, 1]], - ["0-0", [0, 0]], - ])("%s -> %s", async (cell, value) => { - const table = pl.DataFrame([pl.Series("name", [cell], pl.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "yearmonth" as const }], - } - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(value) - }) -}) - -describe("stringifyYearmonthField", () => { - it.each([ - [[2000, 1], "2000-01"], - [[2023, 12], "2023-12"], - [[0, 0], "0000-00"], - ])("%s -> %s", async (value, expected) => { - const table = pl - .DataFrame([pl.Series("name", [value], pl.List(pl.Int16))]) - .lazy() - - const schema = { - fields: [{ name: "name", type: "yearmonth" as const }], - } - - const result = await denormalizeTable(table, schema) - const frame = await result.collect() - - expect(frame.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/yearmonth.ts b/table/field/types/yearmonth.ts deleted file mode 100644 index 18819f80..00000000 --- a/table/field/types/yearmonth.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { YearmonthField } from "@dpkit/metadata" -import * as pl from "nodejs-polars" - -export function parseYearmonthField( - _field: YearmonthField, - fieldExpr: pl.Expr, -) { - fieldExpr = fieldExpr.str.split("-").cast(pl.List(pl.Int16)) - - return fieldExpr -} - -export function stringifyYearmonthField( - field: YearmonthField, - fieldExpr: pl.Expr, -) { - return pl - .concatString( - [ - fieldExpr.lst.get(0).cast(pl.String).str.zFill(4), - fieldExpr.lst.get(1).cast(pl.String).str.zFill(2), - ], - "-", - ) - .alias(field.name) as pl.Expr -} diff --git a/table/helpers.ts b/table/helpers.ts deleted file mode 100644 index 7bd24bd6..00000000 --- a/table/helpers.ts +++ /dev/null @@ -1,14 +0,0 @@ -import * as pl from "nodejs-polars" - -export function isObject(value: any): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -export function arrayDiff(a: string[], b: string[]) { - return a.filter(x => !b.includes(x)) -} - -export function evaluateExpression(expr: pl.Expr) { - // @ts-ignore - return pl.select(expr.alias("value")).toRecords()[0].value -} diff --git a/table/index.ts b/table/index.ts deleted file mode 100644 index 4e083d78..00000000 --- a/table/index.ts +++ /dev/null @@ -1,42 +0,0 @@ -export type { DataRecord } from "./data/index.ts" -export type { DataRow } from "./data/index.ts" -export type { DenormalizeFieldOptions } from "./field/index.ts" -export type { DialectOptions } from "./dialect/index.ts" -export type { Frame } from "./table/index.ts" -export type { InferDialectOptions } from "./dialect/index.ts" -export type { InferSchemaOptions } from "./schema/index.ts" -export type { LoadTableOptions } from "./plugin.ts" -export type { PolarsField } from "./field/index.ts" -export type { PolarsSchema } from "./schema/index.ts" -export type { SaveTableOptions } from "./plugin.ts" -export type { SchemaOptions } from "./schema/index.ts" -export type { Table } from "./table/index.ts" -export type { TablePlugin } from "./plugin.ts" - -export { ArrowPlugin } from "./plugins/arrow/index.ts" -export { CsvPlugin } from "./plugins/csv/index.ts" -export { InlinePlugin } from "./plugins/inline/index.ts" -export { JsonPlugin } from "./plugins/json/index.ts" -export { OdsPlugin } from "./plugins/ods/index.ts" -export { ParquetPlugin } from "./plugins/parquet/index.ts" -export { XlsxPlugin } from "./plugins/xlxs/index.ts" - -export { denormalizeTable } from "./table/index.ts" -export { inferSchemaFromSample } from "./schema/index.ts" -export { inferSchemaFromTable } from "./schema/index.ts" -export { inspectTable } from "./table/index.ts" -export { loadArrowTable } from "./plugins/arrow/index.ts" -export { loadCsvTable } from "./plugins/csv/index.ts" -export { loadInlineTable } from "./plugins/inline/index.ts" -export { loadJsonTable } from "./plugins/json/index.ts" -export { loadOdsTable } from "./plugins/ods/index.ts" -export { loadParquetTable } from "./plugins/parquet/index.ts" -export { loadXlsxTable } from "./plugins/xlxs/index.ts" -export { normalizeTable } from "./table/index.ts" -export { queryTable } from "./table/index.ts" -export { saveArrowTable } from "./plugins/arrow/index.ts" -export { saveCsvTable } from "./plugins/csv/index.ts" -export { saveJsonTable } from "./plugins/json/index.ts" -export { saveOdsTable } from "./plugins/ods/index.ts" -export { saveParquetTable } from "./plugins/parquet/index.ts" -export { saveXlsxTable } from "./plugins/xlxs/index.ts" diff --git a/table/package.json b/table/package.json deleted file mode 100644 index 04f2bc07..00000000 --- a/table/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@dpkit/table", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "table", - "arrow", - "csv", - "inline", - "json", - "ods", - "parquet", - "xlsx" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "@dpkit/dataset": "workspace:*", - "csv-sniffer": "^0.1.1", - "nodejs-polars": "^0.22.2", - "p-all": "^5.0.1", - "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" - } -} diff --git a/table/plugin.ts b/table/plugin.ts deleted file mode 100644 index 4aa56e1b..00000000 --- a/table/plugin.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { SavePackageOptions } from "@dpkit/dataset" -import type { DatasetPlugin } from "@dpkit/dataset" -import type { Dialect, Package, Resource, Schema } from "@dpkit/metadata" -import type { DialectOptions, InferDialectOptions } from "./dialect/index.ts" -import type { InferSchemaOptions, SchemaOptions } from "./schema/index.ts" -import type { Table } from "./table/index.ts" - -export type LoadTableOptions = InferDialectOptions & - InferSchemaOptions & { - denormalized?: boolean - } - -export type SaveTableOptions = DialectOptions & - SchemaOptions & { - path: string - format?: string - dialect?: Dialect - schema?: Schema - overwrite?: boolean - } - -export interface TablePlugin extends DatasetPlugin { - savePackage?( - dataPackage: Package, - options: SavePackageOptions & { plugins?: TablePlugin[] }, - ): Promise<{ path?: string } | undefined> - - inferDialect?( - resource: Partial, - options?: InferDialectOptions, - ): Promise - - inferSchema?( - resource: Partial, - options?: InferSchemaOptions, - ): Promise - - loadTable?( - resource: Partial, - options?: LoadTableOptions, - ): Promise - - saveTable?( - table: Table, - options: SaveTableOptions, - ): Promise -} diff --git a/table/plugins/arrow/index.ts b/table/plugins/arrow/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/arrow/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/arrow/plugin.spec.ts b/table/plugins/arrow/plugin.spec.ts deleted file mode 100644 index 3babaeee..00000000 --- a/table/plugins/arrow/plugin.spec.ts +++ /dev/null @@ -1,212 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { ArrowPlugin } from "./plugin.ts" -import * as tableModule from "./table/index.ts" - -vi.mock("./table/index.ts", () => ({ - loadArrowTable: vi.fn(), - saveArrowTable: vi.fn(), -})) - -describe("ArrowPlugin", () => { - let plugin: ArrowPlugin - let mockLoadArrowTable: ReturnType - let mockSaveArrowTable: ReturnType - - beforeEach(() => { - plugin = new ArrowPlugin() - mockLoadArrowTable = vi.mocked(tableModule.loadArrowTable) - mockSaveArrowTable = vi.mocked(tableModule.saveArrowTable) - vi.clearAllMocks() - }) - - describe("loadTable", () => { - it("should load table from arrow file", async () => { - const resource: Partial = { - path: "test.arrow", - } - const mockTable = pl.DataFrame().lazy() - mockLoadArrowTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should load table from feather file", async () => { - const resource: Partial = { - path: "test.feather", - } - const mockTable = pl.DataFrame().lazy() - mockLoadArrowTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-arrow files", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadArrowTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit arrow format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "arrow", - } - const mockTable = pl.DataFrame().lazy() - mockLoadArrowTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should handle explicit feather format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "feather", - } - const mockTable = pl.DataFrame().lazy() - mockLoadArrowTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should pass through load options", async () => { - const resource: Partial = { - path: "test.arrow", - } - const options = { denormalized: true } - const mockTable = pl.DataFrame().lazy() - mockLoadArrowTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource, options) - - expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, options) - }) - - it("should handle paths with directories", async () => { - const resource: Partial = { - path: "/path/to/data.arrow", - } - const mockTable = pl.DataFrame().lazy() - mockLoadArrowTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource) - - expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined) - }) - - it("should return undefined for parquet files", async () => { - const resource: Partial = { - path: "test.parquet", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadArrowTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("saveTable", () => { - it("should save table to arrow file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.arrow" } - mockSaveArrowTable.mockResolvedValue("output.arrow") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.arrow") - }) - - it("should save table to feather file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.feather" } - mockSaveArrowTable.mockResolvedValue("output.feather") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.feather") - }) - - it("should return undefined for non-arrow files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit arrow format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "arrow" as const } - mockSaveArrowTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.txt") - }) - - it("should handle explicit feather format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "feather" as const } - mockSaveArrowTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.txt") - }) - - it("should handle paths with directories", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "/path/to/output.arrow" } - mockSaveArrowTable.mockResolvedValue("/path/to/output.arrow") - - await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options) - }) - - it("should return undefined for files without extension", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for parquet files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.parquet" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveArrowTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/table/plugins/arrow/plugin.ts b/table/plugins/arrow/plugin.ts deleted file mode 100644 index e1a9f01f..00000000 --- a/table/plugins/arrow/plugin.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { - LoadTableOptions, - SaveTableOptions, - TablePlugin, -} from "../../plugin.ts" -import type { Table } from "../../table/index.ts" -import { loadArrowTable, saveArrowTable } from "./table/index.ts" - -export class ArrowPlugin implements TablePlugin { - async loadTable(resource: Partial, options?: LoadTableOptions) { - const isArrow = getIsArrow(resource) - if (!isArrow) return undefined - - return await loadArrowTable(resource, options) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const isArrow = getIsArrow({ path, format }) - if (!isArrow) return undefined - - return await saveArrowTable(table, options) - } -} - -function getIsArrow(resource: Partial) { - const format = inferFormat(resource) - return format === "arrow" || format === "feather" -} diff --git a/table/plugins/arrow/table/fixtures/generated/loadArrowTable-file-variations-should-load-remote-file-multipart_3177840048/recording.har b/table/plugins/arrow/table/fixtures/generated/loadArrowTable-file-variations-should-load-remote-file-multipart_3177840048/recording.har deleted file mode 100644 index af5f0656..00000000 --- a/table/plugins/arrow/table/fixtures/generated/loadArrowTable-file-variations-should-load-remote-file-multipart_3177840048/recording.har +++ /dev/null @@ -1,292 +0,0 @@ -{ - "log": { - "_recordingName": "loadArrowTable-file variations-should load remote file (multipart)", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "53e95bba88b50cbcd890fa128c23cf63", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 116, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow" - }, - "response": { - "bodySize": 733, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 733, - "text": "QVJST1cxAAD/////oAAAAAQAAADy////FAAAAAQAAQAAAAoACwAIAAoABAD4////DAAAAAgACAAAAAQAAgAAADQAAAAEAAAAwP///xwAAAAQAAAACAAAAAEYAAAAAAAA/P///wQABAAEAAAAbmFtZQAAAADs////MAAAACAAAAAYAAAAAQMAABAAEgAEABAAEQAIAAAADAAAAAAA+v///wIABgAGAAQAAgAAAGlkAAD/////yAAAAAQAAADs////gAAAAAAAAAAUAAAABAADAAwAEwAQABIADAAEAOb///8CAAAAAAAAAHQAAAAoAAAAFAAAAAAADgAYAAQADAAQAAAAFAABAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAA8D8AAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAAABlbmdsaXNoAAAAAAAGAAAA5Lit5paHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////AAAAAAQAAADs////QAAAADgAAAAUAAAABAAAAAwAEgAQAAQACAAMAAEAAACwAAAAAAAAANAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAPj///8MAAAACAAIAAAABAACAAAANAAAAAQAAADA////HAAAABAAAAAIAAAAARgAAAAAAAD8////BAAEAAQAAABuYW1lAAAAAOz///8wAAAAIAAAABgAAAABAwAAEAASAAQAEAARAAgAAAAMAAAAAAD6////AgAGAAYABAACAAAAaWQAywAAAEFSUk9XMQ==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "733" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:19 GMT" - }, - { - "name": "etag", - "value": "W/\"9e81aff5aa78977ba8717ed086cda7e0297023e1956b05cc91d5ed2cc62c9078\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:19 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "d25e14f7a38be7d0c3ceedf7bf1fb70507509a16" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "9BC7:2B2025:7E2476:96CC8C:68EA3DE6" - }, - { - "name": "x-served-by", - "value": "cache-lis1490049-LIS" - }, - { - "name": "x-timer", - "value": "S1760181739.056625,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 874, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:19.108Z", - "time": 41, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 41 - } - }, - { - "_id": "53e95bba88b50cbcd890fa128c23cf63", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 116, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow" - }, - "response": { - "bodySize": 733, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 733, - "text": "QVJST1cxAAD/////oAAAAAQAAADy////FAAAAAQAAQAAAAoACwAIAAoABAD4////DAAAAAgACAAAAAQAAgAAADQAAAAEAAAAwP///xwAAAAQAAAACAAAAAEYAAAAAAAA/P///wQABAAEAAAAbmFtZQAAAADs////MAAAACAAAAAYAAAAAQMAABAAEgAEABAAEQAIAAAADAAAAAAA+v///wIABgAGAAQAAgAAAGlkAAD/////yAAAAAQAAADs////gAAAAAAAAAAUAAAABAADAAwAEwAQABIADAAEAOb///8CAAAAAAAAAHQAAAAoAAAAFAAAAAAADgAYAAQADAAQAAAAFAABAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAA8D8AAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAAABlbmdsaXNoAAAAAAAGAAAA5Lit5paHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////AAAAAAQAAADs////QAAAADgAAAAUAAAABAAAAAwAEgAQAAQACAAMAAEAAACwAAAAAAAAANAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAPj///8MAAAACAAIAAAABAACAAAANAAAAAQAAADA////HAAAABAAAAAIAAAAARgAAAAAAAD8////BAAEAAQAAABuYW1lAAAAAOz///8wAAAAIAAAABgAAAABAwAAEAASAAQAEAARAAgAAAAMAAAAAAD6////AgAGAAYABAACAAAAaWQAywAAAEFSUk9XMQ==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "733" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:19 GMT" - }, - { - "name": "etag", - "value": "W/\"9e81aff5aa78977ba8717ed086cda7e0297023e1956b05cc91d5ed2cc62c9078\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:19 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "e521331f3f693a35968fd36710f871075a856195" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "9BC7:2B2025:7E2476:96CC8C:68EA3DE6" - }, - { - "name": "x-served-by", - "value": "cache-lis1490030-LIS" - }, - { - "name": "x-timer", - "value": "S1760181739.144874,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 874, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:19.108Z", - "time": 129, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 129 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/arrow/table/fixtures/generated/loadArrowTable-file-variations-should-load-remote-file_1224085963/recording.har b/table/plugins/arrow/table/fixtures/generated/loadArrowTable-file-variations-should-load-remote-file_1224085963/recording.har deleted file mode 100644 index 155a0422..00000000 --- a/table/plugins/arrow/table/fixtures/generated/loadArrowTable-file-variations-should-load-remote-file_1224085963/recording.har +++ /dev/null @@ -1,153 +0,0 @@ -{ - "log": { - "_recordingName": "loadArrowTable-file variations-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "53e95bba88b50cbcd890fa128c23cf63", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 116, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow" - }, - "response": { - "bodySize": 733, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 733, - "text": "QVJST1cxAAD/////oAAAAAQAAADy////FAAAAAQAAQAAAAoACwAIAAoABAD4////DAAAAAgACAAAAAQAAgAAADQAAAAEAAAAwP///xwAAAAQAAAACAAAAAEYAAAAAAAA/P///wQABAAEAAAAbmFtZQAAAADs////MAAAACAAAAAYAAAAAQMAABAAEgAEABAAEQAIAAAADAAAAAAA+v///wIABgAGAAQAAgAAAGlkAAD/////yAAAAAQAAADs////gAAAAAAAAAAUAAAABAADAAwAEwAQABIADAAEAOb///8CAAAAAAAAAHQAAAAoAAAAFAAAAAAADgAYAAQADAAQAAAAFAABAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAA8D8AAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAAABlbmdsaXNoAAAAAAAGAAAA5Lit5paHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////AAAAAAQAAADs////QAAAADgAAAAUAAAABAAAAAwAEgAQAAQACAAMAAEAAACwAAAAAAAAANAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAPj///8MAAAACAAIAAAABAACAAAANAAAAAQAAADA////HAAAABAAAAAIAAAAARgAAAAAAAD8////BAAEAAQAAABuYW1lAAAAAOz///8wAAAAIAAAABgAAAABAwAAEAASAAQAEAARAAgAAAAMAAAAAAD6////AgAGAAYABAACAAAAaWQAywAAAEFSUk9XMQ==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "733" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:18 GMT" - }, - { - "name": "etag", - "value": "W/\"9e81aff5aa78977ba8717ed086cda7e0297023e1956b05cc91d5ed2cc62c9078\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:18 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "65703cd4a046c7e37a4a09d7110cd7f0d478b976" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "9BC7:2B2025:7E2476:96CC8C:68EA3DE6" - }, - { - "name": "x-served-by", - "value": "cache-lis1490049-LIS" - }, - { - "name": "x-timer", - "value": "S1760181739.665136,VS0,VE186" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 877, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:18.565Z", - "time": 488, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 488 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/arrow/table/index.ts b/table/plugins/arrow/table/index.ts deleted file mode 100644 index a65e5663..00000000 --- a/table/plugins/arrow/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadArrowTable } from "./load.ts" -export { saveArrowTable } from "./save.ts" diff --git a/table/plugins/arrow/table/load.spec.ts b/table/plugins/arrow/table/load.spec.ts deleted file mode 100644 index a20c2b83..00000000 --- a/table/plugins/arrow/table/load.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadArrowTable } from "./load.ts" - -useRecording() - -describe("loadArrowTable", () => { - describe("file variations", () => { - it("should load local file", async () => { - const path = getTempFilePath() - pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path) - - const table = await loadArrowTable({ path }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load local file (multipart)", async () => { - const path1 = getTempFilePath() - const path2 = getTempFilePath() - pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path1) - pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path2) - - const table = await loadArrowTable({ path: [path1, path2] }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file", async () => { - const table = await loadArrowTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file (multipart)", async () => { - const table = await loadArrowTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) -}) diff --git a/table/plugins/arrow/table/load.ts b/table/plugins/arrow/table/load.ts deleted file mode 100644 index 043b0dd9..00000000 --- a/table/plugins/arrow/table/load.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { prefetchFiles } from "@dpkit/dataset" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import * as pl from "nodejs-polars" - -export async function loadArrowTable( - resource: Partial, - options?: LoadTableOptions, -) { - const [firstPath, ...restPaths] = await prefetchFiles(resource.path) - if (!firstPath) { - throw new Error("Resource path is not defined") - } - - let table = pl.scanIPC(firstPath) - if (restPaths.length) { - table = pl.concat([table, ...restPaths.map(path => pl.scanIPC(path))]) - } - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} diff --git a/table/plugins/arrow/table/save.spec.ts b/table/plugins/arrow/table/save.spec.ts deleted file mode 100644 index 92afa9c3..00000000 --- a/table/plugins/arrow/table/save.spec.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { loadArrowTable } from "./load.ts" -import { saveArrowTable } from "./save.ts" - -describe("saveArrowTable", () => { - it("should save table to Arrow file", async () => { - const path = getTempFilePath() - const source = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice", "Bob", "Charlie"], - }) - .lazy() - - await saveArrowTable(source, { path }) - - const table = await loadArrowTable({ path }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1.0, name: "Alice" }, - { id: 2.0, name: "Bob" }, - { id: 3.0, name: "Charlie" }, - ]) - }) - - it("should save and load various data types", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveArrowTable(source, { - path, - fieldTypes: { - array: "array", - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadArrowTable({ path }, { denormalized: true }) - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: true, - date: "2025-01-01", - datetime: new Date(Date.UTC(2025, 0, 1)), - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: [1.0, 2.0, 3.0], - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) -}) diff --git a/table/plugins/arrow/table/save.ts b/table/plugins/arrow/table/save.ts deleted file mode 100644 index 7c80bf24..00000000 --- a/table/plugins/arrow/table/save.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { assertLocalPathVacant } from "@dpkit/dataset" -import type { SaveTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { denormalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" - -// TODO: rebase on sinkIPC when it is available -// https://github.com/pola-rs/nodejs-polars/issues/353 - -export async function saveArrowTable(table: Table, options: SaveTableOptions) { - const { path, overwrite } = options - - if (!overwrite) { - await assertLocalPathVacant(path) - } - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - table = await denormalizeTable(table, schema, { - nativeTypes: [ - "boolean", - "datetime", - "integer", - "list", - "number", - "string", - "year", - ], - }) - - const frame = await table.collect() - frame.writeIPC(path) - - return path -} diff --git a/table/plugins/csv/csv-sniffer.d.ts b/table/plugins/csv/csv-sniffer.d.ts deleted file mode 100644 index d9eb649a..00000000 --- a/table/plugins/csv/csv-sniffer.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module "csv-sniffer" diff --git a/table/plugins/csv/dialect/index.ts b/table/plugins/csv/dialect/index.ts deleted file mode 100644 index 1b388ce8..00000000 --- a/table/plugins/csv/dialect/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { inferCsvDialect } from "./infer.ts" diff --git a/table/plugins/csv/dialect/infer.spec.ts b/table/plugins/csv/dialect/infer.spec.ts deleted file mode 100644 index 0b832121..00000000 --- a/table/plugins/csv/dialect/infer.spec.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { writeTempFile } from "@dpkit/dataset" -import { describe, expect, it } from "vitest" -import { inferCsvDialect } from "./infer.ts" - -describe("inferCsvDialect", () => { - it("should infer a simple CSV file", async () => { - const path = await writeTempFile("id,name\n1,english\n2,中文") - const dialect = await inferCsvDialect({ path }) - - expect(dialect).toEqual({ - delimiter: ",", - }) - }) - - it("should infer quoteChar", async () => { - const path = await writeTempFile('id,name\n1,"John Doe"\n2,"Jane Smith"') - const dialect = await inferCsvDialect({ path }) - - expect(dialect).toEqual({ - delimiter: ",", - quoteChar: '"', - }) - }) - - it("should infer quoteChar with single quotes", async () => { - const path = await writeTempFile("id,name\n1,'John Doe'\n2,'Jane Smith'") - const dialect = await inferCsvDialect({ path }) - - expect(dialect).toEqual({ - delimiter: ",", - quoteChar: "'", - }) - }) - - // TODO: it gives false positives - it.skip("should infer header false when no header present", async () => { - const path = await writeTempFile("1,english\n2,中文\n3,español") - const dialect = await inferCsvDialect({ path }) - - expect(dialect).toEqual({ - delimiter: ",", - header: false, - }) - }) - - it("should not set header when header is present", async () => { - const path = await writeTempFile("id,name\n1,english\n2,中文") - const dialect = await inferCsvDialect({ path }) - - expect(dialect).toEqual({ - delimiter: ",", - }) - }) - - // TODO: recover if possible with csv-sniffer - it.skip("should infer complex CSV with quotes and header", async () => { - const path = await writeTempFile( - 'name,description\n"Product A","A great product with, commas"\n"Product B","Another product"', - ) - - const dialect = await inferCsvDialect({ path }) - expect(dialect).toEqual({ - delimiter: ",", - quoteChar: '"', - }) - }) -}) diff --git a/table/plugins/csv/dialect/infer.ts b/table/plugins/csv/dialect/infer.ts deleted file mode 100644 index cb961484..00000000 --- a/table/plugins/csv/dialect/infer.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { text } from "node:stream/consumers" -import { loadFileStream } from "@dpkit/dataset" -import type { Dialect, Resource } from "@dpkit/metadata" -import { default as CsvSnifferFactory } from "csv-sniffer" - -const CSV_DELIMITERS = [",", ";", ":", "|", "\t", "^", "*", "&"] -const TSV_DELIMITERS = ["\t"] - -export async function inferCsvDialect( - resource: Partial, - options?: { - sampleBytes?: number - }, -) { - const { sampleBytes = 10_000 } = options ?? {} - const isTabs = resource.format === "tsv" - - const dialect: Dialect = {} - - if (resource.path) { - const stream = await loadFileStream(resource.path, { - maxBytes: sampleBytes, - }) - - const sample = await text(stream) - const result = sniffSample(sample, isTabs ? TSV_DELIMITERS : CSV_DELIMITERS) - - if (result?.delimiter) { - dialect.delimiter = result.delimiter - } - - if (result?.quoteChar) { - dialect.quoteChar = result.quoteChar - } - - //if (result.lineTerminator) { - // dialect.lineTerminator = result.lineTerminator - //} - - // TODO: it gives false positives - //if (!result.hasHeader) { - // dialect.header = false - //} - } - - return dialect -} - -// Sniffer can fail for some reasons -function sniffSample(sample: string, delimiters: string[]) { - try { - const CsvSniffer = CsvSnifferFactory() - const sniffer = new CsvSniffer(delimiters) - const result = sniffer.sniff(sample) - return result - } catch { - return undefined - } -} diff --git a/table/plugins/csv/index.ts b/table/plugins/csv/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/csv/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/csv/plugin.spec.ts b/table/plugins/csv/plugin.spec.ts deleted file mode 100644 index c2a283c7..00000000 --- a/table/plugins/csv/plugin.spec.ts +++ /dev/null @@ -1,224 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { CsvPlugin } from "./plugin.ts" -import * as tableModule from "./table/index.ts" - -vi.mock("./table/index.ts", () => ({ - loadCsvTable: vi.fn(), - saveCsvTable: vi.fn(), -})) - -describe("CsvPlugin", () => { - let plugin: CsvPlugin - let mockLoadCsvTable: ReturnType - let mockSaveCsvTable: ReturnType - - beforeEach(() => { - plugin = new CsvPlugin() - mockLoadCsvTable = vi.mocked(tableModule.loadCsvTable) - mockSaveCsvTable = vi.mocked(tableModule.saveCsvTable) - vi.clearAllMocks() - }) - - describe("loadTable", () => { - it("should load table from csv file", async () => { - const resource: Partial = { - path: "test.csv", - } - const mockTable = pl.DataFrame().lazy() - mockLoadCsvTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadCsvTable).toHaveBeenCalledWith( - { ...resource, format: "csv" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should load table from tsv file", async () => { - const resource: Partial = { - path: "test.tsv", - } - const mockTable = pl.DataFrame().lazy() - mockLoadCsvTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadCsvTable).toHaveBeenCalledWith( - { ...resource, format: "tsv" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-csv files", async () => { - const resource: Partial = { - path: "test.json", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadCsvTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "csv", - } - const mockTable = pl.DataFrame().lazy() - mockLoadCsvTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadCsvTable).toHaveBeenCalledWith( - { ...resource, format: "csv" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should pass through load options", async () => { - const resource: Partial = { - path: "test.csv", - } - const options = { denormalized: true } - const mockTable = pl.DataFrame().lazy() - mockLoadCsvTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource, options) - - expect(mockLoadCsvTable).toHaveBeenCalledWith( - { ...resource, format: "csv" }, - options, - ) - }) - - it("should handle paths with directories", async () => { - const resource: Partial = { - path: "/path/to/data.csv", - } - const mockTable = pl.DataFrame().lazy() - mockLoadCsvTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource) - - expect(mockLoadCsvTable).toHaveBeenCalledWith( - { ...resource, format: "csv" }, - undefined, - ) - }) - - it("should handle explicit tsv format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "tsv", - } - const mockTable = pl.DataFrame().lazy() - mockLoadCsvTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadCsvTable).toHaveBeenCalledWith( - { ...resource, format: "tsv" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - }) - - describe("saveTable", () => { - it("should save table to csv file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - mockSaveCsvTable.mockResolvedValue("output.csv") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).toHaveBeenCalledWith(table, { - ...options, - format: "csv", - }) - expect(result).toBe("output.csv") - }) - - it("should save table to tsv file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.tsv" } - mockSaveCsvTable.mockResolvedValue("output.tsv") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).toHaveBeenCalledWith(table, { - ...options, - format: "tsv", - }) - expect(result).toBe("output.tsv") - }) - - it("should return undefined for non-csv files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.json" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "csv" as const } - mockSaveCsvTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).toHaveBeenCalledWith(table, { - ...options, - format: "csv", - }) - expect(result).toBe("output.txt") - }) - - it("should handle paths with directories", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "/path/to/output.csv" } - mockSaveCsvTable.mockResolvedValue("/path/to/output.csv") - - await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).toHaveBeenCalledWith(table, { - ...options, - format: "csv", - }) - }) - - it("should return undefined for files without extension", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit tsv format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "tsv" as const } - mockSaveCsvTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveCsvTable).toHaveBeenCalledWith(table, { - ...options, - format: "tsv", - }) - expect(result).toBe("output.txt") - }) - }) -}) diff --git a/table/plugins/csv/plugin.ts b/table/plugins/csv/plugin.ts deleted file mode 100644 index efce7541..00000000 --- a/table/plugins/csv/plugin.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { LoadTableOptions, SaveTableOptions } from "../../plugin.ts" -import type { TablePlugin } from "../../plugin.ts" -import type { Table } from "../../table/index.ts" -import { loadCsvTable, saveCsvTable } from "./table/index.ts" - -export class CsvPlugin implements TablePlugin { - async loadTable(resource: Partial, options?: LoadTableOptions) { - const csvFormat = getCsvFormat(resource) - if (!csvFormat) return undefined - - return await loadCsvTable({ ...resource, format: csvFormat }, options) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const csvFormat = getCsvFormat({ path, format }) - if (!csvFormat) return undefined - - return await saveCsvTable(table, { ...options, format: csvFormat }) - } -} - -function getCsvFormat(resource: Partial) { - const format = inferFormat(resource) - return format === "csv" || format === "tsv" ? format : undefined -} diff --git a/table/plugins/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file-multipart_959749322/recording.har b/table/plugins/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file-multipart_959749322/recording.har deleted file mode 100644 index d0fc15be..00000000 --- a/table/plugins/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file-multipart_959749322/recording.har +++ /dev/null @@ -1,274 +0,0 @@ -{ - "log": { - "_recordingName": "loadCsvTable-should load remote file (multipart)", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "591cb55d2cbf9fec4de80b7ceaeb928f", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 112, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/csv/table/fixtures/table.csv" - }, - "response": { - "bodySize": 14, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 14, - "text": "404: Not Found" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "14" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 10:44:34 GMT" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 10:49:34 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "ac817ac54911f85e8bbe755e37acd6ff2e92743d" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "E0CD:1634AC:7752B4:8F0EB2:68EA3511" - }, - { - "name": "x-served-by", - "value": "cache-lis1490046-LIS" - }, - { - "name": "x-timer", - "value": "S1760179475.607128,VS0,VE145" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 772, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 404, - "statusText": "Not Found" - }, - "startedDateTime": "2025-10-11T10:44:34.474Z", - "time": 312, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 312 - } - }, - { - "_id": "591cb55d2cbf9fec4de80b7ceaeb928f", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 112, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/csv/table/fixtures/table.csv" - }, - "response": { - "bodySize": 14, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 14, - "text": "404: Not Found" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "14" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 10:44:34 GMT" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 10:49:34 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "6d1a4cbe4c2c56f5db5bbcbbde2b0a3221d109bf" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "E0CD:1634AC:7752B4:8F0EB2:68EA3511" - }, - { - "name": "x-served-by", - "value": "cache-lis1490025-LIS" - }, - { - "name": "x-timer", - "value": "S1760179475.598556,VS0,VE153" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 773, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 404, - "statusText": "Not Found" - }, - "startedDateTime": "2025-10-11T10:44:34.474Z", - "time": 321, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 321 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file_2012170585/recording.har b/table/plugins/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file_2012170585/recording.har deleted file mode 100644 index 05925d70..00000000 --- a/table/plugins/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file_2012170585/recording.har +++ /dev/null @@ -1,144 +0,0 @@ -{ - "log": { - "_recordingName": "loadCsvTable-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "591cb55d2cbf9fec4de80b7ceaeb928f", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 112, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/csv/table/fixtures/table.csv" - }, - "response": { - "bodySize": 14, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 14, - "text": "404: Not Found" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "14" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 10:43:30 GMT" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 10:48:30 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "b76e0dfc90ce2b4256fb50d3a0e40f67daa46df6" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "24A5:1CE312:76F8D5:8EA967:68EA34D1" - }, - { - "name": "x-served-by", - "value": "cache-lis1490057-LIS" - }, - { - "name": "x-timer", - "value": "S1760179411.757578,VS0,VE162" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 773, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 404, - "statusText": "Not Found" - }, - "startedDateTime": "2025-10-11T10:43:30.613Z", - "time": 335, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 335 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/csv/table/fixtures/table.csv b/table/plugins/csv/table/fixtures/table.csv deleted file mode 100644 index 84b1ed50..00000000 --- a/table/plugins/csv/table/fixtures/table.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name -1,english -2,中文 diff --git a/table/plugins/csv/table/index.ts b/table/plugins/csv/table/index.ts deleted file mode 100644 index 1286ac75..00000000 --- a/table/plugins/csv/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadCsvTable } from "./load.ts" -export { saveCsvTable } from "./save.ts" diff --git a/table/plugins/csv/table/load.spec.ts b/table/plugins/csv/table/load.spec.ts deleted file mode 100644 index 436ac717..00000000 --- a/table/plugins/csv/table/load.spec.ts +++ /dev/null @@ -1,383 +0,0 @@ -import { Buffer } from "node:buffer" -import { writeTempFile } from "@dpkit/dataset" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadCsvTable } from "./load.ts" - -useRecording() - -describe("loadCsvTable", () => { - it("should load local file", async () => { - const path = await writeTempFile("id,name\n1,english\n2,中文") - const table = await loadCsvTable({ path }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load local file (multipart)", async () => { - const path1 = await writeTempFile("id,name\n1,english") - const path2 = await writeTempFile("id,name\n2,中文\n3,german") - - const table = await loadCsvTable({ path: [path1, path2] }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 3, name: "german" }, - ]) - }) - - it.fails("should load remote file", async () => { - const table = await loadCsvTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/csv/table/fixtures/table.csv", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it.fails("should load remote file (multipart)", async () => { - const table = await loadCsvTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/csv/table/fixtures/table.csv", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/csv/table/fixtures/table.csv", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should handle windows line terminator by default", async () => { - const path = await writeTempFile("id,name\r\n1,english\r\n2,中文") - const table = await loadCsvTable({ path }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should handle custom delimiter", async () => { - const path = await writeTempFile("id|name\n1|alice\n2|bob") - const table = await loadCsvTable({ - path, - dialect: { delimiter: "|" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should handle files without header", async () => { - const path = await writeTempFile("1,alice\n2,bob") - const table = await loadCsvTable({ - path, - dialect: { header: false }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { field1: 1, field2: "alice" }, - { field1: 2, field2: "bob" }, - ]) - }) - - it("should handle custom line terminator", async () => { - const path = await writeTempFile("id,name|1,alice|2,bob") - const table = await loadCsvTable({ - path, - dialect: { lineTerminator: "|" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it.skip("should handle escape char", async () => { - const path = await writeTempFile( - "id,name\n1,apple|,fruits\n2,orange|,fruits", - ) - - const table = await loadCsvTable({ - path, - dialect: { escapeChar: "|" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "apple,fruits" }, - { id: 2, name: "orange,fruits" }, - ]) - }) - - it("should handle custom quote character", async () => { - const path = await writeTempFile("id,name\n1,'alice smith'\n2,'bob jones'") - - const table = await loadCsvTable({ - path, - dialect: { quoteChar: "'" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "alice smith" }, - { id: 2, name: "bob jones" }, - ]) - }) - - it("should handle double quote by default", async () => { - const path = await writeTempFile( - 'id,name\n1,"alice""smith"\n2,"bob""jones"', - ) - - const table = await loadCsvTable({ - path, - dialect: { doubleQuote: true }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: 'alice"smith' }, - { id: 2, name: 'bob"jones' }, - ]) - }) - - it.skip("should handle disabling double quote", async () => { - const path = await writeTempFile( - 'id,name\n1,"alice""smith"\n2,"bob""jones"', - ) - - const table = await loadCsvTable({ - path, - dialect: { doubleQuote: false }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "alicesmith" }, - { id: 2, name: "bobjones" }, - ]) - }) - - it("should handle comment character", async () => { - const path = await writeTempFile( - "# This is a comment\nid,name\n1,alice\n# Another comment\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { commentChar: "#" }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should support headerRows", async () => { - const path = await writeTempFile("#comment\nid,name\n1,alice\n2,bob") - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2] }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should support headerJoin", async () => { - const path = await writeTempFile( - "#comment\nid,name\nint,str\n1,alice\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2, 3], headerJoin: "_" }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id_int: 1, name_str: "alice" }, - { id_int: 2, name_str: "bob" }, - ]) - }) - - it("should support commentRows", async () => { - const path = await writeTempFile("id,name\n1,alice\ncomment\n2,bob") - - const table = await loadCsvTable({ - path, - dialect: { commentRows: [3] }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should support headerRows and commentRows", async () => { - const path = await writeTempFile( - "#comment\nid,name\n1,alice\n#comment\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2], commentRows: [4] }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: 1, name: "alice" }, - { id: 2, name: "bob" }, - ]) - }) - - it("should support headerJoin and commentRows", async () => { - const path = await writeTempFile( - "#comment\nid,name\nint,str\n1,alice\n#comment\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2, 3], headerJoin: "_", commentRows: [5] }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id_int: 1, name_str: "alice" }, - { id_int: 2, name_str: "bob" }, - ]) - }) - - it("should handle null sequence", async () => { - const path = await writeTempFile( - "id,name,age\n1,alice,25\n2,N/A,30\n3,bob,N/A", - ) - const table = await loadCsvTable({ - path, - dialect: { nullSequence: "N/A" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "alice", age: 25 }, - { id: 2, name: null, age: 30 }, - { id: 3, name: "bob", age: null }, - ]) - }) - - it("should handle skip initial space", async () => { - const path = await writeTempFile( - "id,name,category\n1, alice, fruits\n2, bob, vegetables\n3,charlie,grains", - ) - const table = await loadCsvTable({ - path, - dialect: { skipInitialSpace: true }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "alice", category: "fruits" }, - { id: 2, name: "bob", category: "vegetables" }, - { id: 3, name: "charlie", category: "grains" }, - ]) - }) - - it("should handle multiple dialect options together", async () => { - const path = await writeTempFile( - "#comment\nid|'full name'|age\n1|'alice smith'|25\n2|'bob jones'|30", - ) - const table = await loadCsvTable({ - path, - dialect: { - delimiter: "|", - quoteChar: "'", - commentChar: "#", - header: true, - }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, "full name": "alice smith", age: 25 }, - { id: 2, "full name": "bob jones", age: 30 }, - ]) - }) - - it("should handle utf8 encoding", async () => { - const path = await writeTempFile( - Buffer.from("id,name\n1,café\n2,naïve", "utf8"), - ) - - const table = await loadCsvTable({ - path, - encoding: "utf8", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "café" }, - { id: 2, name: "naïve" }, - ]) - }) - - // TODO: currently not supported by nodejs-polars - it.skip("should handle utf16 encoding", async () => { - const path = await writeTempFile( - Buffer.from("id,name\n1,café\n2,naïve", "utf16le"), - ) - - const table = await loadCsvTable({ - path, - encoding: "utf16", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "café" }, - { id: 2, name: "naïve" }, - ]) - }) - - // TODO: currently not supported by nodejs-polars - it.skip("should handle latin1 encoding", async () => { - const path = await writeTempFile( - Buffer.from("id,name\n1,café\n2,résumé", "latin1"), - ) - - const table = await loadCsvTable({ - path, - encoding: "latin1", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "café" }, - { id: 2, name: "résumé" }, - ]) - }) -}) - -describe("loadCsvTable (format=tsv)", () => { - it("should load local file", async () => { - const path = await writeTempFile("id\tname\n1\tenglish\n2\t中文") - const table = await loadCsvTable({ path, format: "tsv" }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) -}) diff --git a/table/plugins/csv/table/load.ts b/table/plugins/csv/table/load.ts deleted file mode 100644 index 71bccdc6..00000000 --- a/table/plugins/csv/table/load.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { Dialect, Resource } from "@dpkit/metadata" -import { resolveDialect, resolveSchema } from "@dpkit/metadata" -import { prefetchFiles } from "@dpkit/dataset" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { joinHeaderRows } from "../../../table/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import { skipCommentRows } from "../../../table/index.ts" -import { stripInitialSpace } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" -import * as pl from "nodejs-polars" -import { inferCsvDialect } from "../dialect/index.ts" - -// TODO: Condier using sample to extract header first -// for better commentChar + headerRows/commentRows support -// (consult with the Data Package Working Group) - -export async function loadCsvTable( - resource: Partial & { format?: "csv" | "tsv" }, - options?: LoadTableOptions, -) { - const paths = await prefetchFiles(resource.path) - if (!paths.length) { - throw new Error("Resource path is not defined") - } - - let dialect = await resolveDialect(resource.dialect) - if (!dialect) { - dialect = await inferCsvDialect({ ...resource, path: paths[0] }, options) - } - - const scanOptions = getScanOptions(resource, dialect) - const tables: Table[] = [] - for (const path of paths) { - const table = pl.scanCSV(path, scanOptions) - tables.push(table) - } - - // There is no way to specify column names in nodejs-polars by default - // so we have to rename `column_*` to `field*` is table doesn't have header - let table = pl.concat(tables) - if (!scanOptions.hasHeader) { - table = table.rename( - Object.fromEntries( - table.columns.map(name => [name, name.replace("column_", "field")]), - ), - ) - } - - if (dialect) { - table = await joinHeaderRows(table, { dialect }) - table = skipCommentRows(table, { dialect }) - table = stripInitialSpace(table, { dialect }) - } - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} - -function getScanOptions(resource: Partial, dialect?: Dialect) { - const options: Partial = { - inferSchemaLength: 0, - truncateRaggedLines: true, - } - - if (resource.encoding) { - options.encoding = resource.encoding - - // Polars supports only utf-8 and utf-8-lossy encodings - if (options.encoding === "utf-8") { - options.encoding = "utf8" - } - - if (options.encoding !== "utf8") { - throw new Error(`Encoding ${options.encoding} for CSV files is not supported`) - } - } - - options.skipRows = getRowsToSkip(dialect) - options.hasHeader = dialect?.header !== false - options.eolChar = dialect?.lineTerminator ?? "\n" - options.sep = dialect?.delimiter ?? "," - - // TODO: try convincing nodejs-polars to support escapeChar - // https://github.com/pola-rs/polars/issues/3074 - //options.escapeChar = dialect?.escapeChar - - options.quoteChar = dialect?.quoteChar ?? '"' - options.nullValues = dialect?.nullSequence - - // TODO: try convincing nodejs-polars to support doubleQuote - //options.doubleQuote = dialect?.doubleQuote ?? true - - // TODO: remove ts-ignore when issues is fixed - // https://github.com/pola-rs/nodejs-polars/issues/334 - // @ts-ignore - options.commentPrefix = dialect?.commentChar - - return options -} - -function getRowsToSkip(dialect?: Dialect) { - const headerRows = getHeaderRows(dialect) - return headerRows[0] ? headerRows[0] - 1 : 0 -} - -function getHeaderRows(dialect?: Dialect) { - return dialect?.header !== false ? (dialect?.headerRows ?? [1]) : [] -} diff --git a/table/plugins/csv/table/save.spec.ts b/table/plugins/csv/table/save.spec.ts deleted file mode 100644 index e311b41e..00000000 --- a/table/plugins/csv/table/save.spec.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { readFile } from "node:fs/promises" -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { loadCsvTable } from "./load.ts" -import { saveCsvTable } from "./save.ts" - -describe("saveCsvTable", () => { - it("should save table to file", async () => { - const path = getTempFilePath() - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice", "Bob", "Charlie"], - }) - .lazy() - - await saveCsvTable(table, { path }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual("id,name\n1.0,Alice\n2.0,Bob\n3.0,Charlie\n") - }) - - it("should save with custom delimiter", async () => { - const path = getTempFilePath() - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice", "Bob", "Charlie"], - }) - .lazy() - - await saveCsvTable(table, { - path, - dialect: { delimiter: ";" }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual("id;name\n1.0;Alice\n2.0;Bob\n3.0;Charlie\n") - }) - - it("should save without header", async () => { - const path = getTempFilePath() - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice", "Bob", "Charlie"], - }) - .lazy() - - await saveCsvTable(table, { - path, - dialect: { header: false }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual("1.0,Alice\n2.0,Bob\n3.0,Charlie\n") - }) - - it("should save with custom quote char", async () => { - const path = getTempFilePath() - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice,Smith", "Bob,Jones", "Charlie,Brown"], - }) - .lazy() - - await saveCsvTable(table, { - path, - dialect: { quoteChar: "'" }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - "id,name\n1.0,'Alice,Smith'\n2.0,'Bob,Jones'\n3.0,'Charlie,Brown'\n", - ) - }) - - it("should save and load various data types", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveCsvTable(source, { - path, - fieldTypes: { - array: "array", - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadCsvTable({ path }, { denormalized: true }) - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: "true", - date: "2025-01-01", - datetime: "2025-01-01T00:00:00", - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: "1", - list: "1.0,2.0,3.0", - number: "1.1", - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: "2025", - yearmonth: "2025-01", - }, - ]) - }) -}) - -describe("saveCsvTable (format=tsv)", () => { - it("should save table to file", async () => { - const path = getTempFilePath() - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice", "Bob", "Charlie"], - }) - .lazy() - - await saveCsvTable(table, { path, format: "tsv" }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual("id\tname\n1.0\tAlice\n2.0\tBob\n3.0\tCharlie\n") - }) -}) diff --git a/table/plugins/csv/table/save.ts b/table/plugins/csv/table/save.ts deleted file mode 100644 index a7576ddc..00000000 --- a/table/plugins/csv/table/save.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { assertLocalPathVacant } from "@dpkit/dataset" -import type { SaveTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { denormalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" - -export async function saveCsvTable( - table: Table, - options: SaveTableOptions & { format?: "csv" | "tsv" }, -) { - const { path, format, overwrite } = options - const isTabs = format === "tsv" - - if (!overwrite) { - await assertLocalPathVacant(path) - } - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - table = await denormalizeTable(table, schema, { - nativeTypes: ["string"], - }) - - await table - .sinkCSV(path, { - maintainOrder: true, - includeHeader: options.dialect?.header ?? true, - separator: options.dialect?.delimiter ?? (isTabs ? "\t" : ","), - //lineTerminator: options.dialect?.lineTerminator ?? "\r\n", - quoteChar: options.dialect?.quoteChar ?? '"', - }) - .collect() - - return path -} diff --git a/table/plugins/inline/index.ts b/table/plugins/inline/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/inline/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/inline/plugin.ts b/table/plugins/inline/plugin.ts deleted file mode 100644 index cf952f75..00000000 --- a/table/plugins/inline/plugin.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import type { LoadTableOptions, TablePlugin } from "../../plugin.ts" -import { loadInlineTable } from "./table/index.ts" - -export class InlinePlugin implements TablePlugin { - async loadTable(resource: Resource, options?: LoadTableOptions) { - const isInline = getIsInline(resource) - if (!isInline) return undefined - - return await loadInlineTable(resource, options) - } -} - -function getIsInline(resource: Resource) { - const isTable = resource.type === "table" || !!resource.schema - return isTable && !!resource.data -} diff --git a/table/plugins/inline/table/index.ts b/table/plugins/inline/table/index.ts deleted file mode 100644 index 34861391..00000000 --- a/table/plugins/inline/table/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { loadInlineTable } from "./load.ts" diff --git a/table/plugins/inline/table/load.spec.ts b/table/plugins/inline/table/load.spec.ts deleted file mode 100644 index a0b08d8c..00000000 --- a/table/plugins/inline/table/load.spec.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { describe, expect, it } from "vitest" -import { loadInlineTable } from "./load.ts" - -describe("loadInlineTable", () => { - it("should throw on no data", async () => { - const resource = { name: "test" } - - await expect(loadInlineTable(resource)).rejects.toThrow( - "Resource data is not defined or tabular", - ) - }) - - it("should throw on bad data", async () => { - const resource = { name: "test", data: "bad" } - - await expect(loadInlineTable(resource)).rejects.toThrow( - "Resource data is not defined or tabular", - ) - }) - - it("should read arrays", async () => { - const resource = { - name: "test", - type: "table" as const, - data: [ - ["id", "name"], - [1, "english"], - [2, "中文"], - ], - } - - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]).toEqual(frame.toRecords()) - }) - - it("should read objects", async () => { - const resource = { - name: "test", - type: "table", - data: [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ], - schema: undefined, - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]).toEqual(frame.toRecords()) - }) - - it("should handle longer rows", async () => { - const resource = { - name: "test", - type: "table", - data: [ - ["id", "name"], - [1, "english"], - [2, "中文", "bad"], // extra cell - ], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]).toEqual(frame.toRecords()) - }) - - it("should handle shorter rows", async () => { - const resource = { - name: "test", - type: "table", - data: [ - ["id", "name"], - [1, "english"], - [2], // missing cell - ], - schema: { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - }, - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { id: 1, name: "english" }, - { id: 2, name: null }, - ]).toEqual(frame.toRecords()) - }) - - it("should handle various data types", async () => { - const resource = { - name: "test", - type: "table", - data: [ - { - string: "string", - number: 1, - boolean: true, - date: new Date("2025-01-01"), - time: new Date("2025-01-01"), - datetime: new Date("2025-01-01"), - }, - ], - schema: undefined, - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { - string: "string", - number: 1, - boolean: true, - date: new Date("2025-01-01"), - time: new Date("2025-01-01"), - datetime: new Date("2025-01-01"), - }, - ]).toEqual(frame.toRecords()) - }) - - it("should handle objects with shorter rows", async () => { - const resource = { - name: "test", - type: "table", - data: [{ id: 1, name: "english" }, { id: 2, name: "中文" }, { id: 3 }], - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 3, name: null }, - ]).toEqual(frame.toRecords()) - }) - - it("should handle objects with longer rows", async () => { - const resource = { - name: "test", - type: "table", - data: [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 3, name: "german", extra: "extra" }, - ], - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const frame = await table.collect() - - expect([ - { id: 1, name: "english", extra: null }, - { id: 2, name: "中文", extra: null }, - { id: 3, name: "german", extra: "extra" }, - ]).toEqual(frame.toRecords()) - }) -}) diff --git a/table/plugins/inline/table/load.ts b/table/plugins/inline/table/load.ts deleted file mode 100644 index 5ef6b05a..00000000 --- a/table/plugins/inline/table/load.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { resolveDialect } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { getRecordsFromRows } from "../../../data/index.ts" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import * as pl from "nodejs-polars" - -export async function loadInlineTable( - resource: Partial, - options?: LoadTableOptions, -) { - const data = resource.data - if (!Array.isArray(data)) { - throw new Error("Resource data is not defined or tabular") - } - - const dialect = await resolveDialect(resource.dialect) - const isRows = data.every(row => Array.isArray(row)) - - const records = isRows ? getRecordsFromRows(data, dialect) : data - let table = pl.DataFrame(records).lazy() - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} diff --git a/table/plugins/json/buffer/decode.ts b/table/plugins/json/buffer/decode.ts deleted file mode 100644 index 0a672643..00000000 --- a/table/plugins/json/buffer/decode.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Buffer } from "node:buffer" - -export function decodeJsonBuffer( - buffer: Buffer, - options: { isLines: boolean }, -) { - const string = buffer.toString("utf-8") - - return options.isLines - ? string - .split("\n") - .filter(Boolean) - .map(line => JSON.parse(line)) - : JSON.parse(string) -} diff --git a/table/plugins/json/buffer/encode.ts b/table/plugins/json/buffer/encode.ts deleted file mode 100644 index 952e85cd..00000000 --- a/table/plugins/json/buffer/encode.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Buffer } from "node:buffer" - -export function encodeJsonBuffer(data: any, options: { isLines: boolean }) { - const string = options.isLines - ? data.map((line: any) => JSON.stringify(line)).join("\n") - : JSON.stringify(data, null, 2) - - return Buffer.from(string) -} diff --git a/table/plugins/json/buffer/index.ts b/table/plugins/json/buffer/index.ts deleted file mode 100644 index e1f1ad14..00000000 --- a/table/plugins/json/buffer/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { encodeJsonBuffer } from "./encode.ts" -export { decodeJsonBuffer } from "./decode.ts" diff --git a/table/plugins/json/index.ts b/table/plugins/json/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/json/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/json/plugin.spec.ts b/table/plugins/json/plugin.spec.ts deleted file mode 100644 index 370eb38a..00000000 --- a/table/plugins/json/plugin.spec.ts +++ /dev/null @@ -1,227 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { JsonPlugin } from "./plugin.ts" -import * as loadModule from "./table/load.ts" -import * as saveModule from "./table/save.ts" - -vi.mock("./table/load.ts", () => ({ - loadJsonTable: vi.fn(), -})) - -vi.mock("./table/save.ts", () => ({ - saveJsonTable: vi.fn(), -})) - -describe("JsonPlugin", () => { - let plugin: JsonPlugin - let mockLoadJsonTable: ReturnType - let mockSaveJsonTable: ReturnType - - beforeEach(() => { - plugin = new JsonPlugin() - mockLoadJsonTable = vi.mocked(loadModule.loadJsonTable) - mockSaveJsonTable = vi.mocked(saveModule.saveJsonTable) - vi.clearAllMocks() - }) - - describe("loadTable", () => { - it("should load table from json file", async () => { - const resource: Partial = { - path: "test.json", - } - const mockTable = pl.DataFrame().lazy() - mockLoadJsonTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadJsonTable).toHaveBeenCalledWith( - { ...resource, format: "json" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should load table from jsonl file", async () => { - const resource: Partial = { - path: "test.jsonl", - } - const mockTable = pl.DataFrame().lazy() - mockLoadJsonTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadJsonTable).toHaveBeenCalledWith( - { ...resource, format: "jsonl" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should load table from ndjson file", async () => { - const resource: Partial = { - path: "test.ndjson", - } - const mockTable = pl.DataFrame().lazy() - mockLoadJsonTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadJsonTable).toHaveBeenCalledWith( - { ...resource, format: "ndjson" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-json files", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadJsonTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "json", - } - const mockTable = pl.DataFrame().lazy() - mockLoadJsonTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadJsonTable).toHaveBeenCalledWith( - { ...resource, format: "json" }, - undefined, - ) - expect(result).toEqual(mockTable) - }) - - it("should pass through load options", async () => { - const resource: Partial = { - path: "test.json", - } - const options = { denormalized: true } - const mockTable = pl.DataFrame().lazy() - mockLoadJsonTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource, options) - - expect(mockLoadJsonTable).toHaveBeenCalledWith( - { ...resource, format: "json" }, - options, - ) - }) - - it("should handle paths with directories", async () => { - const resource: Partial = { - path: "/path/to/data.json", - } - const mockTable = pl.DataFrame().lazy() - mockLoadJsonTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource) - - expect(mockLoadJsonTable).toHaveBeenCalledWith( - { ...resource, format: "json" }, - undefined, - ) - }) - }) - - describe("saveTable", () => { - it("should save table to json file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.json" } - mockSaveJsonTable.mockResolvedValue({ path: "output.json" }) - - const result = await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).toHaveBeenCalledWith(table, { - ...options, - format: "json", - }) - expect(result).toEqual({ path: "output.json" }) - }) - - it("should save table to jsonl file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.jsonl" } - mockSaveJsonTable.mockResolvedValue({ path: "output.jsonl" }) - - const result = await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).toHaveBeenCalledWith(table, { - ...options, - format: "jsonl", - }) - expect(result).toEqual({ path: "output.jsonl" }) - }) - - it("should save table to ndjson file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.ndjson" } - mockSaveJsonTable.mockResolvedValue({ path: "output.ndjson" }) - - const result = await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).toHaveBeenCalledWith(table, { - ...options, - format: "ndjson", - }) - expect(result).toEqual({ path: "output.ndjson" }) - }) - - it("should return undefined for non-json files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "json" as const } - mockSaveJsonTable.mockResolvedValue({ path: "output.txt" }) - - const result = await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).toHaveBeenCalledWith(table, { - ...options, - format: "json", - }) - expect(result).toEqual({ path: "output.txt" }) - }) - - it("should handle paths with directories", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "/path/to/output.json" } - mockSaveJsonTable.mockResolvedValue({ path: "/path/to/output.json" }) - - await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).toHaveBeenCalledWith(table, { - ...options, - format: "json", - }) - }) - - it("should return undefined for files without extension", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveJsonTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/table/plugins/json/plugin.ts b/table/plugins/json/plugin.ts deleted file mode 100644 index 4c6b3fed..00000000 --- a/table/plugins/json/plugin.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { - LoadTableOptions, - SaveTableOptions, - TablePlugin, -} from "../../plugin.ts" -import type { Table } from "../../table/index.ts" -import { loadJsonTable } from "./table/index.ts" -import { saveJsonTable } from "./table/index.ts" - -export class JsonPlugin implements TablePlugin { - async loadTable(resource: Partial, options?: LoadTableOptions) { - const jsonFormat = getJsonFormat(resource) - if (!jsonFormat) return undefined - - return await loadJsonTable({ ...resource, format: jsonFormat }, options) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const jsonFormat = getJsonFormat({ path, format }) - if (!jsonFormat) return undefined - - return await saveJsonTable(table, { ...options, format: jsonFormat }) - } -} - -function getJsonFormat(resource: Partial) { - const format = inferFormat(resource) - return format === "json" || format === "jsonl" || format === "ndjson" - ? format - : undefined -} diff --git a/table/plugins/json/table/fixtures/generated/loadJsonTable-file-variations-should-load-remote-file-multipart_2057098191/recording.har b/table/plugins/json/table/fixtures/generated/loadJsonTable-file-variations-should-load-remote-file-multipart_2057098191/recording.har deleted file mode 100644 index fdfe1d9f..00000000 --- a/table/plugins/json/table/fixtures/generated/loadJsonTable-file-variations-should-load-remote-file-multipart_2057098191/recording.har +++ /dev/null @@ -1,298 +0,0 @@ -{ - "log": { - "_recordingName": "loadJsonTable-file variations-should load remote file (multipart)", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "faadc82281c34c649fdf6fc5ef58d1a3", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 114, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.json" - }, - "response": { - "bodySize": 67, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 67, - "text": "[{ \"id\": 1, \"name\": \"english\" }, { \"id\": 2, \"name\": \"中文\" }]\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "67" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:23:17 GMT" - }, - { - "name": "etag", - "value": "W/\"fa647b433c911502c1bbd02dd6f1fd56cf63ad751835d323f01bc20feca076e7\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:28:17 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "7e01ef48cc640131025ec29215e294adeda2edee" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "A67A:2B2025:7E508D:96FE9A:68EA3E24" - }, - { - "name": "x-served-by", - "value": "cache-lis1490051-LIS" - }, - { - "name": "x-timer", - "value": "S1760181797.046487,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 898, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:23:16.956Z", - "time": 239, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 239 - } - }, - { - "_id": "faadc82281c34c649fdf6fc5ef58d1a3", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 114, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.json" - }, - "response": { - "bodySize": 67, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 67, - "text": "[{ \"id\": 1, \"name\": \"english\" }, { \"id\": 2, \"name\": \"中文\" }]\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "67" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:23:17 GMT" - }, - { - "name": "etag", - "value": "W/\"fa647b433c911502c1bbd02dd6f1fd56cf63ad751835d323f01bc20feca076e7\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:28:17 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "15ce999735acf644ffe8741b41eecdcbdbf27f82" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "A67A:2B2025:7E508D:96FE9A:68EA3E24" - }, - { - "name": "x-served-by", - "value": "cache-lis1490037-LIS" - }, - { - "name": "x-timer", - "value": "S1760181797.155003,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 898, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:23:16.956Z", - "time": 328, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 328 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/json/table/fixtures/generated/loadJsonTable-file-variations-should-load-remote-file_3069001120/recording.har b/table/plugins/json/table/fixtures/generated/loadJsonTable-file-variations-should-load-remote-file_3069001120/recording.har deleted file mode 100644 index cc0a08ba..00000000 --- a/table/plugins/json/table/fixtures/generated/loadJsonTable-file-variations-should-load-remote-file_3069001120/recording.har +++ /dev/null @@ -1,156 +0,0 @@ -{ - "log": { - "_recordingName": "loadJsonTable-file variations-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "faadc82281c34c649fdf6fc5ef58d1a3", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 114, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.json" - }, - "response": { - "bodySize": 67, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 67, - "text": "[{ \"id\": 1, \"name\": \"english\" }, { \"id\": 2, \"name\": \"中文\" }]\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "67" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:23:16 GMT" - }, - { - "name": "etag", - "value": "W/\"fa647b433c911502c1bbd02dd6f1fd56cf63ad751835d323f01bc20feca076e7\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:28:16 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "d119a986d4dd1ae4a34a78f89c147a0b0d9037ef" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "A67A:2B2025:7E508D:96FE9A:68EA3E24" - }, - { - "name": "x-served-by", - "value": "cache-lis1490051-LIS" - }, - { - "name": "x-timer", - "value": "S1760181797.625750,VS0,VE171" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 901, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:23:16.435Z", - "time": 463, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 463 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_278600870/recording.har b/table/plugins/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_278600870/recording.har deleted file mode 100644 index 40c6e714..00000000 --- a/table/plugins/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_278600870/recording.har +++ /dev/null @@ -1,298 +0,0 @@ -{ - "log": { - "_recordingName": "loadJsonTable (format=jsonl)-file variations-should load remote file (multipart)", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "b864cf284ec4bab3e6e7b79881054128", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 115, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.jsonl" - }, - "response": { - "bodySize": 60, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 60, - "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中文\"}\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "60" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:23:17 GMT" - }, - { - "name": "etag", - "value": "W/\"d29c5f690944180ee9bfc22556d796070363b387a7cc05ae8ae36f3d657c337c\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:28:17 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "56564a547598dbd863efbe05e43195f9a167c641" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "BF90:10CB6B:7A5633:9304AA:68EA3E25" - }, - { - "name": "x-served-by", - "value": "cache-lis1490037-LIS" - }, - { - "name": "x-timer", - "value": "S1760181798.838033,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 898, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:23:17.847Z", - "time": 143, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 143 - } - }, - { - "_id": "b864cf284ec4bab3e6e7b79881054128", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 115, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.jsonl" - }, - "response": { - "bodySize": 60, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 60, - "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中文\"}\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "60" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:23:17 GMT" - }, - { - "name": "etag", - "value": "W/\"d29c5f690944180ee9bfc22556d796070363b387a7cc05ae8ae36f3d657c337c\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:28:17 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "b78d30df5911023c6f21c67848e06c3fef6ea144" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "BF90:10CB6B:7A5633:9304AA:68EA3E25" - }, - { - "name": "x-served-by", - "value": "cache-lis1490051-LIS" - }, - { - "name": "x-timer", - "value": "S1760181798.836332,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 898, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:23:17.847Z", - "time": 143, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 143 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1248351709/recording.har b/table/plugins/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1248351709/recording.har deleted file mode 100644 index 4cf4929e..00000000 --- a/table/plugins/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1248351709/recording.har +++ /dev/null @@ -1,156 +0,0 @@ -{ - "log": { - "_recordingName": "loadJsonTable (format=jsonl)-file variations-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "b864cf284ec4bab3e6e7b79881054128", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 115, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.jsonl" - }, - "response": { - "bodySize": 60, - "content": { - "mimeType": "text/plain; charset=utf-8", - "size": 60, - "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中文\"}\n" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-encoding", - "value": "gzip" - }, - { - "name": "content-length", - "value": "60" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "text/plain; charset=utf-8" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:23:17 GMT" - }, - { - "name": "etag", - "value": "W/\"d29c5f690944180ee9bfc22556d796070363b387a7cc05ae8ae36f3d657c337c\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:28:17 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "8e34af7eb8496145a7cdc993416910c25cab24da" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "BF90:10CB6B:7A5633:9304AA:68EA3E25" - }, - { - "name": "x-served-by", - "value": "cache-lis1490051-LIS" - }, - { - "name": "x-timer", - "value": "S1760181798.526531,VS0,VE175" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 901, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:23:17.579Z", - "time": 217, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 217 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/json/table/fixtures/table.json b/table/plugins/json/table/fixtures/table.json deleted file mode 100644 index 9efcfcd4..00000000 --- a/table/plugins/json/table/fixtures/table.json +++ /dev/null @@ -1 +0,0 @@ -[{ "id": 1, "name": "english" }, { "id": 2, "name": "中文" }] diff --git a/table/plugins/json/table/fixtures/table.jsonl b/table/plugins/json/table/fixtures/table.jsonl deleted file mode 100644 index a431c033..00000000 --- a/table/plugins/json/table/fixtures/table.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"id":1,"name":"english"} -{"id":2,"name":"中文"} diff --git a/table/plugins/json/table/index.ts b/table/plugins/json/table/index.ts deleted file mode 100644 index 0b10c6ac..00000000 --- a/table/plugins/json/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadJsonTable } from "./load.ts" -export { saveJsonTable } from "./save.ts" diff --git a/table/plugins/json/table/load.spec.ts b/table/plugins/json/table/load.spec.ts deleted file mode 100644 index b047eff1..00000000 --- a/table/plugins/json/table/load.spec.ts +++ /dev/null @@ -1,240 +0,0 @@ -import { writeTempFile } from "@dpkit/dataset" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadJsonTable } from "./load.ts" - -useRecording() - -describe("loadJsonTable", () => { - describe("file variations", () => { - it("should load local file", async () => { - const body = '[{"id":1,"name":"english"},{"id":2,"name":"中文"}]' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ path }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load local file (multipart)", async () => { - const body = '[{"id":1,"name":"english"},{"id":2,"name":"中文"}]' - const path1 = await writeTempFile(body) - const path2 = await writeTempFile(body) - - const table = await loadJsonTable({ - path: [path1, path2], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file", async () => { - const table = await loadJsonTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.json", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file (multipart)", async () => { - const table = await loadJsonTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.json", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.json", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) - - describe("dialect variations", () => { - it("should handle property", async () => { - const body = '{"key": [{"id":1,"name":"english"},{"id":2,"name":"中文"}]}' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - dialect: { property: "key" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should handle item keys", async () => { - const body = '[{"id":1,"name":"english"},{"id":2,"name":"中文"}]' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - dialect: { itemKeys: ["name"] }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { name: "english" }, - { name: "中文" }, - ]) - }) - - it("should handle item type (array)", async () => { - const body = '[["id","name"],[1,"english"],[2,"中文"]]' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - dialect: { itemType: "array" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load item type (object)", async () => { - const body = '[{"id":1,"name":"english"},{"id":2,"name":"中文"}]' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - dialect: { itemType: "object" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) -}) - -describe("loadJsonTable (format=jsonl)", () => { - describe("file variations", () => { - it("should load local file", async () => { - const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ path, format: "jsonl" }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load local file (multipart)", async () => { - const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' - const path1 = await writeTempFile(body) - const path2 = await writeTempFile(body) - - const table = await loadJsonTable({ - path: [path1, path2], - format: "jsonl", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file", async () => { - const table = await loadJsonTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.jsonl", - format: "jsonl", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file (multipart)", async () => { - const table = await loadJsonTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.jsonl", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/json/table/fixtures/table.jsonl", - ], - format: "jsonl", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) - - describe("dialect variations", () => { - it("should handle item keys", async () => { - const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - format: "jsonl", - dialect: { itemKeys: ["name"] }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { name: "english" }, - { name: "中文" }, - ]) - }) - - it("should handle item type (array)", async () => { - const body = '["id","name"]\n[1,"english"]\n[2,"中文"]' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - format: "jsonl", - dialect: { itemType: "array" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load item type (object)", async () => { - const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' - const path = await writeTempFile(body) - - const table = await loadJsonTable({ - path, - format: "jsonl", - dialect: { itemType: "object" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) -}) diff --git a/table/plugins/json/table/load.ts b/table/plugins/json/table/load.ts deleted file mode 100644 index b98ecd8f..00000000 --- a/table/plugins/json/table/load.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { Dialect, Resource } from "@dpkit/metadata" -import { resolveDialect } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { loadFile, prefetchFiles } from "@dpkit/dataset" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" -import * as pl from "nodejs-polars" -import { decodeJsonBuffer } from "../buffer/index.ts" - -export async function loadJsonTable( - resource: Partial & { format?: "json" | "jsonl" | "ndjson" }, - options?: LoadTableOptions, -) { - const isLines = resource.format === "jsonl" || resource.format === "ndjson" - - const paths = await prefetchFiles(resource.path) - if (!paths.length) { - throw new Error("Resource path is not defined") - } - - const dialect = await resolveDialect(resource.dialect) - - const tables: Table[] = [] - for (const path of paths) { - if (isLines && !dialect) { - const table = pl.scanJson(path) - tables.push(table) - continue - } - - const buffer = await loadFile(path) - let data = decodeJsonBuffer(buffer, { isLines }) - if (dialect) { - data = processData(data, dialect) - } - - const table = pl.DataFrame(data).lazy() - tables.push(table) - } - - let table = pl.concat(tables) - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} - -function processData(data: any, dialect: Dialect) { - if (dialect.property) { - data = data[dialect.property] - } - - if (dialect.itemType === "array") { - const keys = data[0] - - data = data - .slice(1) - .map((row: any) => - Object.fromEntries( - keys.map((key: any, index: number) => [key, row[index]]), - ), - ) - } - - if (dialect.itemKeys) { - const keys = dialect.itemKeys - - data = data.map((row: any) => - Object.fromEntries(keys.map((key: any) => [key, row[key]])), - ) - } - - return data -} diff --git a/table/plugins/json/table/parse.ts b/table/plugins/json/table/parse.ts deleted file mode 100644 index a516b88d..00000000 --- a/table/plugins/json/table/parse.ts +++ /dev/null @@ -1,5 +0,0 @@ -export function parseJson(string: string, options: { isLines: boolean }) { - return options.isLines - ? string.split("\n").map(line => JSON.parse(line)) - : JSON.parse(string) -} diff --git a/table/plugins/json/table/save.spec.ts b/table/plugins/json/table/save.spec.ts deleted file mode 100644 index 6147017f..00000000 --- a/table/plugins/json/table/save.spec.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { readFile } from "node:fs/promises" -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { loadJsonTable } from "./load.ts" -import { saveJsonTable } from "./save.ts" - -const row1 = { id: 1, name: "english" } -const row2 = { id: 2, name: "中文" } -const table = pl.readRecords([row1, row2]).lazy() - -describe("saveJsonTable", () => { - it("should save table to file", async () => { - const path = getTempFilePath() - await saveJsonTable(table, { path }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual(JSON.stringify([row1, row2], null, 2)) - }) - - it("should handle property", async () => { - const path = getTempFilePath() - - await saveJsonTable(table, { - path, - dialect: { property: "key" }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual(JSON.stringify({ key: [row1, row2] }, null, 2)) - }) - - it("should handle item keys", async () => { - const path = getTempFilePath() - - await saveJsonTable(table, { - path, - dialect: { itemKeys: ["name"] }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - JSON.stringify([{ name: row1.name }, { name: row2.name }], null, 2), - ) - }) - - it("should handle item type (array)", async () => { - const path = getTempFilePath() - - await saveJsonTable(table, { - path, - dialect: { itemType: "array" }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - JSON.stringify( - [Object.keys(row1), Object.values(row1), Object.values(row2)], - null, - 2, - ), - ) - }) - - it("should save and load various data types", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveJsonTable(source, { - path, - fieldTypes: { - array: "array", - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadJsonTable({ path }, { denormalized: true }) - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: true, - date: "2025-01-01", - datetime: "2025-01-01T00:00:00", - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: [1.0, 2.0, 3.0], - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) -}) - -describe("saveJsonTable (format=jsonl)", () => { - it("should save table to file", async () => { - const path = getTempFilePath() - - await saveJsonTable(table, { path, format: "jsonl" }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - [JSON.stringify(row1), JSON.stringify(row2)].join("\n"), - ) - }) - - it("should handle item keys", async () => { - const path = getTempFilePath() - await saveJsonTable(table, { - path, - format: "jsonl", - dialect: { itemKeys: ["name"] }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - [ - JSON.stringify({ name: row1.name }), - JSON.stringify({ name: row2.name }), - ].join("\n"), - ) - }) - - it("should handle item type (array)", async () => { - const path = getTempFilePath() - await saveJsonTable(table, { - path, - format: "jsonl", - dialect: { itemType: "array" }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - [ - JSON.stringify(Object.keys(row1)), - JSON.stringify(Object.values(row1)), - JSON.stringify(Object.values(row2)), - ].join("\n"), - ) - }) - - it("should handle item type (object)", async () => { - const path = getTempFilePath() - await saveJsonTable(table, { - path, - format: "jsonl", - dialect: { itemType: "object" }, - }) - - const content = await readFile(path, "utf-8") - expect(content).toEqual( - [JSON.stringify(row1), JSON.stringify(row2)].join("\n"), - ) - }) -}) diff --git a/table/plugins/json/table/save.ts b/table/plugins/json/table/save.ts deleted file mode 100644 index a26c5357..00000000 --- a/table/plugins/json/table/save.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { saveFile } from "@dpkit/dataset" -import type { Dialect } from "@dpkit/metadata" -import type { SaveTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { denormalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" -import { decodeJsonBuffer, encodeJsonBuffer } from "../buffer/index.ts" - -// TODO: rebase on sinkJSON when it is available -// https://github.com/pola-rs/nodejs-polars/issues/353 - -export async function saveJsonTable( - table: Table, - options: SaveTableOptions & { format?: "json" | "jsonl" | "ndjson" }, -) { - const { path, dialect, overwrite, format } = options - const isLines = format === "jsonl" || format === "ndjson" - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - table = await denormalizeTable(table, schema, { - nativeTypes: ["boolean", "integer", "list", "number", "string", "year"], - }) - - // We use polars to serialize the data - // But encode it manually to support dialects/formatting - const frame = await table.collect() - let buffer = frame.writeJSON({ format: isLines ? "lines" : "json" }) - let data = decodeJsonBuffer(buffer, { isLines }) - - if (dialect) { - data = processData(data, dialect) - } - - buffer = encodeJsonBuffer(data, { isLines }) - await saveFile(path, buffer, { overwrite }) - - return path -} - -function processData(records: Record[], dialect: Dialect) { - let data: any = records - - if (dialect.itemKeys) { - const keys = dialect.itemKeys - data = data.map((row: any) => - Object.fromEntries(keys.map((key: any) => [key, row[key]])), - ) - } - - if (dialect.itemType === "array") { - const keys = dialect.itemKeys ?? Object.keys(data[0]) - data = [keys, ...data.map((row: any) => keys.map((key: any) => row[key]))] - } - - if (dialect.property) { - data = { [dialect.property]: data } - } - - return data -} diff --git a/table/plugins/ods/index.ts b/table/plugins/ods/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/ods/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/ods/plugin.spec.ts b/table/plugins/ods/plugin.spec.ts deleted file mode 100644 index 54834113..00000000 --- a/table/plugins/ods/plugin.spec.ts +++ /dev/null @@ -1,188 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { OdsPlugin } from "./plugin.ts" -import * as loadModule from "./table/load.ts" -import * as saveModule from "./table/save.ts" - -vi.mock("./table/load.ts", () => ({ - loadOdsTable: vi.fn(), -})) - -vi.mock("./table/save.ts", () => ({ - saveOdsTable: vi.fn(), -})) - -describe("OdsPlugin", () => { - let plugin: OdsPlugin - let mockLoadOdsTable: ReturnType - let mockSaveOdsTable: ReturnType - - beforeEach(() => { - plugin = new OdsPlugin() - mockLoadOdsTable = vi.mocked(loadModule.loadOdsTable) - mockSaveOdsTable = vi.mocked(saveModule.saveOdsTable) - vi.clearAllMocks() - }) - - describe("loadTable", () => { - it("should load table from ods file", async () => { - const resource: Partial = { - path: "test.ods", - } - const mockTable = pl.DataFrame().lazy() - mockLoadOdsTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadOdsTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-ods files", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "ods", - } - const mockTable = pl.DataFrame().lazy() - mockLoadOdsTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadOdsTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should pass through load options", async () => { - const resource: Partial = { - path: "test.ods", - } - const options = { denormalized: true } - const mockTable = pl.DataFrame().lazy() - mockLoadOdsTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource, options) - - expect(mockLoadOdsTable).toHaveBeenCalledWith(resource, options) - }) - - it("should handle paths with directories", async () => { - const resource: Partial = { - path: "/path/to/data.ods", - } - const mockTable = pl.DataFrame().lazy() - mockLoadOdsTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource) - - expect(mockLoadOdsTable).toHaveBeenCalledWith(resource, undefined) - }) - - it("should return undefined for json files", async () => { - const resource: Partial = { - path: "test.json", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for xlsx files", async () => { - const resource: Partial = { - path: "test.xlsx", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("saveTable", () => { - it("should save table to ods file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.ods" } - mockSaveOdsTable.mockResolvedValue("output.ods") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.ods") - }) - - it("should return undefined for non-ods files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "ods" as const } - mockSaveOdsTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.txt") - }) - - it("should handle paths with directories", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "/path/to/output.ods" } - mockSaveOdsTable.mockResolvedValue("/path/to/output.ods") - - await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).toHaveBeenCalledWith(table, options) - }) - - it("should return undefined for files without extension", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for xlsx files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.xlsx" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for json files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.json" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveOdsTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/table/plugins/ods/plugin.ts b/table/plugins/ods/plugin.ts deleted file mode 100644 index 6abeef3a..00000000 --- a/table/plugins/ods/plugin.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { - LoadTableOptions, - SaveTableOptions, - TablePlugin, -} from "../../plugin.ts" -import type { Table } from "../../table/index.ts" -import { loadOdsTable } from "./table/index.ts" -import { saveOdsTable } from "./table/index.ts" - -export class OdsPlugin implements TablePlugin { - async loadTable(resource: Partial, options?: LoadTableOptions) { - const isOds = getIsOds(resource) - if (!isOds) return undefined - - return await loadOdsTable(resource, options) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const isOds = getIsOds({ path, format }) - if (!isOds) return undefined - - return await saveOdsTable(table, options) - } -} - -function getIsOds(resource: Partial) { - const format = inferFormat(resource) - return ["ods"].includes(format ?? "") -} diff --git a/table/plugins/ods/table/fixtures/generated/loadOdsTable-file-variations-should-load-multipart-remote-file_2072534240/recording.har b/table/plugins/ods/table/fixtures/generated/loadOdsTable-file-variations-should-load-multipart-remote-file_2072534240/recording.har deleted file mode 100644 index 980e324f..00000000 --- a/table/plugins/ods/table/fixtures/generated/loadOdsTable-file-variations-should-load-multipart-remote-file_2072534240/recording.har +++ /dev/null @@ -1,292 +0,0 @@ -{ - "log": { - "_recordingName": "loadOdsTable-file variations-should load multipart remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "9d5062962d85f533a00d57678f1e9995", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 112, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/ods/table/fixtures/table.ods" - }, - "response": { - "bodySize": 11614, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 11614, - "text": "UEsDBBQAAAgAALJWS1uFbDmKLgAAAC4AAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlvbi92bmQub2FzaXMub3BlbmRvY3VtZW50LnNwcmVhZHNoZWV0UEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3BvcHVwbWVudS9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABwAAABDb25maWd1cmF0aW9uczIvcHJvZ3Jlc3NiYXIvUEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAQ29uZmlndXJhdGlvbnMyL2Zsb2F0ZXIvUEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvdG9vbGJhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABoAAABDb25maWd1cmF0aW9uczIvdG9vbHBhbmVsL1BLAwQUAAgICACyVktbAAAAAAAAAAAAAAAAJwAAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbAMAUEsHCAAAAAACAAAAAAAAAFBLAwQUAAAIAACyVktbAAAAAAAAAAAAAAAAHwAAAENvbmZpZ3VyYXRpb25zMi9pbWFnZXMvQml0bWFwcy9QSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAoAAABzdHlsZXMueG1s5Rtrc9u48Xt/hUaZ3tzNlOZDsmPpYmXuck3ba5LeJGn78QYmIQoNSHBA0LLz8X5Pf1V/SRcvvknRcuQkrexxQuwudrEv7ILQs+e3CZ3dYJ4Tll7N/TNvPsNpyCKSxlfzv79/6VzOn29+94xttyTE64iFRYJT4eTijuJ8BsRpvs44zmEQCTVHwdM1QznJ1ylKcL4W4ZplOLWk6y7NWrHV42GeL8TVfCdEtnbd/X5/tl+cMR6779+6EuYIfCtcix3zKKJ92IHnLdzYjZBAzg3B+yeW4nYnkl4Kf7VauQpqUSOWDEztu4Dh4BtYQG6x85CTTExdvsauL3zLeDKVWuLWaRMkdgNrunRfA1D9ef3K4mtrTuVmbF/jxxgr2UkCjWG1s3T1c7WyqZxuc+psmROyJAPHuKZtpvtRrntOBOaloSlJPwwbWkJLQ3O0H12S77kSpyZJOCpJiGhYTl6hZgWnCikKXUyxXHju+md+6c6SDtzbqIvHZbxtWZFGOla0/vBthjmRIEQV2boxQ11vKlQn+6VErlOLSp6DxKLFmW2nuxjQBSVPBKYf97GVq5DqFpzKS+JCdqsLGvFFNJ18EdVpeSZG5Dx3Oc4YF6UxbuLJpriJByIh3CE+2SgKuWFRqbjJJkUt3gkWaCqxxK3TUnaEc5vsU5uhPmVaJNeYT7YdbAcdD98STK31S/v1SsKYk+QOSSHLsGxdo9bTGcraZrqcb+zOuWWwa25RiJ0IhzTfPNNylMMz/SzZXs1/wv9C/yhm71AK2wv4gUVLCL27mn+DMpZ/X8PRA/NZY0qJ68Q4BfkhA+V3ucBJAyUjIoRd4wZxokLJHZfpJcd4QKAa6JQSvCJgauUn45pp4U3Rzp7k+acSDabcTpJNIk4QjrMEpQ8R7g0TTGli9uLnv87evRgUro34KH7VZjp7i+OCIj5ZSkvwyNL+hG9QimKgmyBphfzJpHSHEosZ1xW6XU2Et6igpm63MxthVY53Qkzp3KJniKOYo2znZJDqMBcEin0NAmyYhWVORHKBUllGemfLVeCTtFKerAS6pErUgVjeMg3OyUcA+14m1BhFaVygGIaoHghhsxAcpH71ft6e1oENAKXt9FnhyLktjuaggZaJhX3cWYjhZgEv3nR5yi2a4tsDXEusXr4ldEfanEvQX94oq/eYc4qNlTFJWBrYPDdsBE6cC84+SHkog031yWJ5cY6W85ksmWCno7SEPA1W23CrLLKHqRyW6eYvZY58NiT5DkVs74BH5lg4t9JTfP9Sekof/K4O12LKkh5qNSdhEbhABrqqfGzMR6X7oUKwPEPSQUmEmUZFNNshO3tWpKEolBcqqSH0iNR2aR+SYueaYwR9BGiGhGJErrrJSRphWX3ITllNIuVQHfUW0RyXVrCxlFeW7VvVeFQVOXagoJWKVMyNiQQvQChdMDHJXkj9/r7jwH2bVysYg3sEI8UCyiPnA+apWrAW4yGBGjw8UE2GLWOplZ4P5uSKcig3Vxhljh7JAX1L+jQ5oB77tpZUaPOBhOAOU4J+8BDZrPTW8jTIafIbyTQ69lWqAY0ySiKTDxLEwXFgOtXf/MA52+8wivJfA+9XvwcH/D6SJx/emff08mmZVhooIZb1uo28KpU1OTeS2xY+oWcgMo04MUji7DCJd8K6dAtoJDFcIDYyFMlDNAfCWwm4WC2lgDXINRNCHjH1AineigEQ12JUsFouBf8jOVHuN5SBe6gaGdjAP8FefvqyvLdsGEg2I4VmRwRD8sCC004zmDQMfDhl9BQrPZXHWP81CelEaU5KXeUrk6d0x24ad6cn63jzFtLMPCUkVc13DHQRiYnIgYdi0DNnOUdYcMhR4V0fK3/h/eLZFdwwCt4pj0ZUgA8IAUU2SRB1MgqVBQgQzOvijUO7wlsopJAiq/ZLt2Qug28zK1eoHrsru0uuGbVzNXdpi1vbqTf/+e3f5YytSTY9gPuocz6UNVS1YNOrB5/2Ip3+Rf6faV8rL0FZWQKkEdGl9Q2iBf72u29i8f1V6bQoy+hdY/81Tu0OW/KelUJPe3hgS/hKM+4XlFQ7WXPIZn+G8gg8d8Rm0yu1imioo6IkTp2cFVw2VVtyayfn0O0g2XlAiKXCVkHjnRokhGrSq7ku04arjtZGf2FaEj0mKaDLEzBXWA3vTb0GgRXdW6Om5DSlNskhp9w5DYyZf5TWrcWmaJ0z/YLUgaQiV7jyjlpHML6O4ITrGDwfMGLvOEk/QI3sbEnXbw74QNAqioa6xqEObLIq34McJ4ywI3U0Tfa+Jm6S7GrRBwW/RuEHuX2mUbN7CiuXQzFLYVO+po7gcnEp7sAEgEvYoDokO8Yj2cx5Z0+XmZipDm725NKTP0f7VHnapT7TVfuSMZGeVr2fNHbsOq26BtJnPWiUwCakBuFlVBmM6SnqDqTTL+K/Lg3K+hWXdleIEAFqLdiq05wu9KKYwwJ5ejCA0eI0WaXvYMMo8q81Wf2Jsego2c2yj0lXYajT1el85eLCu8e+/QYXgiP62GqwWftUalit7qeGH9Gje4JUwClVEIa2752mgn8idYZ+UjV8EQv9I+eMP37cKyFPpoSt+gx0I/VNVAPsLjsELjfZ+3UzP4Sysfry9oNB9X12XY00fhrhyL7PWOIYV/W8R3HVIxQ10FkaRR3XWD5AUbXK9gtT1GJUUYvHVlSkPkcratrq3+J8/DjxHrln6CRA19rN1uSE9Xj1gv2hx0xaOcdFSFOx6o57HazO4UGQ2rtQ/bLPuE3va9WWX5Y4KtvdxGt5if5HJl8gzrxZ4M0Wnh6PruavfRijji8Hd4H3UfLW9w7EDifY3EHQE/9N3wxvIGiF5g28V+Sa4xayQmtgRbA4315yaGVLd5iMSoN16KrgHmMXHMmuQ3eYHVLx3ZHTv0QLb3GYsCvp4mKFJhAu2oRoscBTOC7bhJdANoXjeZswXK3CcaVqwosuYeD7eIxwVx06dA2JR0m3jFK2x5EzOMf5uY8ur1UW6Dp4c7C6wadit7q4J7NQggQJHQs4+Mo0eJy3lYOvWm2mirEDOYQVoiHe6yzx5z1I3fPu5rUqyh1xXe3LMh21WZqxLWPy0lFzg5BrsFc2vLNgdb4w9yggJcYAMzcsmoP2bkVztHZTY3VpLke4w1IZcT6HpPa2SVvMpkhuxxYTbBj8j9mwfp4cnC1X1Xlyrd42d24UmffUt2Q97aQnf0oV1TAgxGLccZjOcr8Oj/pSlXbYv93B9GoACcrLKcorl2ZQzjT2rrweET1JT4u/eaa+IJWZf/Mdxhp78/z582due9CMZC0ltMwvLdlsLToNjVUe4fkhVK3GUtBfipyijOQzMyBXqTP/xrfi1cY6EtvpGjaaILHBHJfY7VjokNHemq9ijdgs6NhMP3Ecy/ewUvb7mlEPuJtv9f8EEbSOqp+/6+iuwbExpOK0JQU0AXiwH4B9vURy1DUSCF4vOIeS3fHBSz3PVb+eZ6SQiJs/zKzAyeGpJZKd2vfX54t1sDhbLFeBt1penksWa/VbLrrPwZvr+2q83hrYrROoKz+b1apOoMc+f5S4/RnP7f+S9+a/UEsHCGPvHlh3CgAAJD4AAFBLAwQUAAgICACyVktbAAAAAAAAAAAAAAAADAAAAG1hbmlmZXN0LnJkZs2TzW6DMBCE7zyFZc7YQC8FBXIoyrlqn8A1hlgFL/KaEt6+jpNWUaSq6p/U465GM9+OtJvtYRzIi7KowVQ0YyklykhotekrOrsuuaXbOtrYtisfmh3xaoOlnyq6d24qOV+WhS03DGzPs6IoeJrzPE+8IsHVOHFIDMa0jggJHo1CafXkfBo5zuIJZldRdOugkHn3ID2L3TqpoLIKYbZSvYe2IJGBQI0JTMqEdIMcuk5LxTOW81E5waHt4sdgvdODojxg8CuOz9jeiAym5V7gvbDuXIPffJVoeu5jenXTxfHfI5RgnDLuT+q7O3n/5/4uz/8Z4q+0dkRsQM6jZ/qQ57TyH1VHr1BLBwi092jSBQEAAIMDAABQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAsAAABjb250ZW50LnhtbL1ZzXLbNhC+9yk06kxvMCw5ndiq7RySyUxbJ4fY7fQKkUsSDUhwAFCUc2svfYNe+gy95zGSd+j0OboACQqkRYVuHF+UCPvt7of9AyCfP9vmYrYBpbksLuaLo+P5DIpIxrxIL+Y/3bwkp/Nnl1+dyyThEaxiGVU5FIZEsjD47wy1C70qFWj8xowzUqliJZnmelWwHPTKRCtZQuF1V3d1Vs5vsx5pfWIu5pkx5YrSuq6P6pMjqVJ684ZaGTGwNdSjUxXHYh96eXx8QlMaM8PIhkP9tdfYZibfq7E4OzujTtpBNR8xvaC/vLq6jjLIGeGFNqyIYKcVf1qrAydS5XoEv6SN2INjmY9aRgSBDQa0Q+tI8dJMTUeDDhNhXU/VtthQN2cmG4nxKX2FQvfx6srjm/Ka6q0txsCflLJzZxUahI/OE9p83+1sqqetFiSRWO15iYW6FkOn9UGvteIGVJdqwYu344VnpV2iFasPbmlxTC0mYBIdZBIxEXXGd9CyUsKB4oiCALtxTRdHi669rB62WxsulXYDIJFVETe928QPtiUobkVMOLVVz0IYN21uxeRcO3CobXZ8PqlsBp5lMr3EUG/Z+WSY+sM1dkYdKMzgVF8Wi+M2JBqrk3i6+kkc6qrSHOD5LVVQSmXCLt+2rjrwvpxKaUeMjBMceFjVshxpfb1JJ+d2k460VpQxNTnLDtwrEZuJyTXCBr5zMGyqssWGukL+j25pYx5YCE0WVb4GNbkY8Ly70zIJBxFPy7EkuR4m2Go35lrN4LrwZH7p7wbNCaJpt5DgHYEkLAISQyT05XlDrFueNd8tj4v5C/iV/VzNrlmBBxgWhoflXNxezL9hpdTfBZhmYT7rmbRYkkKBG8IZp2+1gbwHKbmJ8FzaMMVds9LDnF4qgBFCgehLMrjimHtXOIcjM8BNiU7NtX4oamgymcTNAieQUzJnxeeQey2NdJGYPf/hx9n181FyQ+Cj1NXQ6ewNpJVgajJLr/DIbF/AhhUsRb0JTHfgB2NJxwZLu84qg4VjeEScnW7iuM/eniK56Jy1/N05gMeRqPJi7jXDRVLiTARlOOhZIldrBewtWQOef2jQuvYWW3jNY3sJPj46PT3jheMf0Bnnpsa4KVkPiOFKyKoR2cUMeJoZ63zx9BSdHyZcaSCyNDxngoTaRlVwH97LB+S9XDw9eSTihu0PuF/M8UQFRUqWAvGHVcIqYQa7CnbUXCpirkvBbls+rTX7KMDbHslljJaEImY9nWoEY3ULQnhJyZR9orsvBwlbpUFV14qVLqT2eLdfdtPBXqbvZs2mAG+vMUbftSUWv1T9HTtFWRl84SCVhAndF9l1YjIlqzQj7eOgkMUBkLktBxjne+xgwn05uebvUL44Lo1bE6xIK0wqZqFZiPCaZhTG9OomUPKEFFaZW3Z8dMZi+/y7QxRveqAc295WvLm6LVFvb69mUxv2NxZeVLLS/UhupqB68RpDdlEheLFkhY9NI/Th8bJ32W7AuTh5wfPXfXuu7lpZf5tBBA4AHCH7LhCwHaHUSTM+JNWJvt9Hq5OOExuBuKhCXmasKaM71adAcEj2VESXsLHa7gBt8+wa6e5koKNnXStYy/h2dycvcXbGOgMwl+fNULIPcrw72P4gGoydRX5e7Wwm2NFEsDUI7TuWegPus9VoOu7DXx/ef/zt4+84m5rlcPTY6dpTbY/TPVB3KreG3aunhWqMbQnMAD5ilh4RNyOtGWL7Zl2fsD0f9ri0h+2AHdrrHjpMVNBOG20Uhmo+8z9o7JGhISspL3l8Ttv/ntOh8Yd3Z7dy0CEdBOJhI5MIycy8J8G23U+9wXbMF48aJyhSwXX2BUK1/JxQLSeFanm/UO3pLnt3+MwI/vP+73///OOeAQxXPFNLKSawtX8FsL8l6OB63xtatDfW6MjfIS7/A1BLBwieO1oxpgUAAMgYAABQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAgAAABtZXRhLnhtbI1TwY7bIBS89yssdq82BtvEQY5X6qGnrVqpqdRb5MBbl5ZAhPE6/fva2M7utjnsjTfMMG8eUD1cTjp6Btcpa3aIJCmKwAgrlWl36Pv+U1yih/pDZZ+elAAurehPYHx8At9Eo9R0vHVS6h366f2ZYzwMQzJkiXUtpmma4RbLxjfxs4LhDi2KSbxDvTPcNp3quGlO0HEvuD2DWS34C5eHtuZaiqvVuXc6GEmBQcMk6jBJCF65F63M71udke12i8PuSrXWXolTF3PeNUaO5/rKDtV7IyyzCyGW9auB56hepztlrauQWDho/MiIx+lBTVPCYpLGlO1JwVPCiyIpGSMpy7ZlhW8oKin4IqXFJCVkT0ZdxjOWsM2mpOWG5hVeabMrSOXHi49l78JZ9dc9+ZwX3xaL/7bfqsQfoaGr6T/sBZ65LRgYxdbVj+ro4EsIjmmR0CRPsvtHZfrL4UfJDiyPXjEOZ2d/gfC4oOn9x15pGWeLzcuJs8P1iXZ+bLLzSkQB981RQyxsb/w4djSDArReMbZg9jg5rWiKcF3hNxeEb32G+i9QSwcIF2vF1qIBAABKAwAAUEsDBBQACAgIALJWS1sAAAAAAAAAAAAAAAAMAAAAc2V0dGluZ3MueG1s7VrbcuI2GL7vU2S47WQx5NCFSbJjiE1IAgGfINzsCFuAiiy5kswhd+1r9EX6GLtvVNmGTEJwNmtwpt3ZXBAfpO//9es/y2efFj4+mEHGESXnhdIHpXAAiUs9RMbnBdvSDz8WPl38ckZHI+TCqkfd0IdEHHIohBzCD+R0wqsuJSMkJ4SMVCngiFcJ8CGvCrdKA0jW06pPR1djYsmTBUZkel6YCBFUi8X5fP5hfvSBsnGxVKlUivHb9VBK6ePACDvhLB5cVpTjYnL/ODq+eytjq1XGjK2un4jmuHCxlsN6+Rdnq7Uk/w6RgH4km4PV44jYeUGyXJ0hOH+UWmHbvOdzHMTREEOVQWDRoLB+KZaBfImIKFwoZ8WXIN8FfAtHIh/kHvLEZBv08UnpdGf0K4jGk62cl45PTt4Kf+iD4BARDy6gt0kKzrfvUTxH6gtbvoVhOG96G1xywaQCFC4idSh9F6cR6AafFpAC+Rajz6d8+fvLP1///PpX6Q0qWA8Zp6xDORLSAvpb5Z1tL58j329DLmdDVl2BZtAMMBIGIGO4Kf4JZdnB1wzv2W7WsEaaVu+Iu1//sUatUSGov0fgAaW+JVG2b9kOoA7A4SZqor1KVhmAMYzM+1X004zg5oTOGwxteo4hpRgCUrgQLIQZjYO4UpjQs+BC3MnANsJ0fgvHwF2m0RoBzDMSS4BN6e0wvEUE6pSIOWXTHEjVKSHQFZTZHJoEBIa8yYFMc0yk9GoyAE3VkYCsFWKBcLQ0BHHqdr1CcMvDpwEm7XUcCd7qCuMYkRKEHoNBNnlcUYYe5K4CbLqMYjwELDXulz4eV366hV3dQkQg1r8Og1EGkYOSR2QGkNGYf75/HxTht6nIE1oNhVSOfPB1yvwQA94CbJq6hB3Fn4/zj1DrFL8QTKLy5dOjcvkkY2r+TDVz2NgrwCXnoU8MOr+CwJMlWS5EzAmEQjrMHNCb/C4UUawwl/6QYm7C1Ai1C5HYaq9kHomjXFI6eY1E/j9LcPo2sSjUWtQAXMbCHAgkwHJNSeGXGwUDcqlc6SWOkjUcbMJvrXN2hTfDoYdmiO+5QnsBvp35rKqTwKsLxM2lTEoZJeghXUuzW8TKXdcA29ozWCURGZfxM53+IdLpVQto+wAOxdt7fMmDkIHI2r+n2adiqTodmZSLazqsA+JCnEP1FwR4KbeUXQIBcoAPBa0D7EpzE6nOOjt8fQIYcKWa1KkfMMgjn7T3skDzh9BTOQJEVjQoEJGR5ZAMxGSidWC4yJ/Qq9g72G8Mfit1/R2EdUdi1c1tLTkmxz9C8urL5F6WnCPIZLi77DT32PVr8svVGYwpLTynTPUGMhKbdSckrghj/5wLoXdJ8t8reco9w5d5zNQOPBkvWtRL8eRHGaEp8AxpbVQ6jhw4j2M1ZG15k9Jbu+p8vgUy2F5D8blVUiqHrVIp67FDQqwDAsh0Rn2pVuFmR3+Pi9oKL5dyelxDBLCldIjzq1+LyqTbVxYTt+wsTd9R+oqyaF1OhWVpi9ZY/U/9dUtO134ImrZead3329idBtjTK+2WpZ7c2pJvq7ke2lPVlhqs+e+qmqpOTHmlefLH9CvIaOjKvaku6qS2BL0TZdBvVky9NrIa+sOgf30z6BsjKYP5nVmS+Nqy7lfkOCf05DzQq4QdpzZzibG872Gl7rdnbgNj90FZRNfDxmLiNezKwNe5W7Zvhj19OSjj8L5n/OH128owwrlU5reXKr9+ULnk9PdheTFzfQO7VwbtWE0lwaksBw2nDHpt3LHUG1uvad1yRfLgHHuaIWloN3ZDJwOnHUDfPu1aXVWtNdWuUmnbmm73FcOxtYXekzKyFEPX5jXN69f4oD9pD8sGvvs9cIa+CIdHXnip1rr2FN85mmP2lZOaZduOqr2mG1w19OtrZ/o4Hqha27H11ulrcvzZQ/jZQ3gKboIZtCahPyQA5VAsRfBO8unHHaljyvPQnf93Q/k92u05nhTk2bDO/RBlSQRYmHGqYcDR9szpt2zYSUIW5RkWlBXqnor5tE9YktKxHj9aNU7y+55FJpmvZGxxAZXijXY7qU1tLBVffE5WTPvQ7uJfUEsHCGG5cxAbBgAAqicAAFBLAwQUAAAIAACyVktblt0F1bQKAAC0CgAAGAAAAFRodW1ibmFpbHMvdGh1bWJuYWlsLnBuZ4lQTkcNChoKAAAADUlIRFIAAAFVAAAB0QgDAAAA7fPa0gAAAwBQTFRFAQECAgUKDAUBDQgGCwkKBwwVDREaEAYAEgsFEgwLGQwDFA8RGhELEhQZHBYTHRwcAQ8gDRYjExomFB8zGSEqFSE0GyQyGSY5Gyg5IhYNJBwWKCEbMyMWMyUaNikdOSQTOCYZPCgXOykaKSkpJCo0KjA1NS0mNDMyNzo9ODc2OTg3Pj4+Lz9XMTtDLkJXO0JKNUVVOU1jPlNpQS0dQS8jRjQkTDwwVD0qREE9TEA1SEQ/UkIzUUc8VEg7WkUzXUg2YUw3ZFA+aFI9Q0JDSkZCT0hBQUtVQEtaS1RdWk5FV1lcRFZnQFRrR1tuT1pmSF12VFtkTGJ6XGRrUGh+X2l1XGt7aVRBZVxTc1xIbmBSd2BLd2dXa2tsYWt1ZW55ZnJ/b3Bzcm5rf3Rpd3V0VGqCW3CGYXGCYXaNbXaBbHiEaX2TbISbfYOJc4WUcIadc4mef4iTdIuie5One5awgGpUh3BagXFihHlujHlnjn5tg355knxlkH9wjIJ5mIJslIR0oIt1rJR9sJh/ioiHg4uTjZKXl4yCko+NnI+Dl5CJlZWViJuth5+1lJyljKC2m6Krl6m6m6i1pJeHpZiKq5aArpiCrJyLpJ2YspyFs56IraOat6KPvKSNs6eavaaRvKiTvKuYpqOjpKaqpKirq6elrKijrauqoquyoq+6qKyzo7C7qrK6tKujtrCqu7KnvbWtlq3EnLPJn7nSp7fGpbzSt77GrMPXusLKtMTStcjavsfQuc3dutDfus/gvdHjw6qSx7Gcx7ajyLOgzbqly7ytxb620L6ow8G6y8O71MCr08W35dC4xcPEwsbJxMrNysfCzsnDzczMxs3TzNLY1M3G1NDM3dHD2tPK09TU09bZ09je29XQ2djW2drcwtXkxdnoytfkz9nizN3r1dzizuHx2eLq1+j35NfH4dfO5dnK6NvG69zM49vT7+DP6+HX+era6ejn4+z04/L96vD07PT89ezj8/Hs/fHl+/Tt//nr8vLz9Pb48/3++fXy/fz0/v7+up/nrgAAAAlwSFlzAAALEwAACxMBAJqcGAAAB1pJREFUeNrt231sldUdwPHbDkiJIPZSQBA1oZS6NJOhoiwGZ2EZTIlvc4M53GCvZLxsyIxC4gsyxpJFkc0qmws4lbHNbZqB4MaikkEGzD9sscI0zk0CbgNteaul3Pa3597bYvfHZttck1k+39De25bchA+n55znPG0qVPhSCKh+cFT3rMw9a/7pURyFU107JXlsi4N9XsdR8Bng4BlUC6i6bU7E9yqvefzMvxbwdbd968kJF6yKODS9qur+iO+uv6P8i423Vd6YzDIPT666+XCvV106OFYX/XzHxKI3Cvi6D6dufmFx8oojr979g6ItUd3v3ieHVyzfXrIgHuy3qv7yj/Z61QeGxWVfifhjqpCqP+l7IJoHbIi9mbYYuTA+Pi5i0oURM6bFqG9GvNX/2V4/VofF+cm/9GD/1yNTsNetGRpxYvhT8cPyqsmpW+KK5L/ts9dH3DQthpdVVlYWbTwdVOdG/Kuge4DVQzJxYsCmP6c2tMT5C/Oq1+ZVb2nc+1pLW29XrRkWV16UOBR0BlidG6ub1qQz8WZqUU51Rn6sXpY8HH800+vH6uBkTr3hyxP6FnIPsDqdqA747VvF18y+bmJ6f3VCWZ2M1RnjYlfJdcvO6/2r1Z5fROyaPa9xXSG3O4fWJ+/W7Y+XZs87fHzFvh2bk81W8rbj6eRLS2auzPR6Vb1vqu/b4GlN3lr+81Ntp42qqFKlqg+i6qHH4uWVHR/8ejfVLrY4/d+vJVpiTTrWTIk4mfszaiHVLtX8scn9N/2Pr69Nd9rBXUK1a7298p0B7aq7PlU1dXc0zHpheuWcTDR/qfLG7bPiuXT2jLz5tqqqOcfikvl3VE7dT7ULNbSrvtl//iu3lh5pKBn7xG+KF8VNpZt/PDKdea40e+pw6+i67ed+PcaX3btj1EVUu1B+rLbFt7PnJ+ctOlH8VO5YZfh9EbefFYnqPUNj0rjkkqsxLv50xNIhVLs8VjNR3bey4sPFc0/0+UOien1bycaI5/tlkhlg6bD455iBY1dm4uJkXq2h2jXVjdn1Pa6Y1vjqK02Zhv5bIiblVdf2y43VYclQfnFJyVdze4DsLQOqXZgBfpebAe48O3lYd7jhzC25mwDDFyS4Z2VXq3vOjof2Ze8VxPiF+cNtqu/RrgmfSF0weUNW99yPLJ9YeqCh5NmEc1rUpG64amJZbrUaEjPS85aM+Eb2xks8MDhD9T2vnZateGj5srrs06YVM79zNFofPxzxp80Re5bVbU1n3n40/rE+4lczZz2Rv7bKHW5T7WF3fSFZtC50DtDTMrnFKrLH1a0dn2uNl0eMHjP6b6f+UltkqBai+t2ZMFZF9bRVbaWq/1/VptpOHzTWUS1IO8s6rfs7B+6nWhDV0Z13U5cv6NjKUu1mL/0sf4+v+dLy8soRReUVFed8JtYOKu9o0Hyq3a3tyn5jP/S17LOGczbU/+XBsroXX7t9bDTuTZ7V1tbWj5z76j6q3e3u0sOxM5X96fSG8gPtM8CPcpf/l96XvNtWdswM0P327M6esG48pTo00xarx2bvWT9fdiRi/Fzzao9qiZr0sZzqL5MZYFBd7b7F+Z8Arr467hyaodqTTsYjfZ7JzasDsitUUXl5Rcm43I9WHh9zVfkWO6uedVffZ3KnfO/MPNK+X901L6cdd6emJcvZSardb3G68xq/891v+ePVpY+NGv13Y7UHbR34Rufz6I6rgNb6z/eZsj+aPpea+vRRqt1tRiqZTM9Y1THH5q9Y22Jr8Sd/n98jTO/zfardrbE+2evXnvpdmKb2A5WWd387pukk1cJddLU/niZbK+erVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmqK/0b2Ut5cTw/GlMAAAAASUVORK5CYIJQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAABUAAABNRVRBLUlORi9tYW5pZmVzdC54bWytlE1uwjAQhfecIvK2ik2rLiqLwKJST0AP4NrjYMkZW/5BcPs6EYFUFYi07DyZyffeGytZbQ6drfYQonHYkGe6JBWgdMpg25DP7Uf9RjbrxaoTaDTExMdDVd7DeC4bkgNyJ6KJHEUHkSfJnQdUTuYOMPGf83xQOlcTA6/khLYODiM3tHwEaZdRiVSmT0Jw8BBM3xKWO62NBD4hDErrRXWJoI2FuoyH48WAztbWXqRdQ9hVX5clgDKiTkcPDRHeWyMHQ2yPig47oNPoNPoAQsUdQCJsjpV3h9q0OQz0+MKElGChlC4wmUPo4SXqFWf/07ozbszYW6DZUDklzBOP6Wgh3siSyk2yvj0LOz6jQek78pSpp9kaJXW6fQ9/9A5JPBwaIaXyZT9+09td7r5QGBtZGo/UY3tFxHSiBdb3i8qK/fq7rL8BUEsHCL+IH6hFAQAAmAQAAFBLAQIUABQAAAgAALJWS1uFbDmKLgAAAC4AAAAIAAAAAAAAAAAAAAAAAAAAAABtaW1ldHlwZVBLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAAAAAAAAAAAAAAFQAAABDb25maWd1cmF0aW9uczIvcG9wdXBtZW51L1BLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAIwAAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAADCAAAAQ29uZmlndXJhdGlvbnMyL3Byb2dyZXNzYmFyL1BLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAPwAAABDb25maWd1cmF0aW9uczIvZmxvYXRlci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAGgAAAAAAAAAAAAAAAAAyAQAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAABqAQAAQ29uZmlndXJhdGlvbnMyL3Rvb2xiYXIvUEsBAhQAFAAACAAAslZLWwAAAAAAAAAAAAAAABoAAAAAAAAAAAAAAAAAoAEAAENvbmZpZ3VyYXRpb25zMi90b29scGFuZWwvUEsBAhQAFAAICAgAslZLWwAAAAACAAAAAAAAACcAAAAAAAAAAAAAAAAA2AEAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbFBLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAfAAAAAAAAAAAAAAAAAC8CAABDb25maWd1cmF0aW9uczIvaW1hZ2VzL0JpdG1hcHMvUEsBAhQAFAAICAgAslZLW2PvHlh3CgAAJD4AAAoAAAAAAAAAAAAAAAAAbAIAAHN0eWxlcy54bWxQSwECFAAUAAgICACyVktbtPdo0gUBAACDAwAADAAAAAAAAAAAAAAAAAAbDQAAbWFuaWZlc3QucmRmUEsBAhQAFAAICAgAslZLW547WjGmBQAAyBgAAAsAAAAAAAAAAAAAAAAAWg4AAGNvbnRlbnQueG1sUEsBAhQAFAAICAgAslZLWxdrxdaiAQAASgMAAAgAAAAAAAAAAAAAAAAAORQAAG1ldGEueG1sUEsBAhQAFAAICAgAslZLW2G5cxAbBgAAqicAAAwAAAAAAAAAAAAAAAAAERYAAHNldHRpbmdzLnhtbFBLAQIUABQAAAgAALJWS1uW3QXVtAoAALQKAAAYAAAAAAAAAAAAAAAAAGYcAABUaHVtYm5haWxzL3RodW1ibmFpbC5wbmdQSwECFAAUAAgICACyVktbv4gfqEUBAACYBAAAFQAAAAAAAAAAAAAAAABQJwAATUVUQS1JTkYvbWFuaWZlc3QueG1sUEsFBgAAAAARABEAcAQAANgoAAAAAA==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "11614" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:53 GMT" - }, - { - "name": "etag", - "value": "W/\"a2a96f688ebe63c7cda5df94e3848c593229592f0016d4dfce728db6d8ed183a\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:53 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "e793bec42bcef2568f19106dbb2ca8e97698a372" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "82FD:3314BB:816C59:9A7653:68EA3E0B" - }, - { - "name": "x-served-by", - "value": "cache-lis1490029-LIS" - }, - { - "name": "x-timer", - "value": "S1760181773.066688,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 876, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:53.114Z", - "time": 63, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 63 - } - }, - { - "_id": "9d5062962d85f533a00d57678f1e9995", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 112, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/ods/table/fixtures/table.ods" - }, - "response": { - "bodySize": 11614, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 11614, - "text": "UEsDBBQAAAgAALJWS1uFbDmKLgAAAC4AAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlvbi92bmQub2FzaXMub3BlbmRvY3VtZW50LnNwcmVhZHNoZWV0UEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3BvcHVwbWVudS9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABwAAABDb25maWd1cmF0aW9uczIvcHJvZ3Jlc3NiYXIvUEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAQ29uZmlndXJhdGlvbnMyL2Zsb2F0ZXIvUEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvdG9vbGJhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABoAAABDb25maWd1cmF0aW9uczIvdG9vbHBhbmVsL1BLAwQUAAgICACyVktbAAAAAAAAAAAAAAAAJwAAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbAMAUEsHCAAAAAACAAAAAAAAAFBLAwQUAAAIAACyVktbAAAAAAAAAAAAAAAAHwAAAENvbmZpZ3VyYXRpb25zMi9pbWFnZXMvQml0bWFwcy9QSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAoAAABzdHlsZXMueG1s5Rtrc9u48Xt/hUaZ3tzNlOZDsmPpYmXuck3ba5LeJGn78QYmIQoNSHBA0LLz8X5Pf1V/SRcvvknRcuQkrexxQuwudrEv7ILQs+e3CZ3dYJ4Tll7N/TNvPsNpyCKSxlfzv79/6VzOn29+94xttyTE64iFRYJT4eTijuJ8BsRpvs44zmEQCTVHwdM1QznJ1ylKcL4W4ZplOLWk6y7NWrHV42GeL8TVfCdEtnbd/X5/tl+cMR6779+6EuYIfCtcix3zKKJ92IHnLdzYjZBAzg3B+yeW4nYnkl4Kf7VauQpqUSOWDEztu4Dh4BtYQG6x85CTTExdvsauL3zLeDKVWuLWaRMkdgNrunRfA1D9ef3K4mtrTuVmbF/jxxgr2UkCjWG1s3T1c7WyqZxuc+psmROyJAPHuKZtpvtRrntOBOaloSlJPwwbWkJLQ3O0H12S77kSpyZJOCpJiGhYTl6hZgWnCikKXUyxXHju+md+6c6SDtzbqIvHZbxtWZFGOla0/vBthjmRIEQV2boxQ11vKlQn+6VErlOLSp6DxKLFmW2nuxjQBSVPBKYf97GVq5DqFpzKS+JCdqsLGvFFNJ18EdVpeSZG5Dx3Oc4YF6UxbuLJpriJByIh3CE+2SgKuWFRqbjJJkUt3gkWaCqxxK3TUnaEc5vsU5uhPmVaJNeYT7YdbAcdD98STK31S/v1SsKYk+QOSSHLsGxdo9bTGcraZrqcb+zOuWWwa25RiJ0IhzTfPNNylMMz/SzZXs1/wv9C/yhm71AK2wv4gUVLCL27mn+DMpZ/X8PRA/NZY0qJ68Q4BfkhA+V3ucBJAyUjIoRd4wZxokLJHZfpJcd4QKAa6JQSvCJgauUn45pp4U3Rzp7k+acSDabcTpJNIk4QjrMEpQ8R7g0TTGli9uLnv87evRgUro34KH7VZjp7i+OCIj5ZSkvwyNL+hG9QimKgmyBphfzJpHSHEosZ1xW6XU2Et6igpm63MxthVY53Qkzp3KJniKOYo2znZJDqMBcEin0NAmyYhWVORHKBUllGemfLVeCTtFKerAS6pErUgVjeMg3OyUcA+14m1BhFaVygGIaoHghhsxAcpH71ft6e1oENAKXt9FnhyLktjuaggZaJhX3cWYjhZgEv3nR5yi2a4tsDXEusXr4ldEfanEvQX94oq/eYc4qNlTFJWBrYPDdsBE6cC84+SHkog031yWJ5cY6W85ksmWCno7SEPA1W23CrLLKHqRyW6eYvZY58NiT5DkVs74BH5lg4t9JTfP9Sekof/K4O12LKkh5qNSdhEbhABrqqfGzMR6X7oUKwPEPSQUmEmUZFNNshO3tWpKEolBcqqSH0iNR2aR+SYueaYwR9BGiGhGJErrrJSRphWX3ITllNIuVQHfUW0RyXVrCxlFeW7VvVeFQVOXagoJWKVMyNiQQvQChdMDHJXkj9/r7jwH2bVysYg3sEI8UCyiPnA+apWrAW4yGBGjw8UE2GLWOplZ4P5uSKcig3Vxhljh7JAX1L+jQ5oB77tpZUaPOBhOAOU4J+8BDZrPTW8jTIafIbyTQ69lWqAY0ySiKTDxLEwXFgOtXf/MA52+8wivJfA+9XvwcH/D6SJx/emff08mmZVhooIZb1uo28KpU1OTeS2xY+oWcgMo04MUji7DCJd8K6dAtoJDFcIDYyFMlDNAfCWwm4WC2lgDXINRNCHjH1AineigEQ12JUsFouBf8jOVHuN5SBe6gaGdjAP8FefvqyvLdsGEg2I4VmRwRD8sCC004zmDQMfDhl9BQrPZXHWP81CelEaU5KXeUrk6d0x24ad6cn63jzFtLMPCUkVc13DHQRiYnIgYdi0DNnOUdYcMhR4V0fK3/h/eLZFdwwCt4pj0ZUgA8IAUU2SRB1MgqVBQgQzOvijUO7wlsopJAiq/ZLt2Qug28zK1eoHrsru0uuGbVzNXdpi1vbqTf/+e3f5YytSTY9gPuocz6UNVS1YNOrB5/2Ip3+Rf6faV8rL0FZWQKkEdGl9Q2iBf72u29i8f1V6bQoy+hdY/81Tu0OW/KelUJPe3hgS/hKM+4XlFQ7WXPIZn+G8gg8d8Rm0yu1imioo6IkTp2cFVw2VVtyayfn0O0g2XlAiKXCVkHjnRokhGrSq7ku04arjtZGf2FaEj0mKaDLEzBXWA3vTb0GgRXdW6Om5DSlNskhp9w5DYyZf5TWrcWmaJ0z/YLUgaQiV7jyjlpHML6O4ITrGDwfMGLvOEk/QI3sbEnXbw74QNAqioa6xqEObLIq34McJ4ywI3U0Tfa+Jm6S7GrRBwW/RuEHuX2mUbN7CiuXQzFLYVO+po7gcnEp7sAEgEvYoDokO8Yj2cx5Z0+XmZipDm725NKTP0f7VHnapT7TVfuSMZGeVr2fNHbsOq26BtJnPWiUwCakBuFlVBmM6SnqDqTTL+K/Lg3K+hWXdleIEAFqLdiq05wu9KKYwwJ5ejCA0eI0WaXvYMMo8q81Wf2Jsego2c2yj0lXYajT1el85eLCu8e+/QYXgiP62GqwWftUalit7qeGH9Gje4JUwClVEIa2752mgn8idYZ+UjV8EQv9I+eMP37cKyFPpoSt+gx0I/VNVAPsLjsELjfZ+3UzP4Sysfry9oNB9X12XY00fhrhyL7PWOIYV/W8R3HVIxQ10FkaRR3XWD5AUbXK9gtT1GJUUYvHVlSkPkcratrq3+J8/DjxHrln6CRA19rN1uSE9Xj1gv2hx0xaOcdFSFOx6o57HazO4UGQ2rtQ/bLPuE3va9WWX5Y4KtvdxGt5if5HJl8gzrxZ4M0Wnh6PruavfRijji8Hd4H3UfLW9w7EDifY3EHQE/9N3wxvIGiF5g28V+Sa4xayQmtgRbA4315yaGVLd5iMSoN16KrgHmMXHMmuQ3eYHVLx3ZHTv0QLb3GYsCvp4mKFJhAu2oRoscBTOC7bhJdANoXjeZswXK3CcaVqwosuYeD7eIxwVx06dA2JR0m3jFK2x5EzOMf5uY8ur1UW6Dp4c7C6wadit7q4J7NQggQJHQs4+Mo0eJy3lYOvWm2mirEDOYQVoiHe6yzx5z1I3fPu5rUqyh1xXe3LMh21WZqxLWPy0lFzg5BrsFc2vLNgdb4w9yggJcYAMzcsmoP2bkVztHZTY3VpLke4w1IZcT6HpPa2SVvMpkhuxxYTbBj8j9mwfp4cnC1X1Xlyrd42d24UmffUt2Q97aQnf0oV1TAgxGLccZjOcr8Oj/pSlXbYv93B9GoACcrLKcorl2ZQzjT2rrweET1JT4u/eaa+IJWZf/Mdxhp78/z582due9CMZC0ltMwvLdlsLToNjVUe4fkhVK3GUtBfipyijOQzMyBXqTP/xrfi1cY6EtvpGjaaILHBHJfY7VjokNHemq9ijdgs6NhMP3Ecy/ewUvb7mlEPuJtv9f8EEbSOqp+/6+iuwbExpOK0JQU0AXiwH4B9vURy1DUSCF4vOIeS3fHBSz3PVb+eZ6SQiJs/zKzAyeGpJZKd2vfX54t1sDhbLFeBt1penksWa/VbLrrPwZvr+2q83hrYrROoKz+b1apOoMc+f5S4/RnP7f+S9+a/UEsHCGPvHlh3CgAAJD4AAFBLAwQUAAgICACyVktbAAAAAAAAAAAAAAAADAAAAG1hbmlmZXN0LnJkZs2TzW6DMBCE7zyFZc7YQC8FBXIoyrlqn8A1hlgFL/KaEt6+jpNWUaSq6p/U465GM9+OtJvtYRzIi7KowVQ0YyklykhotekrOrsuuaXbOtrYtisfmh3xaoOlnyq6d24qOV+WhS03DGzPs6IoeJrzPE+8IsHVOHFIDMa0jggJHo1CafXkfBo5zuIJZldRdOugkHn3ID2L3TqpoLIKYbZSvYe2IJGBQI0JTMqEdIMcuk5LxTOW81E5waHt4sdgvdODojxg8CuOz9jeiAym5V7gvbDuXIPffJVoeu5jenXTxfHfI5RgnDLuT+q7O3n/5/4uz/8Z4q+0dkRsQM6jZ/qQ57TyH1VHr1BLBwi092jSBQEAAIMDAABQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAsAAABjb250ZW50LnhtbL1ZzXLbNhC+9yk06kxvMCw5ndiq7RySyUxbJ4fY7fQKkUsSDUhwAFCUc2svfYNe+gy95zGSd+j0OboACQqkRYVuHF+UCPvt7of9AyCfP9vmYrYBpbksLuaLo+P5DIpIxrxIL+Y/3bwkp/Nnl1+dyyThEaxiGVU5FIZEsjD47wy1C70qFWj8xowzUqliJZnmelWwHPTKRCtZQuF1V3d1Vs5vsx5pfWIu5pkx5YrSuq6P6pMjqVJ684ZaGTGwNdSjUxXHYh96eXx8QlMaM8PIhkP9tdfYZibfq7E4OzujTtpBNR8xvaC/vLq6jjLIGeGFNqyIYKcVf1qrAydS5XoEv6SN2INjmY9aRgSBDQa0Q+tI8dJMTUeDDhNhXU/VtthQN2cmG4nxKX2FQvfx6srjm/Ka6q0txsCflLJzZxUahI/OE9p83+1sqqetFiSRWO15iYW6FkOn9UGvteIGVJdqwYu344VnpV2iFasPbmlxTC0mYBIdZBIxEXXGd9CyUsKB4oiCALtxTRdHi669rB62WxsulXYDIJFVETe928QPtiUobkVMOLVVz0IYN21uxeRcO3CobXZ8PqlsBp5lMr3EUG/Z+WSY+sM1dkYdKMzgVF8Wi+M2JBqrk3i6+kkc6qrSHOD5LVVQSmXCLt+2rjrwvpxKaUeMjBMceFjVshxpfb1JJ+d2k460VpQxNTnLDtwrEZuJyTXCBr5zMGyqssWGukL+j25pYx5YCE0WVb4GNbkY8Ly70zIJBxFPy7EkuR4m2Go35lrN4LrwZH7p7wbNCaJpt5DgHYEkLAISQyT05XlDrFueNd8tj4v5C/iV/VzNrlmBBxgWhoflXNxezL9hpdTfBZhmYT7rmbRYkkKBG8IZp2+1gbwHKbmJ8FzaMMVds9LDnF4qgBFCgehLMrjimHtXOIcjM8BNiU7NtX4oamgymcTNAieQUzJnxeeQey2NdJGYPf/hx9n181FyQ+Cj1NXQ6ewNpJVgajJLr/DIbF/AhhUsRb0JTHfgB2NJxwZLu84qg4VjeEScnW7iuM/eniK56Jy1/N05gMeRqPJi7jXDRVLiTARlOOhZIldrBewtWQOef2jQuvYWW3jNY3sJPj46PT3jheMf0Bnnpsa4KVkPiOFKyKoR2cUMeJoZ63zx9BSdHyZcaSCyNDxngoTaRlVwH97LB+S9XDw9eSTihu0PuF/M8UQFRUqWAvGHVcIqYQa7CnbUXCpirkvBbls+rTX7KMDbHslljJaEImY9nWoEY3ULQnhJyZR9orsvBwlbpUFV14qVLqT2eLdfdtPBXqbvZs2mAG+vMUbftSUWv1T9HTtFWRl84SCVhAndF9l1YjIlqzQj7eOgkMUBkLktBxjne+xgwn05uebvUL44Lo1bE6xIK0wqZqFZiPCaZhTG9OomUPKEFFaZW3Z8dMZi+/y7QxRveqAc295WvLm6LVFvb69mUxv2NxZeVLLS/UhupqB68RpDdlEheLFkhY9NI/Th8bJ32W7AuTh5wfPXfXuu7lpZf5tBBA4AHCH7LhCwHaHUSTM+JNWJvt9Hq5OOExuBuKhCXmasKaM71adAcEj2VESXsLHa7gBt8+wa6e5koKNnXStYy/h2dycvcXbGOgMwl+fNULIPcrw72P4gGoydRX5e7Wwm2NFEsDUI7TuWegPus9VoOu7DXx/ef/zt4+84m5rlcPTY6dpTbY/TPVB3KreG3aunhWqMbQnMAD5ilh4RNyOtGWL7Zl2fsD0f9ri0h+2AHdrrHjpMVNBOG20Uhmo+8z9o7JGhISspL3l8Ttv/ntOh8Yd3Z7dy0CEdBOJhI5MIycy8J8G23U+9wXbMF48aJyhSwXX2BUK1/JxQLSeFanm/UO3pLnt3+MwI/vP+73///OOeAQxXPFNLKSawtX8FsL8l6OB63xtatDfW6MjfIS7/A1BLBwieO1oxpgUAAMgYAABQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAgAAABtZXRhLnhtbI1TwY7bIBS89yssdq82BtvEQY5X6qGnrVqpqdRb5MBbl5ZAhPE6/fva2M7utjnsjTfMMG8eUD1cTjp6Btcpa3aIJCmKwAgrlWl36Pv+U1yih/pDZZ+elAAurehPYHx8At9Eo9R0vHVS6h366f2ZYzwMQzJkiXUtpmma4RbLxjfxs4LhDi2KSbxDvTPcNp3quGlO0HEvuD2DWS34C5eHtuZaiqvVuXc6GEmBQcMk6jBJCF65F63M71udke12i8PuSrXWXolTF3PeNUaO5/rKDtV7IyyzCyGW9auB56hepztlrauQWDho/MiIx+lBTVPCYpLGlO1JwVPCiyIpGSMpy7ZlhW8oKin4IqXFJCVkT0ZdxjOWsM2mpOWG5hVeabMrSOXHi49l78JZ9dc9+ZwX3xaL/7bfqsQfoaGr6T/sBZ65LRgYxdbVj+ro4EsIjmmR0CRPsvtHZfrL4UfJDiyPXjEOZ2d/gfC4oOn9x15pGWeLzcuJs8P1iXZ+bLLzSkQB981RQyxsb/w4djSDArReMbZg9jg5rWiKcF3hNxeEb32G+i9QSwcIF2vF1qIBAABKAwAAUEsDBBQACAgIALJWS1sAAAAAAAAAAAAAAAAMAAAAc2V0dGluZ3MueG1s7VrbcuI2GL7vU2S47WQx5NCFSbJjiE1IAgGfINzsCFuAiiy5kswhd+1r9EX6GLtvVNmGTEJwNmtwpt3ZXBAfpO//9es/y2efFj4+mEHGESXnhdIHpXAAiUs9RMbnBdvSDz8WPl38ckZHI+TCqkfd0IdEHHIohBzCD+R0wqsuJSMkJ4SMVCngiFcJ8CGvCrdKA0jW06pPR1djYsmTBUZkel6YCBFUi8X5fP5hfvSBsnGxVKlUivHb9VBK6ePACDvhLB5cVpTjYnL/ODq+eytjq1XGjK2un4jmuHCxlsN6+Rdnq7Uk/w6RgH4km4PV44jYeUGyXJ0hOH+UWmHbvOdzHMTREEOVQWDRoLB+KZaBfImIKFwoZ8WXIN8FfAtHIh/kHvLEZBv08UnpdGf0K4jGk62cl45PTt4Kf+iD4BARDy6gt0kKzrfvUTxH6gtbvoVhOG96G1xywaQCFC4idSh9F6cR6AafFpAC+Rajz6d8+fvLP1///PpX6Q0qWA8Zp6xDORLSAvpb5Z1tL58j329DLmdDVl2BZtAMMBIGIGO4Kf4JZdnB1wzv2W7WsEaaVu+Iu1//sUatUSGov0fgAaW+JVG2b9kOoA7A4SZqor1KVhmAMYzM+1X004zg5oTOGwxteo4hpRgCUrgQLIQZjYO4UpjQs+BC3MnANsJ0fgvHwF2m0RoBzDMSS4BN6e0wvEUE6pSIOWXTHEjVKSHQFZTZHJoEBIa8yYFMc0yk9GoyAE3VkYCsFWKBcLQ0BHHqdr1CcMvDpwEm7XUcCd7qCuMYkRKEHoNBNnlcUYYe5K4CbLqMYjwELDXulz4eV366hV3dQkQg1r8Og1EGkYOSR2QGkNGYf75/HxTht6nIE1oNhVSOfPB1yvwQA94CbJq6hB3Fn4/zj1DrFL8QTKLy5dOjcvkkY2r+TDVz2NgrwCXnoU8MOr+CwJMlWS5EzAmEQjrMHNCb/C4UUawwl/6QYm7C1Ai1C5HYaq9kHomjXFI6eY1E/j9LcPo2sSjUWtQAXMbCHAgkwHJNSeGXGwUDcqlc6SWOkjUcbMJvrXN2hTfDoYdmiO+5QnsBvp35rKqTwKsLxM2lTEoZJeghXUuzW8TKXdcA29ozWCURGZfxM53+IdLpVQto+wAOxdt7fMmDkIHI2r+n2adiqTodmZSLazqsA+JCnEP1FwR4KbeUXQIBcoAPBa0D7EpzE6nOOjt8fQIYcKWa1KkfMMgjn7T3skDzh9BTOQJEVjQoEJGR5ZAMxGSidWC4yJ/Qq9g72G8Mfit1/R2EdUdi1c1tLTkmxz9C8urL5F6WnCPIZLi77DT32PVr8svVGYwpLTynTPUGMhKbdSckrghj/5wLoXdJ8t8reco9w5d5zNQOPBkvWtRL8eRHGaEp8AxpbVQ6jhw4j2M1ZG15k9Jbu+p8vgUy2F5D8blVUiqHrVIp67FDQqwDAsh0Rn2pVuFmR3+Pi9oKL5dyelxDBLCldIjzq1+LyqTbVxYTt+wsTd9R+oqyaF1OhWVpi9ZY/U/9dUtO134ImrZead3329idBtjTK+2WpZ7c2pJvq7ke2lPVlhqs+e+qmqpOTHmlefLH9CvIaOjKvaku6qS2BL0TZdBvVky9NrIa+sOgf30z6BsjKYP5nVmS+Nqy7lfkOCf05DzQq4QdpzZzibG872Gl7rdnbgNj90FZRNfDxmLiNezKwNe5W7Zvhj19OSjj8L5n/OH128owwrlU5reXKr9+ULnk9PdheTFzfQO7VwbtWE0lwaksBw2nDHpt3LHUG1uvad1yRfLgHHuaIWloN3ZDJwOnHUDfPu1aXVWtNdWuUmnbmm73FcOxtYXekzKyFEPX5jXN69f4oD9pD8sGvvs9cIa+CIdHXnip1rr2FN85mmP2lZOaZduOqr2mG1w19OtrZ/o4Hqha27H11ulrcvzZQ/jZQ3gKboIZtCahPyQA5VAsRfBO8unHHaljyvPQnf93Q/k92u05nhTk2bDO/RBlSQRYmHGqYcDR9szpt2zYSUIW5RkWlBXqnor5tE9YktKxHj9aNU7y+55FJpmvZGxxAZXijXY7qU1tLBVffE5WTPvQ7uJfUEsHCGG5cxAbBgAAqicAAFBLAwQUAAAIAACyVktblt0F1bQKAAC0CgAAGAAAAFRodW1ibmFpbHMvdGh1bWJuYWlsLnBuZ4lQTkcNChoKAAAADUlIRFIAAAFVAAAB0QgDAAAA7fPa0gAAAwBQTFRFAQECAgUKDAUBDQgGCwkKBwwVDREaEAYAEgsFEgwLGQwDFA8RGhELEhQZHBYTHRwcAQ8gDRYjExomFB8zGSEqFSE0GyQyGSY5Gyg5IhYNJBwWKCEbMyMWMyUaNikdOSQTOCYZPCgXOykaKSkpJCo0KjA1NS0mNDMyNzo9ODc2OTg3Pj4+Lz9XMTtDLkJXO0JKNUVVOU1jPlNpQS0dQS8jRjQkTDwwVD0qREE9TEA1SEQ/UkIzUUc8VEg7WkUzXUg2YUw3ZFA+aFI9Q0JDSkZCT0hBQUtVQEtaS1RdWk5FV1lcRFZnQFRrR1tuT1pmSF12VFtkTGJ6XGRrUGh+X2l1XGt7aVRBZVxTc1xIbmBSd2BLd2dXa2tsYWt1ZW55ZnJ/b3Bzcm5rf3Rpd3V0VGqCW3CGYXGCYXaNbXaBbHiEaX2TbISbfYOJc4WUcIadc4mef4iTdIuie5One5awgGpUh3BagXFihHlujHlnjn5tg355knxlkH9wjIJ5mIJslIR0oIt1rJR9sJh/ioiHg4uTjZKXl4yCko+NnI+Dl5CJlZWViJuth5+1lJyljKC2m6Krl6m6m6i1pJeHpZiKq5aArpiCrJyLpJ2YspyFs56IraOat6KPvKSNs6eavaaRvKiTvKuYpqOjpKaqpKirq6elrKijrauqoquyoq+6qKyzo7C7qrK6tKujtrCqu7KnvbWtlq3EnLPJn7nSp7fGpbzSt77GrMPXusLKtMTStcjavsfQuc3dutDfus/gvdHjw6qSx7Gcx7ajyLOgzbqly7ytxb620L6ow8G6y8O71MCr08W35dC4xcPEwsbJxMrNysfCzsnDzczMxs3TzNLY1M3G1NDM3dHD2tPK09TU09bZ09je29XQ2djW2drcwtXkxdnoytfkz9nizN3r1dzizuHx2eLq1+j35NfH4dfO5dnK6NvG69zM49vT7+DP6+HX+era6ejn4+z04/L96vD07PT89ezj8/Hs/fHl+/Tt//nr8vLz9Pb48/3++fXy/fz0/v7+up/nrgAAAAlwSFlzAAALEwAACxMBAJqcGAAAB1pJREFUeNrt231sldUdwPHbDkiJIPZSQBA1oZS6NJOhoiwGZ2EZTIlvc4M53GCvZLxsyIxC4gsyxpJFkc0qmws4lbHNbZqB4MaikkEGzD9sscI0zk0CbgNteaul3Pa3597bYvfHZttck1k+39De25bchA+n55znPG0qVPhSCKh+cFT3rMw9a/7pURyFU107JXlsi4N9XsdR8Bng4BlUC6i6bU7E9yqvefzMvxbwdbd968kJF6yKODS9qur+iO+uv6P8i423Vd6YzDIPT666+XCvV106OFYX/XzHxKI3Cvi6D6dufmFx8oojr979g6ItUd3v3ieHVyzfXrIgHuy3qv7yj/Z61QeGxWVfifhjqpCqP+l7IJoHbIi9mbYYuTA+Pi5i0oURM6bFqG9GvNX/2V4/VofF+cm/9GD/1yNTsNetGRpxYvhT8cPyqsmpW+KK5L/ts9dH3DQthpdVVlYWbTwdVOdG/Kuge4DVQzJxYsCmP6c2tMT5C/Oq1+ZVb2nc+1pLW29XrRkWV16UOBR0BlidG6ub1qQz8WZqUU51Rn6sXpY8HH800+vH6uBkTr3hyxP6FnIPsDqdqA747VvF18y+bmJ6f3VCWZ2M1RnjYlfJdcvO6/2r1Z5fROyaPa9xXSG3O4fWJ+/W7Y+XZs87fHzFvh2bk81W8rbj6eRLS2auzPR6Vb1vqu/b4GlN3lr+81Ntp42qqFKlqg+i6qHH4uWVHR/8ejfVLrY4/d+vJVpiTTrWTIk4mfszaiHVLtX8scn9N/2Pr69Nd9rBXUK1a7298p0B7aq7PlU1dXc0zHpheuWcTDR/qfLG7bPiuXT2jLz5tqqqOcfikvl3VE7dT7ULNbSrvtl//iu3lh5pKBn7xG+KF8VNpZt/PDKdea40e+pw6+i67ed+PcaX3btj1EVUu1B+rLbFt7PnJ+ctOlH8VO5YZfh9EbefFYnqPUNj0rjkkqsxLv50xNIhVLs8VjNR3bey4sPFc0/0+UOien1bycaI5/tlkhlg6bD455iBY1dm4uJkXq2h2jXVjdn1Pa6Y1vjqK02Zhv5bIiblVdf2y43VYclQfnFJyVdze4DsLQOqXZgBfpebAe48O3lYd7jhzC25mwDDFyS4Z2VXq3vOjof2Ze8VxPiF+cNtqu/RrgmfSF0weUNW99yPLJ9YeqCh5NmEc1rUpG64amJZbrUaEjPS85aM+Eb2xks8MDhD9T2vnZateGj5srrs06YVM79zNFofPxzxp80Re5bVbU1n3n40/rE+4lczZz2Rv7bKHW5T7WF3fSFZtC50DtDTMrnFKrLH1a0dn2uNl0eMHjP6b6f+UltkqBai+t2ZMFZF9bRVbaWq/1/VptpOHzTWUS1IO8s6rfs7B+6nWhDV0Z13U5cv6NjKUu1mL/0sf4+v+dLy8soRReUVFed8JtYOKu9o0Hyq3a3tyn5jP/S17LOGczbU/+XBsroXX7t9bDTuTZ7V1tbWj5z76j6q3e3u0sOxM5X96fSG8gPtM8CPcpf/l96XvNtWdswM0P327M6esG48pTo00xarx2bvWT9fdiRi/Fzzao9qiZr0sZzqL5MZYFBd7b7F+Z8Arr467hyaodqTTsYjfZ7JzasDsitUUXl5Rcm43I9WHh9zVfkWO6uedVffZ3KnfO/MPNK+X901L6cdd6emJcvZSardb3G68xq/891v+ePVpY+NGv13Y7UHbR34Rufz6I6rgNb6z/eZsj+aPpea+vRRqt1tRiqZTM9Y1THH5q9Y22Jr8Sd/n98jTO/zfardrbE+2evXnvpdmKb2A5WWd387pukk1cJddLU/niZbK+erVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmqK/0b2Ut5cTw/GlMAAAAASUVORK5CYIJQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAABUAAABNRVRBLUlORi9tYW5pZmVzdC54bWytlE1uwjAQhfecIvK2ik2rLiqLwKJST0AP4NrjYMkZW/5BcPs6EYFUFYi07DyZyffeGytZbQ6drfYQonHYkGe6JBWgdMpg25DP7Uf9RjbrxaoTaDTExMdDVd7DeC4bkgNyJ6KJHEUHkSfJnQdUTuYOMPGf83xQOlcTA6/khLYODiM3tHwEaZdRiVSmT0Jw8BBM3xKWO62NBD4hDErrRXWJoI2FuoyH48WAztbWXqRdQ9hVX5clgDKiTkcPDRHeWyMHQ2yPig47oNPoNPoAQsUdQCJsjpV3h9q0OQz0+MKElGChlC4wmUPo4SXqFWf/07ozbszYW6DZUDklzBOP6Wgh3siSyk2yvj0LOz6jQek78pSpp9kaJXW6fQ9/9A5JPBwaIaXyZT9+09td7r5QGBtZGo/UY3tFxHSiBdb3i8qK/fq7rL8BUEsHCL+IH6hFAQAAmAQAAFBLAQIUABQAAAgAALJWS1uFbDmKLgAAAC4AAAAIAAAAAAAAAAAAAAAAAAAAAABtaW1ldHlwZVBLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAAAAAAAAAAAAAAFQAAABDb25maWd1cmF0aW9uczIvcG9wdXBtZW51L1BLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAIwAAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAADCAAAAQ29uZmlndXJhdGlvbnMyL3Byb2dyZXNzYmFyL1BLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAPwAAABDb25maWd1cmF0aW9uczIvZmxvYXRlci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAGgAAAAAAAAAAAAAAAAAyAQAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAABqAQAAQ29uZmlndXJhdGlvbnMyL3Rvb2xiYXIvUEsBAhQAFAAACAAAslZLWwAAAAAAAAAAAAAAABoAAAAAAAAAAAAAAAAAoAEAAENvbmZpZ3VyYXRpb25zMi90b29scGFuZWwvUEsBAhQAFAAICAgAslZLWwAAAAACAAAAAAAAACcAAAAAAAAAAAAAAAAA2AEAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbFBLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAfAAAAAAAAAAAAAAAAAC8CAABDb25maWd1cmF0aW9uczIvaW1hZ2VzL0JpdG1hcHMvUEsBAhQAFAAICAgAslZLW2PvHlh3CgAAJD4AAAoAAAAAAAAAAAAAAAAAbAIAAHN0eWxlcy54bWxQSwECFAAUAAgICACyVktbtPdo0gUBAACDAwAADAAAAAAAAAAAAAAAAAAbDQAAbWFuaWZlc3QucmRmUEsBAhQAFAAICAgAslZLW547WjGmBQAAyBgAAAsAAAAAAAAAAAAAAAAAWg4AAGNvbnRlbnQueG1sUEsBAhQAFAAICAgAslZLWxdrxdaiAQAASgMAAAgAAAAAAAAAAAAAAAAAORQAAG1ldGEueG1sUEsBAhQAFAAICAgAslZLW2G5cxAbBgAAqicAAAwAAAAAAAAAAAAAAAAAERYAAHNldHRpbmdzLnhtbFBLAQIUABQAAAgAALJWS1uW3QXVtAoAALQKAAAYAAAAAAAAAAAAAAAAAGYcAABUaHVtYm5haWxzL3RodW1ibmFpbC5wbmdQSwECFAAUAAgICACyVktbv4gfqEUBAACYBAAAFQAAAAAAAAAAAAAAAABQJwAATUVUQS1JTkYvbWFuaWZlc3QueG1sUEsFBgAAAAARABEAcAQAANgoAAAAAA==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "11614" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:53 GMT" - }, - { - "name": "etag", - "value": "W/\"a2a96f688ebe63c7cda5df94e3848c593229592f0016d4dfce728db6d8ed183a\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:53 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "7ac95697ee18e1f267cb2857b8fc4f36e8889eb7" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "82FD:3314BB:816C59:9A7653:68EA3E0B" - }, - { - "name": "x-served-by", - "value": "cache-lis1490044-LIS" - }, - { - "name": "x-timer", - "value": "S1760181773.186937,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 876, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:53.114Z", - "time": 172, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 172 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/ods/table/fixtures/generated/loadOdsTable-file-variations-should-load-remote-file_2229433724/recording.har b/table/plugins/ods/table/fixtures/generated/loadOdsTable-file-variations-should-load-remote-file_2229433724/recording.har deleted file mode 100644 index 2d0ed71b..00000000 --- a/table/plugins/ods/table/fixtures/generated/loadOdsTable-file-variations-should-load-remote-file_2229433724/recording.har +++ /dev/null @@ -1,153 +0,0 @@ -{ - "log": { - "_recordingName": "loadOdsTable-file variations-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "9d5062962d85f533a00d57678f1e9995", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 112, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/ods/table/fixtures/table.ods" - }, - "response": { - "bodySize": 11614, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 11614, - "text": "UEsDBBQAAAgAALJWS1uFbDmKLgAAAC4AAAAIAAAAbWltZXR5cGVhcHBsaWNhdGlvbi92bmQub2FzaXMub3BlbmRvY3VtZW50LnNwcmVhZHNoZWV0UEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3BvcHVwbWVudS9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABwAAABDb25maWd1cmF0aW9uczIvcHJvZ3Jlc3NiYXIvUEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAQ29uZmlndXJhdGlvbnMyL2Zsb2F0ZXIvUEsDBBQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABgAAABDb25maWd1cmF0aW9uczIvdG9vbGJhci9QSwMEFAAACAAAslZLWwAAAAAAAAAAAAAAABoAAABDb25maWd1cmF0aW9uczIvdG9vbHBhbmVsL1BLAwQUAAgICACyVktbAAAAAAAAAAAAAAAAJwAAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbAMAUEsHCAAAAAACAAAAAAAAAFBLAwQUAAAIAACyVktbAAAAAAAAAAAAAAAAHwAAAENvbmZpZ3VyYXRpb25zMi9pbWFnZXMvQml0bWFwcy9QSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAoAAABzdHlsZXMueG1s5Rtrc9u48Xt/hUaZ3tzNlOZDsmPpYmXuck3ba5LeJGn78QYmIQoNSHBA0LLz8X5Pf1V/SRcvvknRcuQkrexxQuwudrEv7ILQs+e3CZ3dYJ4Tll7N/TNvPsNpyCKSxlfzv79/6VzOn29+94xttyTE64iFRYJT4eTijuJ8BsRpvs44zmEQCTVHwdM1QznJ1ylKcL4W4ZplOLWk6y7NWrHV42GeL8TVfCdEtnbd/X5/tl+cMR6779+6EuYIfCtcix3zKKJ92IHnLdzYjZBAzg3B+yeW4nYnkl4Kf7VauQpqUSOWDEztu4Dh4BtYQG6x85CTTExdvsauL3zLeDKVWuLWaRMkdgNrunRfA1D9ef3K4mtrTuVmbF/jxxgr2UkCjWG1s3T1c7WyqZxuc+psmROyJAPHuKZtpvtRrntOBOaloSlJPwwbWkJLQ3O0H12S77kSpyZJOCpJiGhYTl6hZgWnCikKXUyxXHju+md+6c6SDtzbqIvHZbxtWZFGOla0/vBthjmRIEQV2boxQ11vKlQn+6VErlOLSp6DxKLFmW2nuxjQBSVPBKYf97GVq5DqFpzKS+JCdqsLGvFFNJ18EdVpeSZG5Dx3Oc4YF6UxbuLJpriJByIh3CE+2SgKuWFRqbjJJkUt3gkWaCqxxK3TUnaEc5vsU5uhPmVaJNeYT7YdbAcdD98STK31S/v1SsKYk+QOSSHLsGxdo9bTGcraZrqcb+zOuWWwa25RiJ0IhzTfPNNylMMz/SzZXs1/wv9C/yhm71AK2wv4gUVLCL27mn+DMpZ/X8PRA/NZY0qJ68Q4BfkhA+V3ucBJAyUjIoRd4wZxokLJHZfpJcd4QKAa6JQSvCJgauUn45pp4U3Rzp7k+acSDabcTpJNIk4QjrMEpQ8R7g0TTGli9uLnv87evRgUro34KH7VZjp7i+OCIj5ZSkvwyNL+hG9QimKgmyBphfzJpHSHEosZ1xW6XU2Et6igpm63MxthVY53Qkzp3KJniKOYo2znZJDqMBcEin0NAmyYhWVORHKBUllGemfLVeCTtFKerAS6pErUgVjeMg3OyUcA+14m1BhFaVygGIaoHghhsxAcpH71ft6e1oENAKXt9FnhyLktjuaggZaJhX3cWYjhZgEv3nR5yi2a4tsDXEusXr4ldEfanEvQX94oq/eYc4qNlTFJWBrYPDdsBE6cC84+SHkog031yWJ5cY6W85ksmWCno7SEPA1W23CrLLKHqRyW6eYvZY58NiT5DkVs74BH5lg4t9JTfP9Sekof/K4O12LKkh5qNSdhEbhABrqqfGzMR6X7oUKwPEPSQUmEmUZFNNshO3tWpKEolBcqqSH0iNR2aR+SYueaYwR9BGiGhGJErrrJSRphWX3ITllNIuVQHfUW0RyXVrCxlFeW7VvVeFQVOXagoJWKVMyNiQQvQChdMDHJXkj9/r7jwH2bVysYg3sEI8UCyiPnA+apWrAW4yGBGjw8UE2GLWOplZ4P5uSKcig3Vxhljh7JAX1L+jQ5oB77tpZUaPOBhOAOU4J+8BDZrPTW8jTIafIbyTQ69lWqAY0ySiKTDxLEwXFgOtXf/MA52+8wivJfA+9XvwcH/D6SJx/emff08mmZVhooIZb1uo28KpU1OTeS2xY+oWcgMo04MUji7DCJd8K6dAtoJDFcIDYyFMlDNAfCWwm4WC2lgDXINRNCHjH1AineigEQ12JUsFouBf8jOVHuN5SBe6gaGdjAP8FefvqyvLdsGEg2I4VmRwRD8sCC004zmDQMfDhl9BQrPZXHWP81CelEaU5KXeUrk6d0x24ad6cn63jzFtLMPCUkVc13DHQRiYnIgYdi0DNnOUdYcMhR4V0fK3/h/eLZFdwwCt4pj0ZUgA8IAUU2SRB1MgqVBQgQzOvijUO7wlsopJAiq/ZLt2Qug28zK1eoHrsru0uuGbVzNXdpi1vbqTf/+e3f5YytSTY9gPuocz6UNVS1YNOrB5/2Ip3+Rf6faV8rL0FZWQKkEdGl9Q2iBf72u29i8f1V6bQoy+hdY/81Tu0OW/KelUJPe3hgS/hKM+4XlFQ7WXPIZn+G8gg8d8Rm0yu1imioo6IkTp2cFVw2VVtyayfn0O0g2XlAiKXCVkHjnRokhGrSq7ku04arjtZGf2FaEj0mKaDLEzBXWA3vTb0GgRXdW6Om5DSlNskhp9w5DYyZf5TWrcWmaJ0z/YLUgaQiV7jyjlpHML6O4ITrGDwfMGLvOEk/QI3sbEnXbw74QNAqioa6xqEObLIq34McJ4ywI3U0Tfa+Jm6S7GrRBwW/RuEHuX2mUbN7CiuXQzFLYVO+po7gcnEp7sAEgEvYoDokO8Yj2cx5Z0+XmZipDm725NKTP0f7VHnapT7TVfuSMZGeVr2fNHbsOq26BtJnPWiUwCakBuFlVBmM6SnqDqTTL+K/Lg3K+hWXdleIEAFqLdiq05wu9KKYwwJ5ejCA0eI0WaXvYMMo8q81Wf2Jsego2c2yj0lXYajT1el85eLCu8e+/QYXgiP62GqwWftUalit7qeGH9Gje4JUwClVEIa2752mgn8idYZ+UjV8EQv9I+eMP37cKyFPpoSt+gx0I/VNVAPsLjsELjfZ+3UzP4Sysfry9oNB9X12XY00fhrhyL7PWOIYV/W8R3HVIxQ10FkaRR3XWD5AUbXK9gtT1GJUUYvHVlSkPkcratrq3+J8/DjxHrln6CRA19rN1uSE9Xj1gv2hx0xaOcdFSFOx6o57HazO4UGQ2rtQ/bLPuE3va9WWX5Y4KtvdxGt5if5HJl8gzrxZ4M0Wnh6PruavfRijji8Hd4H3UfLW9w7EDifY3EHQE/9N3wxvIGiF5g28V+Sa4xayQmtgRbA4315yaGVLd5iMSoN16KrgHmMXHMmuQ3eYHVLx3ZHTv0QLb3GYsCvp4mKFJhAu2oRoscBTOC7bhJdANoXjeZswXK3CcaVqwosuYeD7eIxwVx06dA2JR0m3jFK2x5EzOMf5uY8ur1UW6Dp4c7C6wadit7q4J7NQggQJHQs4+Mo0eJy3lYOvWm2mirEDOYQVoiHe6yzx5z1I3fPu5rUqyh1xXe3LMh21WZqxLWPy0lFzg5BrsFc2vLNgdb4w9yggJcYAMzcsmoP2bkVztHZTY3VpLke4w1IZcT6HpPa2SVvMpkhuxxYTbBj8j9mwfp4cnC1X1Xlyrd42d24UmffUt2Q97aQnf0oV1TAgxGLccZjOcr8Oj/pSlXbYv93B9GoACcrLKcorl2ZQzjT2rrweET1JT4u/eaa+IJWZf/Mdxhp78/z582due9CMZC0ltMwvLdlsLToNjVUe4fkhVK3GUtBfipyijOQzMyBXqTP/xrfi1cY6EtvpGjaaILHBHJfY7VjokNHemq9ijdgs6NhMP3Ecy/ewUvb7mlEPuJtv9f8EEbSOqp+/6+iuwbExpOK0JQU0AXiwH4B9vURy1DUSCF4vOIeS3fHBSz3PVb+eZ6SQiJs/zKzAyeGpJZKd2vfX54t1sDhbLFeBt1penksWa/VbLrrPwZvr+2q83hrYrROoKz+b1apOoMc+f5S4/RnP7f+S9+a/UEsHCGPvHlh3CgAAJD4AAFBLAwQUAAgICACyVktbAAAAAAAAAAAAAAAADAAAAG1hbmlmZXN0LnJkZs2TzW6DMBCE7zyFZc7YQC8FBXIoyrlqn8A1hlgFL/KaEt6+jpNWUaSq6p/U465GM9+OtJvtYRzIi7KowVQ0YyklykhotekrOrsuuaXbOtrYtisfmh3xaoOlnyq6d24qOV+WhS03DGzPs6IoeJrzPE+8IsHVOHFIDMa0jggJHo1CafXkfBo5zuIJZldRdOugkHn3ID2L3TqpoLIKYbZSvYe2IJGBQI0JTMqEdIMcuk5LxTOW81E5waHt4sdgvdODojxg8CuOz9jeiAym5V7gvbDuXIPffJVoeu5jenXTxfHfI5RgnDLuT+q7O3n/5/4uz/8Z4q+0dkRsQM6jZ/qQ57TyH1VHr1BLBwi092jSBQEAAIMDAABQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAsAAABjb250ZW50LnhtbL1ZzXLbNhC+9yk06kxvMCw5ndiq7RySyUxbJ4fY7fQKkUsSDUhwAFCUc2svfYNe+gy95zGSd+j0OboACQqkRYVuHF+UCPvt7of9AyCfP9vmYrYBpbksLuaLo+P5DIpIxrxIL+Y/3bwkp/Nnl1+dyyThEaxiGVU5FIZEsjD47wy1C70qFWj8xowzUqliJZnmelWwHPTKRCtZQuF1V3d1Vs5vsx5pfWIu5pkx5YrSuq6P6pMjqVJ684ZaGTGwNdSjUxXHYh96eXx8QlMaM8PIhkP9tdfYZibfq7E4OzujTtpBNR8xvaC/vLq6jjLIGeGFNqyIYKcVf1qrAydS5XoEv6SN2INjmY9aRgSBDQa0Q+tI8dJMTUeDDhNhXU/VtthQN2cmG4nxKX2FQvfx6srjm/Ka6q0txsCflLJzZxUahI/OE9p83+1sqqetFiSRWO15iYW6FkOn9UGvteIGVJdqwYu344VnpV2iFasPbmlxTC0mYBIdZBIxEXXGd9CyUsKB4oiCALtxTRdHi669rB62WxsulXYDIJFVETe928QPtiUobkVMOLVVz0IYN21uxeRcO3CobXZ8PqlsBp5lMr3EUG/Z+WSY+sM1dkYdKMzgVF8Wi+M2JBqrk3i6+kkc6qrSHOD5LVVQSmXCLt+2rjrwvpxKaUeMjBMceFjVshxpfb1JJ+d2k460VpQxNTnLDtwrEZuJyTXCBr5zMGyqssWGukL+j25pYx5YCE0WVb4GNbkY8Ly70zIJBxFPy7EkuR4m2Go35lrN4LrwZH7p7wbNCaJpt5DgHYEkLAISQyT05XlDrFueNd8tj4v5C/iV/VzNrlmBBxgWhoflXNxezL9hpdTfBZhmYT7rmbRYkkKBG8IZp2+1gbwHKbmJ8FzaMMVds9LDnF4qgBFCgehLMrjimHtXOIcjM8BNiU7NtX4oamgymcTNAieQUzJnxeeQey2NdJGYPf/hx9n181FyQ+Cj1NXQ6ewNpJVgajJLr/DIbF/AhhUsRb0JTHfgB2NJxwZLu84qg4VjeEScnW7iuM/eniK56Jy1/N05gMeRqPJi7jXDRVLiTARlOOhZIldrBewtWQOef2jQuvYWW3jNY3sJPj46PT3jheMf0Bnnpsa4KVkPiOFKyKoR2cUMeJoZ63zx9BSdHyZcaSCyNDxngoTaRlVwH97LB+S9XDw9eSTihu0PuF/M8UQFRUqWAvGHVcIqYQa7CnbUXCpirkvBbls+rTX7KMDbHslljJaEImY9nWoEY3ULQnhJyZR9orsvBwlbpUFV14qVLqT2eLdfdtPBXqbvZs2mAG+vMUbftSUWv1T9HTtFWRl84SCVhAndF9l1YjIlqzQj7eOgkMUBkLktBxjne+xgwn05uebvUL44Lo1bE6xIK0wqZqFZiPCaZhTG9OomUPKEFFaZW3Z8dMZi+/y7QxRveqAc295WvLm6LVFvb69mUxv2NxZeVLLS/UhupqB68RpDdlEheLFkhY9NI/Th8bJ32W7AuTh5wfPXfXuu7lpZf5tBBA4AHCH7LhCwHaHUSTM+JNWJvt9Hq5OOExuBuKhCXmasKaM71adAcEj2VESXsLHa7gBt8+wa6e5koKNnXStYy/h2dycvcXbGOgMwl+fNULIPcrw72P4gGoydRX5e7Wwm2NFEsDUI7TuWegPus9VoOu7DXx/ef/zt4+84m5rlcPTY6dpTbY/TPVB3KreG3aunhWqMbQnMAD5ilh4RNyOtGWL7Zl2fsD0f9ri0h+2AHdrrHjpMVNBOG20Uhmo+8z9o7JGhISspL3l8Ttv/ntOh8Yd3Z7dy0CEdBOJhI5MIycy8J8G23U+9wXbMF48aJyhSwXX2BUK1/JxQLSeFanm/UO3pLnt3+MwI/vP+73///OOeAQxXPFNLKSawtX8FsL8l6OB63xtatDfW6MjfIS7/A1BLBwieO1oxpgUAAMgYAABQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAAAgAAABtZXRhLnhtbI1TwY7bIBS89yssdq82BtvEQY5X6qGnrVqpqdRb5MBbl5ZAhPE6/fva2M7utjnsjTfMMG8eUD1cTjp6Btcpa3aIJCmKwAgrlWl36Pv+U1yih/pDZZ+elAAurehPYHx8At9Eo9R0vHVS6h366f2ZYzwMQzJkiXUtpmma4RbLxjfxs4LhDi2KSbxDvTPcNp3quGlO0HEvuD2DWS34C5eHtuZaiqvVuXc6GEmBQcMk6jBJCF65F63M71udke12i8PuSrXWXolTF3PeNUaO5/rKDtV7IyyzCyGW9auB56hepztlrauQWDho/MiIx+lBTVPCYpLGlO1JwVPCiyIpGSMpy7ZlhW8oKin4IqXFJCVkT0ZdxjOWsM2mpOWG5hVeabMrSOXHi49l78JZ9dc9+ZwX3xaL/7bfqsQfoaGr6T/sBZ65LRgYxdbVj+ro4EsIjmmR0CRPsvtHZfrL4UfJDiyPXjEOZ2d/gfC4oOn9x15pGWeLzcuJs8P1iXZ+bLLzSkQB981RQyxsb/w4djSDArReMbZg9jg5rWiKcF3hNxeEb32G+i9QSwcIF2vF1qIBAABKAwAAUEsDBBQACAgIALJWS1sAAAAAAAAAAAAAAAAMAAAAc2V0dGluZ3MueG1s7VrbcuI2GL7vU2S47WQx5NCFSbJjiE1IAgGfINzsCFuAiiy5kswhd+1r9EX6GLtvVNmGTEJwNmtwpt3ZXBAfpO//9es/y2efFj4+mEHGESXnhdIHpXAAiUs9RMbnBdvSDz8WPl38ckZHI+TCqkfd0IdEHHIohBzCD+R0wqsuJSMkJ4SMVCngiFcJ8CGvCrdKA0jW06pPR1djYsmTBUZkel6YCBFUi8X5fP5hfvSBsnGxVKlUivHb9VBK6ePACDvhLB5cVpTjYnL/ODq+eytjq1XGjK2un4jmuHCxlsN6+Rdnq7Uk/w6RgH4km4PV44jYeUGyXJ0hOH+UWmHbvOdzHMTREEOVQWDRoLB+KZaBfImIKFwoZ8WXIN8FfAtHIh/kHvLEZBv08UnpdGf0K4jGk62cl45PTt4Kf+iD4BARDy6gt0kKzrfvUTxH6gtbvoVhOG96G1xywaQCFC4idSh9F6cR6AafFpAC+Rajz6d8+fvLP1///PpX6Q0qWA8Zp6xDORLSAvpb5Z1tL58j329DLmdDVl2BZtAMMBIGIGO4Kf4JZdnB1wzv2W7WsEaaVu+Iu1//sUatUSGov0fgAaW+JVG2b9kOoA7A4SZqor1KVhmAMYzM+1X004zg5oTOGwxteo4hpRgCUrgQLIQZjYO4UpjQs+BC3MnANsJ0fgvHwF2m0RoBzDMSS4BN6e0wvEUE6pSIOWXTHEjVKSHQFZTZHJoEBIa8yYFMc0yk9GoyAE3VkYCsFWKBcLQ0BHHqdr1CcMvDpwEm7XUcCd7qCuMYkRKEHoNBNnlcUYYe5K4CbLqMYjwELDXulz4eV366hV3dQkQg1r8Og1EGkYOSR2QGkNGYf75/HxTht6nIE1oNhVSOfPB1yvwQA94CbJq6hB3Fn4/zj1DrFL8QTKLy5dOjcvkkY2r+TDVz2NgrwCXnoU8MOr+CwJMlWS5EzAmEQjrMHNCb/C4UUawwl/6QYm7C1Ai1C5HYaq9kHomjXFI6eY1E/j9LcPo2sSjUWtQAXMbCHAgkwHJNSeGXGwUDcqlc6SWOkjUcbMJvrXN2hTfDoYdmiO+5QnsBvp35rKqTwKsLxM2lTEoZJeghXUuzW8TKXdcA29ozWCURGZfxM53+IdLpVQto+wAOxdt7fMmDkIHI2r+n2adiqTodmZSLazqsA+JCnEP1FwR4KbeUXQIBcoAPBa0D7EpzE6nOOjt8fQIYcKWa1KkfMMgjn7T3skDzh9BTOQJEVjQoEJGR5ZAMxGSidWC4yJ/Qq9g72G8Mfit1/R2EdUdi1c1tLTkmxz9C8urL5F6WnCPIZLi77DT32PVr8svVGYwpLTynTPUGMhKbdSckrghj/5wLoXdJ8t8reco9w5d5zNQOPBkvWtRL8eRHGaEp8AxpbVQ6jhw4j2M1ZG15k9Jbu+p8vgUy2F5D8blVUiqHrVIp67FDQqwDAsh0Rn2pVuFmR3+Pi9oKL5dyelxDBLCldIjzq1+LyqTbVxYTt+wsTd9R+oqyaF1OhWVpi9ZY/U/9dUtO134ImrZead3329idBtjTK+2WpZ7c2pJvq7ke2lPVlhqs+e+qmqpOTHmlefLH9CvIaOjKvaku6qS2BL0TZdBvVky9NrIa+sOgf30z6BsjKYP5nVmS+Nqy7lfkOCf05DzQq4QdpzZzibG872Gl7rdnbgNj90FZRNfDxmLiNezKwNe5W7Zvhj19OSjj8L5n/OH128owwrlU5reXKr9+ULnk9PdheTFzfQO7VwbtWE0lwaksBw2nDHpt3LHUG1uvad1yRfLgHHuaIWloN3ZDJwOnHUDfPu1aXVWtNdWuUmnbmm73FcOxtYXekzKyFEPX5jXN69f4oD9pD8sGvvs9cIa+CIdHXnip1rr2FN85mmP2lZOaZduOqr2mG1w19OtrZ/o4Hqha27H11ulrcvzZQ/jZQ3gKboIZtCahPyQA5VAsRfBO8unHHaljyvPQnf93Q/k92u05nhTk2bDO/RBlSQRYmHGqYcDR9szpt2zYSUIW5RkWlBXqnor5tE9YktKxHj9aNU7y+55FJpmvZGxxAZXijXY7qU1tLBVffE5WTPvQ7uJfUEsHCGG5cxAbBgAAqicAAFBLAwQUAAAIAACyVktblt0F1bQKAAC0CgAAGAAAAFRodW1ibmFpbHMvdGh1bWJuYWlsLnBuZ4lQTkcNChoKAAAADUlIRFIAAAFVAAAB0QgDAAAA7fPa0gAAAwBQTFRFAQECAgUKDAUBDQgGCwkKBwwVDREaEAYAEgsFEgwLGQwDFA8RGhELEhQZHBYTHRwcAQ8gDRYjExomFB8zGSEqFSE0GyQyGSY5Gyg5IhYNJBwWKCEbMyMWMyUaNikdOSQTOCYZPCgXOykaKSkpJCo0KjA1NS0mNDMyNzo9ODc2OTg3Pj4+Lz9XMTtDLkJXO0JKNUVVOU1jPlNpQS0dQS8jRjQkTDwwVD0qREE9TEA1SEQ/UkIzUUc8VEg7WkUzXUg2YUw3ZFA+aFI9Q0JDSkZCT0hBQUtVQEtaS1RdWk5FV1lcRFZnQFRrR1tuT1pmSF12VFtkTGJ6XGRrUGh+X2l1XGt7aVRBZVxTc1xIbmBSd2BLd2dXa2tsYWt1ZW55ZnJ/b3Bzcm5rf3Rpd3V0VGqCW3CGYXGCYXaNbXaBbHiEaX2TbISbfYOJc4WUcIadc4mef4iTdIuie5One5awgGpUh3BagXFihHlujHlnjn5tg355knxlkH9wjIJ5mIJslIR0oIt1rJR9sJh/ioiHg4uTjZKXl4yCko+NnI+Dl5CJlZWViJuth5+1lJyljKC2m6Krl6m6m6i1pJeHpZiKq5aArpiCrJyLpJ2YspyFs56IraOat6KPvKSNs6eavaaRvKiTvKuYpqOjpKaqpKirq6elrKijrauqoquyoq+6qKyzo7C7qrK6tKujtrCqu7KnvbWtlq3EnLPJn7nSp7fGpbzSt77GrMPXusLKtMTStcjavsfQuc3dutDfus/gvdHjw6qSx7Gcx7ajyLOgzbqly7ytxb620L6ow8G6y8O71MCr08W35dC4xcPEwsbJxMrNysfCzsnDzczMxs3TzNLY1M3G1NDM3dHD2tPK09TU09bZ09je29XQ2djW2drcwtXkxdnoytfkz9nizN3r1dzizuHx2eLq1+j35NfH4dfO5dnK6NvG69zM49vT7+DP6+HX+era6ejn4+z04/L96vD07PT89ezj8/Hs/fHl+/Tt//nr8vLz9Pb48/3++fXy/fz0/v7+up/nrgAAAAlwSFlzAAALEwAACxMBAJqcGAAAB1pJREFUeNrt231sldUdwPHbDkiJIPZSQBA1oZS6NJOhoiwGZ2EZTIlvc4M53GCvZLxsyIxC4gsyxpJFkc0qmws4lbHNbZqB4MaikkEGzD9sscI0zk0CbgNteaul3Pa3597bYvfHZttck1k+39De25bchA+n55znPG0qVPhSCKh+cFT3rMw9a/7pURyFU107JXlsi4N9XsdR8Bng4BlUC6i6bU7E9yqvefzMvxbwdbd968kJF6yKODS9qur+iO+uv6P8i423Vd6YzDIPT666+XCvV106OFYX/XzHxKI3Cvi6D6dufmFx8oojr979g6ItUd3v3ieHVyzfXrIgHuy3qv7yj/Z61QeGxWVfifhjqpCqP+l7IJoHbIi9mbYYuTA+Pi5i0oURM6bFqG9GvNX/2V4/VofF+cm/9GD/1yNTsNetGRpxYvhT8cPyqsmpW+KK5L/ts9dH3DQthpdVVlYWbTwdVOdG/Kuge4DVQzJxYsCmP6c2tMT5C/Oq1+ZVb2nc+1pLW29XrRkWV16UOBR0BlidG6ub1qQz8WZqUU51Rn6sXpY8HH800+vH6uBkTr3hyxP6FnIPsDqdqA747VvF18y+bmJ6f3VCWZ2M1RnjYlfJdcvO6/2r1Z5fROyaPa9xXSG3O4fWJ+/W7Y+XZs87fHzFvh2bk81W8rbj6eRLS2auzPR6Vb1vqu/b4GlN3lr+81Ntp42qqFKlqg+i6qHH4uWVHR/8ejfVLrY4/d+vJVpiTTrWTIk4mfszaiHVLtX8scn9N/2Pr69Nd9rBXUK1a7298p0B7aq7PlU1dXc0zHpheuWcTDR/qfLG7bPiuXT2jLz5tqqqOcfikvl3VE7dT7ULNbSrvtl//iu3lh5pKBn7xG+KF8VNpZt/PDKdea40e+pw6+i67ed+PcaX3btj1EVUu1B+rLbFt7PnJ+ctOlH8VO5YZfh9EbefFYnqPUNj0rjkkqsxLv50xNIhVLs8VjNR3bey4sPFc0/0+UOien1bycaI5/tlkhlg6bD455iBY1dm4uJkXq2h2jXVjdn1Pa6Y1vjqK02Zhv5bIiblVdf2y43VYclQfnFJyVdze4DsLQOqXZgBfpebAe48O3lYd7jhzC25mwDDFyS4Z2VXq3vOjof2Ze8VxPiF+cNtqu/RrgmfSF0weUNW99yPLJ9YeqCh5NmEc1rUpG64amJZbrUaEjPS85aM+Eb2xks8MDhD9T2vnZateGj5srrs06YVM79zNFofPxzxp80Re5bVbU1n3n40/rE+4lczZz2Rv7bKHW5T7WF3fSFZtC50DtDTMrnFKrLH1a0dn2uNl0eMHjP6b6f+UltkqBai+t2ZMFZF9bRVbaWq/1/VptpOHzTWUS1IO8s6rfs7B+6nWhDV0Z13U5cv6NjKUu1mL/0sf4+v+dLy8soRReUVFed8JtYOKu9o0Hyq3a3tyn5jP/S17LOGczbU/+XBsroXX7t9bDTuTZ7V1tbWj5z76j6q3e3u0sOxM5X96fSG8gPtM8CPcpf/l96XvNtWdswM0P327M6esG48pTo00xarx2bvWT9fdiRi/Fzzao9qiZr0sZzqL5MZYFBd7b7F+Z8Arr467hyaodqTTsYjfZ7JzasDsitUUXl5Rcm43I9WHh9zVfkWO6uedVffZ3KnfO/MPNK+X901L6cdd6emJcvZSardb3G68xq/891v+ePVpY+NGv13Y7UHbR34Rufz6I6rgNb6z/eZsj+aPpea+vRRqt1tRiqZTM9Y1THH5q9Y22Jr8Sd/n98jTO/zfardrbE+2evXnvpdmKb2A5WWd387pukk1cJddLU/niZbK+erVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlVRpUpVVKlSFVWqVEWVKlWqokqVqqhSpSqqVKmKKlWqokqVqqhSpSqqVKmqK/0b2Ut5cTw/GlMAAAAASUVORK5CYIJQSwMEFAAICAgAslZLWwAAAAAAAAAAAAAAABUAAABNRVRBLUlORi9tYW5pZmVzdC54bWytlE1uwjAQhfecIvK2ik2rLiqLwKJST0AP4NrjYMkZW/5BcPs6EYFUFYi07DyZyffeGytZbQ6drfYQonHYkGe6JBWgdMpg25DP7Uf9RjbrxaoTaDTExMdDVd7DeC4bkgNyJ6KJHEUHkSfJnQdUTuYOMPGf83xQOlcTA6/khLYODiM3tHwEaZdRiVSmT0Jw8BBM3xKWO62NBD4hDErrRXWJoI2FuoyH48WAztbWXqRdQ9hVX5clgDKiTkcPDRHeWyMHQ2yPig47oNPoNPoAQsUdQCJsjpV3h9q0OQz0+MKElGChlC4wmUPo4SXqFWf/07ozbszYW6DZUDklzBOP6Wgh3siSyk2yvj0LOz6jQek78pSpp9kaJXW6fQ9/9A5JPBwaIaXyZT9+09td7r5QGBtZGo/UY3tFxHSiBdb3i8qK/fq7rL8BUEsHCL+IH6hFAQAAmAQAAFBLAQIUABQAAAgAALJWS1uFbDmKLgAAAC4AAAAIAAAAAAAAAAAAAAAAAAAAAABtaW1ldHlwZVBLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAaAAAAAAAAAAAAAAAAAFQAAABDb25maWd1cmF0aW9uczIvcG9wdXBtZW51L1BLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAIwAAABDb25maWd1cmF0aW9uczIvbWVudWJhci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAADCAAAAQ29uZmlndXJhdGlvbnMyL3Byb2dyZXNzYmFyL1BLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAPwAAABDb25maWd1cmF0aW9uczIvZmxvYXRlci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAGgAAAAAAAAAAAAAAAAAyAQAAQ29uZmlndXJhdGlvbnMyL3N0YXR1c2Jhci9QSwECFAAUAAAIAACyVktbAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAABqAQAAQ29uZmlndXJhdGlvbnMyL3Rvb2xiYXIvUEsBAhQAFAAACAAAslZLWwAAAAAAAAAAAAAAABoAAAAAAAAAAAAAAAAAoAEAAENvbmZpZ3VyYXRpb25zMi90b29scGFuZWwvUEsBAhQAFAAICAgAslZLWwAAAAACAAAAAAAAACcAAAAAAAAAAAAAAAAA2AEAAENvbmZpZ3VyYXRpb25zMi9hY2NlbGVyYXRvci9jdXJyZW50LnhtbFBLAQIUABQAAAgAALJWS1sAAAAAAAAAAAAAAAAfAAAAAAAAAAAAAAAAAC8CAABDb25maWd1cmF0aW9uczIvaW1hZ2VzL0JpdG1hcHMvUEsBAhQAFAAICAgAslZLW2PvHlh3CgAAJD4AAAoAAAAAAAAAAAAAAAAAbAIAAHN0eWxlcy54bWxQSwECFAAUAAgICACyVktbtPdo0gUBAACDAwAADAAAAAAAAAAAAAAAAAAbDQAAbWFuaWZlc3QucmRmUEsBAhQAFAAICAgAslZLW547WjGmBQAAyBgAAAsAAAAAAAAAAAAAAAAAWg4AAGNvbnRlbnQueG1sUEsBAhQAFAAICAgAslZLWxdrxdaiAQAASgMAAAgAAAAAAAAAAAAAAAAAORQAAG1ldGEueG1sUEsBAhQAFAAICAgAslZLW2G5cxAbBgAAqicAAAwAAAAAAAAAAAAAAAAAERYAAHNldHRpbmdzLnhtbFBLAQIUABQAAAgAALJWS1uW3QXVtAoAALQKAAAYAAAAAAAAAAAAAAAAAGYcAABUaHVtYm5haWxzL3RodW1ibmFpbC5wbmdQSwECFAAUAAgICACyVktbv4gfqEUBAACYBAAAFQAAAAAAAAAAAAAAAABQJwAATUVUQS1JTkYvbWFuaWZlc3QueG1sUEsFBgAAAAARABEAcAQAANgoAAAAAA==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "11614" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:52 GMT" - }, - { - "name": "etag", - "value": "W/\"a2a96f688ebe63c7cda5df94e3848c593229592f0016d4dfce728db6d8ed183a\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:52 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "0e9f98a0dd00daedfb6400383dbfca9bd987d2ab" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "82FD:3314BB:816C59:9A7653:68EA3E0B" - }, - { - "name": "x-served-by", - "value": "cache-lis1490029-LIS" - }, - { - "name": "x-timer", - "value": "S1760181773.654481,VS0,VE186" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 879, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:52.593Z", - "time": 461, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 461 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/ods/table/index.ts b/table/plugins/ods/table/index.ts deleted file mode 100644 index 4159eaf3..00000000 --- a/table/plugins/ods/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadOdsTable } from "./load.ts" -export { saveOdsTable } from "./save.ts" diff --git a/table/plugins/ods/table/load.spec.ts b/table/plugins/ods/table/load.spec.ts deleted file mode 100644 index d7afcf55..00000000 --- a/table/plugins/ods/table/load.spec.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadOdsTable } from "./load.ts" -import { writeTestData } from "./test.ts" - -useRecording() - -const row1 = ["id", "name"] -const row2 = [1, "english"] -const row3 = [2, "中文"] - -const record1 = { id: 1, name: "english" } -const record2 = { id: 2, name: "中文" } - -describe("loadOdsTable", () => { - describe("file variations", () => { - it("should load local file", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadOdsTable({ path }) - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should load local file (multipart)", async () => { - const path1 = getTempFilePath() - const path2 = getTempFilePath() - await writeTestData(path1, [row1, row2, row3]) - await writeTestData(path2, [row1, row2, row3]) - - const table = await loadOdsTable({ path: [path1, path2] }) - expect((await table.collect()).toRecords()).toEqual([ - record1, - record2, - record1, - record2, - ]) - }) - - it("should load remote file", async () => { - const table = await loadOdsTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/ods/table/fixtures/table.ods", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load multipart remote file", async () => { - const table = await loadOdsTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/ods/table/fixtures/table.ods", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/ods/table/fixtures/table.ods", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) - - describe("dialect variations", () => { - it("should support sheetNumber", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3], { sheetNumber: 2 }) - - const table = await loadOdsTable({ - path, - dialect: { sheetNumber: 2 }, - }) - - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should support sheetName", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3], { sheetName: "Sheet2" }) - - const table = await loadOdsTable({ - path, - dialect: { sheetName: "Sheet2" }, - }) - - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should support no header", async () => { - const path = getTempFilePath() - await writeTestData(path, [row2, row3]) - - const table = await loadOdsTable({ - path, - dialect: { header: false }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { field1: 1, field2: "english" }, - { field1: 2, field2: "中文" }, - ]) - }) - - it("should support headerRows offset", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadOdsTable({ - path, - dialect: { headerRows: [2] }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { 1: 2, english: "中文" }, - ]) - }) - - it("should support multiline headerRows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadOdsTable({ - path, - dialect: { headerRows: [1, 2] }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { "id 1": 2, "name english": "中文" }, - ]) - }) - - it("should support headerJoin", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadOdsTable({ - path, - dialect: { headerRows: [1, 2], headerJoin: "-" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { "id-1": 2, "name-english": "中文" }, - ]) - }) - - it("should support commentRows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadOdsTable({ - path, - dialect: { commentRows: [2] }, - }) - - expect((await table.collect()).toRecords()).toEqual([record2]) - }) - - it("should support commentChar", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3, ["#comment"]]) - - const table = await loadOdsTable({ - path, - dialect: { commentChar: "#" }, - }) - - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should handle longer rows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3, [3, "german", "bad"]]) - - const table = await loadOdsTable({ - path, - dialect: { commentChar: "#" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - record1, - record2, - { id: 3, name: "german" }, - ]) - }) - - it("should handle shorter rows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3, [3]]) - - const table = await loadOdsTable({ - path, - dialect: { commentChar: "#" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - record1, - record2, - { id: 3, name: null }, - ]) - }) - }) -}) diff --git a/table/plugins/ods/table/load.ts b/table/plugins/ods/table/load.ts deleted file mode 100644 index 43f21fc9..00000000 --- a/table/plugins/ods/table/load.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { resolveDialect } from "@dpkit/metadata" -import type { Resource } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { loadFile, prefetchFiles } from "@dpkit/dataset" -import type { DataRow } from "../../../data/index.ts" -import { getRecordsFromRows } from "../../../data/index.ts" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" -import * as pl from "nodejs-polars" -import { read, utils } from "xlsx" - -export async function loadOdsTable( - resource: Partial, - options?: LoadTableOptions, -) { - const paths = await prefetchFiles(resource.path) - if (!paths.length) { - throw new Error("Resource path is not defined") - } - - const dialect = await resolveDialect(resource.dialect) - - const tables: Table[] = [] - for (const path of paths) { - const buffer = await loadFile(path) - - const book = read(buffer, { type: "buffer" }) - const sheetIndex = dialect?.sheetNumber ? dialect.sheetNumber - 1 : 0 - const sheetName = dialect?.sheetName ?? book.SheetNames[sheetIndex] - const sheet = sheetName ? book.Sheets[sheetName] : undefined - - if (sheet) { - const rows = utils.sheet_to_json(sheet, { - header: 1, - raw: true, - }) as DataRow[] - - const records = getRecordsFromRows(rows, dialect) - const table = pl.DataFrame(records).lazy() - - tables.push(table) - } - } - - let table = pl.concat(tables) - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} diff --git a/table/plugins/ods/table/save.spec.ts b/table/plugins/ods/table/save.spec.ts deleted file mode 100644 index ed1e9507..00000000 --- a/table/plugins/ods/table/save.spec.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { loadOdsTable } from "./load.ts" -import { saveOdsTable } from "./save.ts" -import { readTestData } from "./test.ts" - -const row1 = { id: 1, name: "english" } -const row2 = { id: 2, name: "中文" } -const table = pl.readRecords([row1, row2]).lazy() - -describe("saveOdsTable", () => { - it("should save table to file", async () => { - const path = getTempFilePath() - await saveOdsTable(table, { path }) - - const data = await readTestData(path) - expect(data).toEqual([row1, row2]) - }) - - it("should save and load various data types", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveOdsTable(source, { - path, - fieldTypes: { - array: "array", - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadOdsTable({ path }, { denormalized: true }) - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: true, - date: "2025-01-01", - datetime: "2025-01-01T00:00:00", - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: "1.0,2.0,3.0", - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) -}) diff --git a/table/plugins/ods/table/save.ts b/table/plugins/ods/table/save.ts deleted file mode 100644 index 56afcd1e..00000000 --- a/table/plugins/ods/table/save.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { saveFile } from "@dpkit/dataset" -import { resolveDialect } from "@dpkit/metadata" -import { utils, write } from "xlsx" -import type { SaveTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { denormalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" - -export async function saveOdsTable(table: Table, options: SaveTableOptions) { - const { path, overwrite } = options - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - table = await denormalizeTable(table, schema, { - nativeTypes: ["boolean", "integer", "number", "string", "year"], - }) - - const frame = await table.collect() - const dialect = await resolveDialect(options.dialect) - const sheetName = dialect?.sheetName ?? "Sheet1" - - const sheet = utils.json_to_sheet(frame.toRecords()) - const book = utils.book_new() - utils.book_append_sheet(book, sheet, sheetName) - - const buffer = write(book, { type: "buffer", bookType: "ods" }) - await saveFile(path, buffer, { overwrite }) - - return path -} diff --git a/table/plugins/ods/table/test.ts b/table/plugins/ods/table/test.ts deleted file mode 100644 index b70b0b94..00000000 --- a/table/plugins/ods/table/test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { readFile } from "node:fs/promises" -import { writeFile } from "node:fs/promises" -import { read, utils, write } from "xlsx" - -// We intentionally don't use dpkit's function here to isolate the tests - -export async function readTestData(path: string) { - const buffer = await readFile(path) - const book = read(buffer, { type: "buffer" }) - const sheetName = book.SheetNames[0] - const sheet = sheetName ? book.Sheets[sheetName] : undefined - return sheet ? utils.sheet_to_json(sheet) : [] -} - -export async function writeTestData( - path: string, - rows: unknown[][], - options?: { sheetNumber?: number; sheetName?: string }, -) { - const book = utils.book_new() - const sheet = utils.aoa_to_sheet(rows) - const sheetNumber = options?.sheetNumber ?? 1 - - for (let i = 0; i < sheetNumber; i++) { - const sheetName = options?.sheetName ?? `Sheet${i + 1}` - utils.book_append_sheet(book, sheet, sheetName) - } - - const buffer = write(book, { type: "buffer", bookType: "ods" }) - await writeFile(path, buffer) -} diff --git a/table/plugins/parquet/index.ts b/table/plugins/parquet/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/parquet/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/parquet/plugin.spec.ts b/table/plugins/parquet/plugin.spec.ts deleted file mode 100644 index b91e99a2..00000000 --- a/table/plugins/parquet/plugin.spec.ts +++ /dev/null @@ -1,188 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { ParquetPlugin } from "./plugin.ts" -import * as loadModule from "./table/load.ts" -import * as saveModule from "./table/save.ts" - -vi.mock("./table/load.ts", () => ({ - loadParquetTable: vi.fn(), -})) - -vi.mock("./table/save.ts", () => ({ - saveParquetTable: vi.fn(), -})) - -describe("ParquetPlugin", () => { - let plugin: ParquetPlugin - let mockLoadParquetTable: ReturnType - let mockSaveParquetTable: ReturnType - - beforeEach(() => { - plugin = new ParquetPlugin() - mockLoadParquetTable = vi.mocked(loadModule.loadParquetTable) - mockSaveParquetTable = vi.mocked(saveModule.saveParquetTable) - vi.clearAllMocks() - }) - - describe("loadTable", () => { - it("should load table from parquet file", async () => { - const resource: Partial = { - path: "test.parquet", - } - const mockTable = pl.DataFrame().lazy() - mockLoadParquetTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadParquetTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-parquet files", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "parquet", - } - const mockTable = pl.DataFrame().lazy() - mockLoadParquetTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadParquetTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should pass through load options", async () => { - const resource: Partial = { - path: "test.parquet", - } - const options = { denormalized: true } - const mockTable = pl.DataFrame().lazy() - mockLoadParquetTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource, options) - - expect(mockLoadParquetTable).toHaveBeenCalledWith(resource, options) - }) - - it("should handle paths with directories", async () => { - const resource: Partial = { - path: "/path/to/data.parquet", - } - const mockTable = pl.DataFrame().lazy() - mockLoadParquetTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource) - - expect(mockLoadParquetTable).toHaveBeenCalledWith(resource, undefined) - }) - - it("should return undefined for arrow files", async () => { - const resource: Partial = { - path: "test.arrow", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for json files", async () => { - const resource: Partial = { - path: "test.json", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("saveTable", () => { - it("should save table to parquet file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.parquet" } - mockSaveParquetTable.mockResolvedValue("output.parquet") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.parquet") - }) - - it("should return undefined for non-parquet files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "parquet" as const } - mockSaveParquetTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.txt") - }) - - it("should handle paths with directories", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "/path/to/output.parquet" } - mockSaveParquetTable.mockResolvedValue("/path/to/output.parquet") - - await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).toHaveBeenCalledWith(table, options) - }) - - it("should return undefined for files without extension", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for arrow files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.arrow" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for json files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.json" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveParquetTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/table/plugins/parquet/plugin.ts b/table/plugins/parquet/plugin.ts deleted file mode 100644 index 76bea279..00000000 --- a/table/plugins/parquet/plugin.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { - LoadTableOptions, - SaveTableOptions, - TablePlugin, -} from "../../plugin.ts" -import type { Table } from "../../table/index.ts" -import { loadParquetTable } from "./table/index.ts" -import { saveParquetTable } from "./table/index.ts" - -export class ParquetPlugin implements TablePlugin { - async loadTable(resource: Partial, options?: LoadTableOptions) { - const isParquet = getIsParquet(resource) - if (!isParquet) return undefined - - return await loadParquetTable(resource, options) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const isParquet = getIsParquet({ path, format }) - if (!isParquet) return undefined - - return await saveParquetTable(table, options) - } -} - -function getIsParquet(resource: Partial) { - const format = inferFormat(resource) - return format === "parquet" -} diff --git a/table/plugins/parquet/table/fixtures/generated/loadParquetTable-file-variations-should-load-remote-file-multipart_3893757127/recording.har b/table/plugins/parquet/table/fixtures/generated/loadParquetTable-file-variations-should-load-remote-file-multipart_3893757127/recording.har deleted file mode 100644 index 4e13cca7..00000000 --- a/table/plugins/parquet/table/fixtures/generated/loadParquetTable-file-variations-should-load-remote-file-multipart_3893757127/recording.har +++ /dev/null @@ -1,292 +0,0 @@ -{ - "log": { - "_recordingName": "loadParquetTable-file variations-should load remote file (multipart)", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "653d6a61a6eccb0e578d514271280781", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 120, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/parquet/table/fixtures/table.parquet" - }, - "response": { - "bodySize": 775, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 775, - "text": "UEFSMRUAFSwVLCwVBBUAFQYVBhw2ACgIAAAAAAAAAEAYCAAAAAAAAPA/AAAAAgAAAAQBAAAAAAAA8D8AAAAAAAAAQBUKGSUABhkYAmlkFQAWBBZ+Fn4mCDw2ACgIAAAAAAAAAEAYCAAAAAAAAPA/AAAVBBUqFSpMFQQVAAAABwAAAGVuZ2xpc2gGAAAA5Lit5paHFQAVEhUSLBUEFRAVBhUGHDYAKAbkuK3mlocYB2VuZ2xpc2gAAAACAAAABAEBAwIVDBk1AAYQGRgEbmFtZRUAFgQWogEWogEm4gE8NgAoBuS4reaWhxgHZW5nbGlzaAAAGRECGRgIAAAAAAAA8D8ZGAgAAAAAAAAAQBUAGRYAABkRAhkYB2VuZ2xpc2gZGAbkuK3mlocVABkWAAAZHBYIFX4WAAAAGRwWpgIVXhYAAAAVAhk8SARyb290FQQAFQolAhgCaWQAFQwlAhgEbmFtZSUATBwAAAAWBBkcGSwmhgEcFQoZJQAGGRgCaWQVABYEFn4WfiYIPDYAKAgAAAAAAAAAQBgIAAAAAAAA8D8AABbcBBUUFuYDFT4AJoQDHBUMGTUABhAZGARuYW1lFQAWBBaiARaiASbiATw2ACgG5Lit5paHGAdlbmdsaXNoAAAW8AQVFhakBBU4ABagAhYEJggWoAIUAAAZHBgMQVJST1c6c2NoZW1hGOABLy8vLy81OEFBQUFFQUFBQTh2Ly8veFFBQUFBRUFBRUFBQUFLQUFzQUNBQUtBQVFBK1AvLy93d0FBQUFJQUFnQUFBQUVBQUlBQUFBMEFBQUFCQUFBQU1ELy8vOGNBQUFBRUFBQUFBZ0FBQUFCRkFBQUFBQUFBUHovLy84RUFBUUFCQUFBQUc1aGJXVUFBQUFBN1AvLy96QUFBQUFnQUFBQUdBQUFBQUVEQUFBUUFCSUFCQUFRQUJFQUNBQUFBQXdBQUFBQUFQci8vLzhDQUFZQUJnQUVBQUlBQUFCcFpBQT0AGAZQb2xhcnMZLBwAABwAAAC8AQAAUEFSMQ==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "775" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:02 GMT" - }, - { - "name": "etag", - "value": "W/\"503f4f766508af8947dcae0e8215e8f7a678857a4123aff4efb8d4b167006bf1\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:02 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "69398502b96bd80419debc59968926b01be5da24" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "A851:1634AC:7D8FEE:9634F7:68EA3DDA" - }, - { - "name": "x-served-by", - "value": "cache-lis1490037-LIS" - }, - { - "name": "x-timer", - "value": "S1760181723.936964,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 874, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:02.976Z", - "time": 53, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 53 - } - }, - { - "_id": "653d6a61a6eccb0e578d514271280781", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 120, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/parquet/table/fixtures/table.parquet" - }, - "response": { - "bodySize": 775, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 775, - "text": "UEFSMRUAFSwVLCwVBBUAFQYVBhw2ACgIAAAAAAAAAEAYCAAAAAAAAPA/AAAAAgAAAAQBAAAAAAAA8D8AAAAAAAAAQBUKGSUABhkYAmlkFQAWBBZ+Fn4mCDw2ACgIAAAAAAAAAEAYCAAAAAAAAPA/AAAVBBUqFSpMFQQVAAAABwAAAGVuZ2xpc2gGAAAA5Lit5paHFQAVEhUSLBUEFRAVBhUGHDYAKAbkuK3mlocYB2VuZ2xpc2gAAAACAAAABAEBAwIVDBk1AAYQGRgEbmFtZRUAFgQWogEWogEm4gE8NgAoBuS4reaWhxgHZW5nbGlzaAAAGRECGRgIAAAAAAAA8D8ZGAgAAAAAAAAAQBUAGRYAABkRAhkYB2VuZ2xpc2gZGAbkuK3mlocVABkWAAAZHBYIFX4WAAAAGRwWpgIVXhYAAAAVAhk8SARyb290FQQAFQolAhgCaWQAFQwlAhgEbmFtZSUATBwAAAAWBBkcGSwmhgEcFQoZJQAGGRgCaWQVABYEFn4WfiYIPDYAKAgAAAAAAAAAQBgIAAAAAAAA8D8AABbcBBUUFuYDFT4AJoQDHBUMGTUABhAZGARuYW1lFQAWBBaiARaiASbiATw2ACgG5Lit5paHGAdlbmdsaXNoAAAW8AQVFhakBBU4ABagAhYEJggWoAIUAAAZHBgMQVJST1c6c2NoZW1hGOABLy8vLy81OEFBQUFFQUFBQTh2Ly8veFFBQUFBRUFBRUFBQUFLQUFzQUNBQUtBQVFBK1AvLy93d0FBQUFJQUFnQUFBQUVBQUlBQUFBMEFBQUFCQUFBQU1ELy8vOGNBQUFBRUFBQUFBZ0FBQUFCRkFBQUFBQUFBUHovLy84RUFBUUFCQUFBQUc1aGJXVUFBQUFBN1AvLy96QUFBQUFnQUFBQUdBQUFBQUVEQUFBUUFCSUFCQUFRQUJFQUNBQUFBQXdBQUFBQUFQci8vLzhDQUFZQUJnQUVBQUlBQUFCcFpBQT0AGAZQb2xhcnMZLBwAABwAAAC8AQAAUEFSMQ==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "775" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:03 GMT" - }, - { - "name": "etag", - "value": "W/\"503f4f766508af8947dcae0e8215e8f7a678857a4123aff4efb8d4b167006bf1\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:03 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "1c8fe117f7fc362a6bb6b0850dbd3e1c5502d45b" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "A851:1634AC:7D8FEE:9634F7:68EA3DDA" - }, - { - "name": "x-served-by", - "value": "cache-lis1490029-LIS" - }, - { - "name": "x-timer", - "value": "S1760181723.047013,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 874, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:02.976Z", - "time": 171, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 171 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/parquet/table/fixtures/generated/loadParquetTable-file-variations-should-load-remote-file_3029162600/recording.har b/table/plugins/parquet/table/fixtures/generated/loadParquetTable-file-variations-should-load-remote-file_3029162600/recording.har deleted file mode 100644 index 25dae4b3..00000000 --- a/table/plugins/parquet/table/fixtures/generated/loadParquetTable-file-variations-should-load-remote-file_3029162600/recording.har +++ /dev/null @@ -1,153 +0,0 @@ -{ - "log": { - "_recordingName": "loadParquetTable-file variations-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "653d6a61a6eccb0e578d514271280781", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 120, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/parquet/table/fixtures/table.parquet" - }, - "response": { - "bodySize": 775, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 775, - "text": "UEFSMRUAFSwVLCwVBBUAFQYVBhw2ACgIAAAAAAAAAEAYCAAAAAAAAPA/AAAAAgAAAAQBAAAAAAAA8D8AAAAAAAAAQBUKGSUABhkYAmlkFQAWBBZ+Fn4mCDw2ACgIAAAAAAAAAEAYCAAAAAAAAPA/AAAVBBUqFSpMFQQVAAAABwAAAGVuZ2xpc2gGAAAA5Lit5paHFQAVEhUSLBUEFRAVBhUGHDYAKAbkuK3mlocYB2VuZ2xpc2gAAAACAAAABAEBAwIVDBk1AAYQGRgEbmFtZRUAFgQWogEWogEm4gE8NgAoBuS4reaWhxgHZW5nbGlzaAAAGRECGRgIAAAAAAAA8D8ZGAgAAAAAAAAAQBUAGRYAABkRAhkYB2VuZ2xpc2gZGAbkuK3mlocVABkWAAAZHBYIFX4WAAAAGRwWpgIVXhYAAAAVAhk8SARyb290FQQAFQolAhgCaWQAFQwlAhgEbmFtZSUATBwAAAAWBBkcGSwmhgEcFQoZJQAGGRgCaWQVABYEFn4WfiYIPDYAKAgAAAAAAAAAQBgIAAAAAAAA8D8AABbcBBUUFuYDFT4AJoQDHBUMGTUABhAZGARuYW1lFQAWBBaiARaiASbiATw2ACgG5Lit5paHGAdlbmdsaXNoAAAW8AQVFhakBBU4ABagAhYEJggWoAIUAAAZHBgMQVJST1c6c2NoZW1hGOABLy8vLy81OEFBQUFFQUFBQTh2Ly8veFFBQUFBRUFBRUFBQUFLQUFzQUNBQUtBQVFBK1AvLy93d0FBQUFJQUFnQUFBQUVBQUlBQUFBMEFBQUFCQUFBQU1ELy8vOGNBQUFBRUFBQUFBZ0FBQUFCRkFBQUFBQUFBUHovLy84RUFBUUFCQUFBQUc1aGJXVUFBQUFBN1AvLy96QUFBQUFnQUFBQUdBQUFBQUVEQUFBUUFCSUFCQUFRQUJFQUNBQUFBQXdBQUFBQUFQci8vLzhDQUFZQUJnQUVBQUlBQUFCcFpBQT0AGAZQb2xhcnMZLBwAABwAAAC8AQAAUEFSMQ==" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "775" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:02 GMT" - }, - { - "name": "etag", - "value": "W/\"503f4f766508af8947dcae0e8215e8f7a678857a4123aff4efb8d4b167006bf1\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:02 GMT" - }, - { - "name": "source-age", - "value": "0" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "MISS" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "c09e1312e933531f0305308bed86788dc728c019" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "A851:1634AC:7D8FEE:9634F7:68EA3DDA" - }, - { - "name": "x-served-by", - "value": "cache-lis1490037-LIS" - }, - { - "name": "x-timer", - "value": "S1760181723.631173,VS0,VE178" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 877, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:02.528Z", - "time": 384, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 384 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/parquet/table/fixtures/table.parquet b/table/plugins/parquet/table/fixtures/table.parquet deleted file mode 100644 index 278fea06..00000000 Binary files a/table/plugins/parquet/table/fixtures/table.parquet and /dev/null differ diff --git a/table/plugins/parquet/table/index.ts b/table/plugins/parquet/table/index.ts deleted file mode 100644 index 9287f51f..00000000 --- a/table/plugins/parquet/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadParquetTable } from "./load.ts" -export { saveParquetTable } from "./save.ts" diff --git a/table/plugins/parquet/table/load.spec.ts b/table/plugins/parquet/table/load.spec.ts deleted file mode 100644 index 00d9e61c..00000000 --- a/table/plugins/parquet/table/load.spec.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadParquetTable } from "./load.ts" - -useRecording() - -describe("loadParquetTable", () => { - describe("file variations", () => { - it("should load local file", async () => { - const path = getTempFilePath() - pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeParquet(path) - - const table = await loadParquetTable({ path }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load local file (multipart)", async () => { - const path1 = getTempFilePath() - const path2 = getTempFilePath() - pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeParquet( - path1, - ) - pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeParquet( - path2, - ) - - const table = await loadParquetTable({ path: [path1, path2] }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file", async () => { - const table = await loadParquetTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/parquet/table/fixtures/table.parquet", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load remote file (multipart)", async () => { - const table = await loadParquetTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/parquet/table/fixtures/table.parquet", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/parquet/table/fixtures/table.parquet", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) -}) diff --git a/table/plugins/parquet/table/load.ts b/table/plugins/parquet/table/load.ts deleted file mode 100644 index 6afa7bc9..00000000 --- a/table/plugins/parquet/table/load.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { prefetchFiles } from "@dpkit/dataset" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import * as pl from "nodejs-polars" - -export async function loadParquetTable( - resource: Partial, - options?: LoadTableOptions, -) { - const [firstPath, ...restPaths] = await prefetchFiles(resource.path) - if (!firstPath) { - throw new Error("Resource path is not defined") - } - - let table = pl.scanParquet(firstPath) - if (restPaths.length) { - table = pl.concat([table, ...restPaths.map(path => pl.scanParquet(path))]) - } - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} diff --git a/table/plugins/parquet/table/save.spec.ts b/table/plugins/parquet/table/save.spec.ts deleted file mode 100644 index 65e88be5..00000000 --- a/table/plugins/parquet/table/save.spec.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { loadParquetTable } from "./load.ts" -import { saveParquetTable } from "./save.ts" - -describe("saveParquetTable", () => { - it("should save table to Parquet file", async () => { - const path = getTempFilePath() - const source = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - name: ["Alice", "Bob", "Charlie"], - }) - .lazy() - - await saveParquetTable(source, { path }) - - const table = await loadParquetTable({ path }) - expect((await table.collect()).toRecords()).toEqual([ - { id: 1.0, name: "Alice" }, - { id: 2.0, name: "Bob" }, - { id: 3.0, name: "Charlie" }, - ]) - }) - - it("should save and load various data types", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveParquetTable(source, { - path, - fieldTypes: { - array: "array", - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadParquetTable({ path }, { denormalized: true }) - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: true, - date: "2025-01-01", - datetime: new Date(Date.UTC(2025, 0, 1)), - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: [1.0, 2.0, 3.0], - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) -}) diff --git a/table/plugins/parquet/table/save.ts b/table/plugins/parquet/table/save.ts deleted file mode 100644 index 66a81e93..00000000 --- a/table/plugins/parquet/table/save.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { assertLocalPathVacant } from "@dpkit/dataset" -import type { SaveTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { denormalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" - -export async function saveParquetTable( - table: Table, - options: SaveTableOptions, -) { - const { path, overwrite } = options - - if (!overwrite) { - await assertLocalPathVacant(path) - } - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - table = await denormalizeTable(table, schema, { - nativeTypes: [ - "boolean", - "datetime", - "integer", - "list", - "number", - "string", - "year", - ], - }) - - await table - .sinkParquet(path, { - maintainOrder: true, - }) - .collect() - - return path -} diff --git a/table/plugins/xlxs/index.ts b/table/plugins/xlxs/index.ts deleted file mode 100644 index 5f4f33d9..00000000 --- a/table/plugins/xlxs/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./table/index.ts" -export * from "./plugin.ts" diff --git a/table/plugins/xlxs/plugin.spec.ts b/table/plugins/xlxs/plugin.spec.ts deleted file mode 100644 index 3733871a..00000000 --- a/table/plugins/xlxs/plugin.spec.ts +++ /dev/null @@ -1,188 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { beforeEach, describe, expect, it, vi } from "vitest" -import { XlsxPlugin } from "./plugin.ts" -import * as loadModule from "./table/load.ts" -import * as saveModule from "./table/save.ts" - -vi.mock("./table/load.ts", () => ({ - loadXlsxTable: vi.fn(), -})) - -vi.mock("./table/save.ts", () => ({ - saveXlsxTable: vi.fn(), -})) - -describe("XlsxPlugin", () => { - let plugin: XlsxPlugin - let mockLoadXlsxTable: ReturnType - let mockSaveXlsxTable: ReturnType - - beforeEach(() => { - plugin = new XlsxPlugin() - mockLoadXlsxTable = vi.mocked(loadModule.loadXlsxTable) - mockSaveXlsxTable = vi.mocked(saveModule.saveXlsxTable) - vi.clearAllMocks() - }) - - describe("loadTable", () => { - it("should load table from xlsx file", async () => { - const resource: Partial = { - path: "test.xlsx", - } - const mockTable = pl.DataFrame().lazy() - mockLoadXlsxTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadXlsxTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should return undefined for non-xlsx files", async () => { - const resource: Partial = { - path: "test.csv", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const resource: Partial = { - path: "test.txt", - format: "xlsx", - } - const mockTable = pl.DataFrame().lazy() - mockLoadXlsxTable.mockResolvedValue(mockTable) - - const result = await plugin.loadTable(resource) - - expect(mockLoadXlsxTable).toHaveBeenCalledWith(resource, undefined) - expect(result).toEqual(mockTable) - }) - - it("should pass through load options", async () => { - const resource: Partial = { - path: "test.xlsx", - } - const options = { denormalized: true } - const mockTable = pl.DataFrame().lazy() - mockLoadXlsxTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource, options) - - expect(mockLoadXlsxTable).toHaveBeenCalledWith(resource, options) - }) - - it("should handle paths with directories", async () => { - const resource: Partial = { - path: "/path/to/data.xlsx", - } - const mockTable = pl.DataFrame().lazy() - mockLoadXlsxTable.mockResolvedValue(mockTable) - - await plugin.loadTable(resource) - - expect(mockLoadXlsxTable).toHaveBeenCalledWith(resource, undefined) - }) - - it("should return undefined for ods files", async () => { - const resource: Partial = { - path: "test.ods", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for json files", async () => { - const resource: Partial = { - path: "test.json", - } - - const result = await plugin.loadTable(resource) - - expect(mockLoadXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) - - describe("saveTable", () => { - it("should save table to xlsx file", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.xlsx" } - mockSaveXlsxTable.mockResolvedValue("output.xlsx") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.xlsx") - }) - - it("should return undefined for non-xlsx files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.csv" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should handle explicit format specification", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.txt", format: "xlsx" as const } - mockSaveXlsxTable.mockResolvedValue("output.txt") - - const result = await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).toHaveBeenCalledWith(table, options) - expect(result).toBe("output.txt") - }) - - it("should handle paths with directories", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "/path/to/output.xlsx" } - mockSaveXlsxTable.mockResolvedValue("/path/to/output.xlsx") - - await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).toHaveBeenCalledWith(table, options) - }) - - it("should return undefined for files without extension", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for ods files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.ods" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - - it("should return undefined for json files", async () => { - const table = pl.DataFrame().lazy() - const options = { path: "output.json" } - - const result = await plugin.saveTable(table, options) - - expect(mockSaveXlsxTable).not.toHaveBeenCalled() - expect(result).toBeUndefined() - }) - }) -}) diff --git a/table/plugins/xlxs/plugin.ts b/table/plugins/xlxs/plugin.ts deleted file mode 100644 index 3480f1b3..00000000 --- a/table/plugins/xlxs/plugin.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Resource } from "@dpkit/metadata" -import { inferFormat } from "@dpkit/metadata" -import type { - LoadTableOptions, - SaveTableOptions, - TablePlugin, -} from "../../plugin.ts" -import type { Table } from "../../table/index.ts" -import { loadXlsxTable } from "./table/index.ts" -import { saveXlsxTable } from "./table/index.ts" - -export class XlsxPlugin implements TablePlugin { - async loadTable(resource: Partial, options?: LoadTableOptions) { - const isXlsx = getIsXlsx(resource) - if (!isXlsx) return undefined - - return await loadXlsxTable(resource, options) - } - - async saveTable(table: Table, options: SaveTableOptions) { - const { path, format } = options - - const isXlsx = getIsXlsx({ path, format }) - if (!isXlsx) return undefined - - return await saveXlsxTable(table, options) - } -} - -function getIsXlsx(resource: Partial) { - const format = inferFormat(resource) - return ["xlsx"].includes(format ?? "") -} diff --git a/table/plugins/xlxs/table/fixtures/generated/loadXlsxTable-file-variations-should-load-multipart-remote-file_2310816283/recording.har b/table/plugins/xlxs/table/fixtures/generated/loadXlsxTable-file-variations-should-load-multipart-remote-file_2310816283/recording.har deleted file mode 100644 index 3714f412..00000000 --- a/table/plugins/xlxs/table/fixtures/generated/loadXlsxTable-file-variations-should-load-multipart-remote-file_2310816283/recording.har +++ /dev/null @@ -1,292 +0,0 @@ -{ - "log": { - "_recordingName": "loadXlsxTable-file variations-should load multipart remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "25af28f3d42b3c31469c3da0dee974ce", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 114, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/xlsx/table/fixtures/table.xlsx" - }, - "response": { - "bodySize": 7284, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 7284, - "text": "UEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAaAAAAeGwvX3JlbHMvd29ya2Jvb2sueG1sLnJlbHO9kstqwzAQRff5CjH7WLb7oBTL2ZRCtm36AUIeWya2JEbTR/6+alMaB4LpwnQl5kpz70Ez1eZjHMQbUuy9U1BkOQh0xje96xS87B7Xd7CpV9UTDprTk2j7EEXqcVGBZQ73UkZjcdQx8wFdumk9jZpTSZ0M2ux1h7LM81tJUw+ozzzFtlFA26YAsTsE/Iu3b9ve4IM3ryM6vhAhOfViMtTUISv4Lo9ikSUzkJcZyiUZIh8GjCeIYz0Xf7Vk/LunfbSIfCL4lRLc1zH7F9f/DFPOwdwsOhirCZtnprTp0/lM5R+YVSXP9r/+BFBLBwiyzuVs6QAAADYDAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAAA8AAAB4bC93b3JrYm9vay54bWyNU0tu2zAQ3fcUAve2Pv7UNiwHrmwhAfpDnCZrShpZrClSIGnLbtFFe41epMdIbtQRZaUuWhRdSCLn8+bNzNP86lhy5wBKMylC4vc94oBIZcbENiQf7uLehDjaUJFRLgWE5ASaXC1ezGupdomUOwfzhQ5JYUw1c12dFlBS3ZcVCPTkUpXU4FVtXV0poJkuAEzJ3cDzxm5JmSAtwkz9D4bMc5bCSqb7EoRpQRRwapC9LlilyWKeMw73bUMOraq3tETaEeUpcRfPtN8rJ6Hpbl/FGB2SnHIN2Ggh63fJR0gNdkQ5J05GDfhTb9iF/AYhDUZiGTQ2hnsGtf7lb64W8Voq9kkKQ/kmVZLzkBi1P1dDooalf/NsmkHd0UR3xuMDE5msQ4IrOl2ca3t8YJkpcIHjwWTY2a6BbQsTkok/DYhjaHLbDCokIw/Tcqa0sUUsCsVODoD1mhs25F50ZHfWfR1hB/r4/fHH09enb37DFs03GRa3UjHoPTDNEo6k1YyhQ91kgwb0EsCWdoKL9OAf6UPLqSOCA0txg8yAwvhI7gU24TddKcjfyAwhlkjm7H9e7/m+Am4ottn3PL+BhaN5rY39nrXIJZ7/0CNniYJWgVaMxNkrFpLPL8fBOJqMg16w9Ac931+Peq8Gw1EvXscxjj5aRdP4CwrTos7wiVr62ij8y24h35xQHMdWpEt/fUyBLy0zF4PbtyXodtJa/ARQSwcIYZjxohwCAAC3AwAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAATAAAAeGwvdGhlbWUvdGhlbWUxLnhtbN2VTW/bMAyG7/sVgu6r4rgJ0iBOMSwLdiiwQ7bdGZm21UiyIant8u+nyE7ir6HDMGDofIlIPXxFioy9uv+hJHlGY0WpExrdTChBzctU6Dyh375u3y8osQ50CrLUmNAjWnq/freCpStQIfHh2i4hoYVz1ZIxy70b7E1ZofZ7WWkUOG+anKUGXryskmw6mcyZAqFpE29+J77MMsFxU/InhdrVIgYlOJ+6LURlKdGgfI5fAkjX5yQ/STxF2JODS7PjIfOafRB7g62A9BCdfqzJ9x+lIc8gEzoJD2XrFbsA0g25LDwN1wDpYfqa3rTWG3I9vQAA576U4dnRAuJJ3LAtqF6O5BDP76DLt/TjAQ9xjD39+MrfDviFp3v6t1d+NuD53R2/3EkLqpfzEX4aRdjhA1RIoQ+jN45n+oJkpfw8is9mESz2DX6lWGt86njtOsPUmiMFj6XZeiA018+oJu5YYQbccx+MAElJJRwvtqCEPPoUKeEFGIvON/N0NCwRWjEbfITvT2QH2r4eye2fRbJe4kroN1rFNXHWblRom2obQsqdO0p8sKFIW0qRbr0zGAG7jEVV+CUNiped2uoE/XMFNixL6q5FXhI6j2enq4PKv2l8b/1SVWlCrc4pAZn7zwF3JgxzZazbgC3qFMJJdYeUcGia95N+m8qsfzmYZcjdLzxX0+/VIqO7fx9mY5nt8+3/Ob/9wljnb8sGH/azZ/0TUEsHCPaw8YIeAgAA0QgAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAADQAAAHhsL3N0eWxlcy54bWztWt1u2jAYvd9TWL5fEyjQdQrpOiqmbdJUDTpNmnZhEifx6tiRY1ro08+Ok5CU0FG6rjCFm8Qn3/l8cvwLxjlbxBTcYJESzoawc2RDgJnHfcLCIbyajl+/gSCViPmIcoaHcIlTeOa+clK5pHgSYSyBysDSIYykTN5aVupFOEbpEU8wU08CLmIkVVGEVpoIjPxUk2JqdW17YMWIMOg6bB6PY5kCj8+ZHMLjEgLm8tFX2gY9CEy6EfeVlA+YYYEotBqD+/XgdxvCBvUwW4dZuRzXCThbqRpAA7hOegduEFX0LJyhGJvyuSBGT4BiQpcG7GrAi5BIlVeGllViUu2Q0D4Q+gX+hb7NwQSxtNGTrZJ84ZJnKcDo02cwGT2UKbvoViOU1vqSBlwnQVJiwcaqAPL76TJRjc5UzzZpsrg/RIcCLTvd/vaElFPiaxXhiFMugAhnQzjOPzrNbP3BKHtRq5KzrC27qLecceGrcVu8ZxcWEPAJCjlD9CoZwgDRFMMSuuC3rABdh+JAqmoECSN9lTzRariUPFY3BUcLMZl3qwFkc8UQyoh417Cobw1Vta9hRssa3KQsv1HGeJjSiWZ8D1bu2ErMIlifTlhWULOedjW/NZnyAkoSuhxznUSKOc6B91lIDTqnJGQxvhd4KbjEnsxm1wx2HVQEgogLcqdS644c5rOZnowl8TRk3h4CiRfyK5fIZFGabgVKpgoszSfMzypWz9JIEHY95WNSPlY2JaUMQLl3jf1CZER8Ra1EWovgnlP2yqfOrj7lOu8bVYWrThXd53DEdFsxG8TsPLZaMa2YVkwrphWzi5je8T6tlL3OXqnp7ZWa7j6pOX1hMVZ1+24285V9fOd41338IliXXhX0RO2Htql/im3bt/hBfBHKqc9s2iP6Wuvaf+vaI8Znv9G0btW0ztbj8+/9TuEpAIv97GeD1rInDM3eP14FnmPpNFW1K+fezWbNnr340Nxny9rZrF0y92Aqa4flxk7WbztZo2NW/q29chZXO0UvUaAPevUhr4h1a5emzeaESsJMyVonjHgcoyK+068RjjcSwA/7Z0ka1EiDRtJcCMy8Zck5qXF6D3Fqdb2p8U6aeJdY6BYsKac1ijltXpmpCqs/YLi/AVBLBwiJLRUOUAMAAMUhAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAABgAAAB4bC93b3Jrc2hlZXRzL3NoZWV0MS54bWy9Vt1u2zYUvt9TaMIQbEBq/Th2nER2kTr1Wsytgzhpgd3REmURoUiVpOw4d91r7EX2GO0b7ZCUZMU2hmDI5gvbPCS/853vkDwnev2QU2eFhSScDd2g47sOZjFPCFsO3bvbyauB60iFWIIoZ3jobrB0X49+iNZc3MsMY+UAAJNDN1OqOPc8GWc4R7LDC8xgJuUiRwqGYunJQmCUmE059ULf73s5Isy1COfiORg8TUmMr3hc5pgpCyIwRQroy4wUskZ7SJ6Flwi0hlBrPi2KV3amwQtO9vByEgsueao6Mc8ravtRnnlnT+J8EOG/Qwp6EOqK6EyFNVgePyfKHIn7sngF2AUotSCUqI0J2B1FBv9aOCmhCosPPIEkp4hKDHMFWuI5VneFmVe3/BoM9bQ3irxq8yhKCORDM3METofuZXD+pqtXmAWfCF7L1n9HZnw9AX4lRbKGM8ZfBUmmhOGn1hu+HnP6DsSAczp0lSirid8xqFYbBFlmQHGKU9XsVmgxxxTHCiftfbNSUfAy3+QLThuABKeopEpzAHdc1PYVUB66TOtJAZIX2sUYU6rjdJ1Yr30P+P0T13nkPJ/HiIJKge+3xh/N9l2r1nOKNrw0ulSz+motOL/XJo3r6yyZKLS+BdLXsGLhOgisK2zZgOStsd3qyC8mIzYdXpOD9v86NxNzZCDXlRKgwmeSqGzoDjr907P+4LTXqAQ5eYe14kD6pNOHibiUiue10Ur3CAmqLVVKuNV+ileYAojh2LaBUxuy94TTKAKdpfnWilNUSJ3TCtT6rsha1xlJEswOujU+c/QwdOEe5US/e/qR25isgf4WJgg6Pa3ZC3vs+ZXP7gGfg86g//I+Az8YVE7D3jbUsO22+1+4DXu1vn5w1vjt7vn1bG7tu4sUGkWCrx1hlLHe7TFoHJpj1zXHbofKoVO4R28vRghdu9PXGZ4DuMewV4J1NfIjb6UJVive7K8ImhUesG6oh/8z9dAQ6xlibIeYpW5XhJa6ErAmhecCNUCfprPZb3fXP/90GR4foYLLi7kp8KEd/Kgf9fA4PPZ/ibxUO8BsSYnMDsff/af4Xzr47l7w4U7wdkW/lbdwh7fXOn6FIEzNCtNVOBkUHugEtjVpuVelGgtUy+aOcEEeOVOIjqFVwaJ1o6DfUiTen/Bsyf2AxJKAY2pKmd85NQ+vsNLUQ6gApl/rhafNB275givQ8dBMZurnFiDlXLXGXlPuywKKTIHFnDzax0m2CpppA6q3IKiGTSFwHQ0xE8ZPwtfsNsNsBtFCLgWBYE2fNnQLLpRABMrXgqL4/pIlnzOims7Cga6sVcRjKGZjnuuGT+o6zJ6Ie1UQOGyaWq3q1hLzgugsBTo6G//ERO0kJE1BeaYmRMitq8Y8S5K3q+3xHEU8SWwDMjpCeXExNt9HX0quLm6h95HOR+hrbniO2PENXkJjI+ykWReE3/789tf3r9//gDu5RdKglk8bNAjNz7VZWU1HXps9DJv2e/Q3UEsHCGiH/ABmBAAAwgsAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAGAAAAHhsL3dvcmtzaGVldHMvc2hlZXQyLnhtbL2Y33OjNhDH3/tXMDz07YzBxj9S2zdXQ3o3k2syTa430zcZhNFEICrJdpy/visJMEbnm0wn4xcbvhK7+9kFvOvFx5eCOnvMBWHl0vUHQ9fBZcJSUm6X7ren2w8z1xESlSmirMRL94iF+3H1y+LA+LPIMZYOGCjF0s2lrG48TyQ5LpAYsAqXsJIxXiAJp3zriYpjlOqLCuoFw+HEKxApXWPhhr/FBssykuCIJbsCl9IY4ZgiCeGLnFSisfaSvsleytEBUJt4OiFGZqW1548tewVJOBMsk4OEFXVoNuXcm59xvvDg/1nyQ0DdE1WpoDFWJG+hLBB/3lUfwHYFmdoQSuRRA7urhbb/wJ2MUIn5V5ZCkTNEBYa1Cm3xI5bfKr0un9gDCEtX8h12vdXCq69dLVIC5VCBORxnS/eTfxP7Q7VF7/ib4IPoHDsiZ4dbiG9HkWi8afEPTtI7UuJz9S92WDP6GZIB92ntXi/8gyFrjcDJNocQ73Am26sl2jxiihOJ0zOL9ztJwc3jsdgw2lpIcYZ2VKogwB/jjb6HmJduqRJKwSarlI81plSRuk6i9n4BB5Ox67wyVjwmiEKa/OGwc/6nvryvqoTeoSPb6cTUq+rZ2jD2rCRld6jKpDFUhiuknsM6CtdBoO7xKZrTubnUEf/WNTmVTBnuHjfFudX3DBS7zgRk4TtJZb50Z4PJdD6ZTcM2S1CUz1ilHIIeDcawkOyEZEUjmtS9QoUapc4/M7m/w3tMwYiOsauBU4PsncW0WkCehf5UGaeoEp2iGt91sMZ1TtIUlz90q30W6GXpBuEEjoh69an33FHVbQoVMIb8yWAUqrS9r1N/GIS11yCctn5HZ35HulyG2byQkESrBWcHh+tojXeTntZhXY7AiuRHxbGisxCBXHlT9xU8JTOdCiX83hfWfSHqC3FH8ICiRQmuixLoMObwHMMjDep+5S+8vUp1w2Z2wPtLbRF6y+h8y7re4p9wLSXuKmfAo58Bj9Ud977EI4s46BGbHX7QIR73iEcWsaXEo0vE45+W+J1xx/17ddwPdG0pkaXE40sw4TVhwj5MaMFYSmQpcXgJZnJNmEkfZmLBWEpkKfHkEsz0mjDTPszUgrGUyFLi6SWY2TVhZn2YmQVjKZGlxLNLMPNrwsz7MHMLxlIiS4nnl2DUL8P1aJS33m/v0OKxpciW4jPJIHmd1qLipJT3lR6lnBy6bRh/To341mrNWwVGhLb/YZy8slIiuob5DPNOtwRDpiSJveCZOeMr4lsCjqnu3/26n9dH0Ovq7w2TkEh9mOthADI3CH1/5kNHNZoEwXAMj1PGmDRLQdgYhwh3FXTOFeaP5BWbbq/Tpevhpm7k/Pq07W5dR5m459phyg7lU47Le8CBMnICNHr6XLoV45IjAj35hqLk+VOZfs+JbMcpB2bNzmiSQIe+ZoUaY4UaLsqz7EUVgUZBhdak7aQkrCKqDLqMJhG3mtlJSZZBakt5S7g4uWrl+zSN96c7c7VgaWrGKrgTOsdwaCyufkVF9dtaf/qB/nrQO+vlhdf1D6ft3wKr/wBQSwcIHv/PMpAEAABaEAAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAUAAAAeGwvc2hhcmVkU3RyaW5ncy54bWyNkEFqAjEYhfeeIvx7zShUiiSZheAJ7AHCzO9MYPJnzJ8RPUFv0E3P0L0I3kZ7DtNC6U5m+Xjfew+eKo++EweM7AJpmM8KEEhVqB01Gt62m+krCE6WatsFQg0nZCjNRDEnkaPEGtqU+pWUXLXoLc9Cj5SdXYjepixjI7mPaGtuEZPv5KIoltJbRyCqMFDS8AJiILcfcP2njWJn1O/Eintb5eXcwRgPCMbVSiaj5A/yBCPrcRR4O399f7yPQpGaznE7tvb+eb1dLv+0zLeZB1BLBwih0YAvzgAAAHQBAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAAAsAAABfcmVscy8ucmVsc62Sz0oDMRCH732KkHt3thVEZLO9iNCbSH2AmMz+YTeZMBl1fXuDCFqppQePSX7zzTdDmt0SZvWKnEeKRm+qWiuMjvwYe6OfDvfrG71rV80jzlZKJA9jyqrUxGz0IJJuAbIbMNhcUcJYXjriYKUcuYdk3WR7hG1dXwP/ZOj2iKn23mje+41Wh/eEl7Cp60aHd+ReAkY50eJXopAt9yhGLzO8EU/PRFNVoBpOu2wvd/l7Tggo1lux4IhxnbhUs4yYv3U8uYdynT8T54Su/nM5uAhGj/68kk3py2jVwNEnaD8AUEsHCGaqgrfgAAAAOwIAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAEQAAAGRvY1Byb3BzL2NvcmUueG1sbVLJbsIwEL33KyLfE8fQNUqC1FacilQJqla9ufYQ3MaOZQ8E/r5OgBRUbvMWv/GMnU+2uo424LxqTEFYkpIIjGikMlVB3hbT+J5EHrmRvG4MFGQHnkzKq1zYTDQOXl1jwaECH4Ug4zNhC7JCtBmlXqxAc58EhwnisnGaY4CuopaLH14BHaXpLdWAXHLktAuM7ZBIDpFSDJF27eo+QAoKNWgw6ClLGP3zIjjtLx7olROnVrizcNF6FAf31qvB2LZt0o57a7g/ox+zl3k/aqxMtyoBpMylyIQDjo0rc3oKQi3BC6cshpXvxTMi4Jqbah32U4KJ3+a9ZaC6zdfc4yy80VKBfNyFjAvcYROZPnBRGCHbD3yU3sdPz4spKUfp6CZmaczYgrHs+iFjd59d0/OAvrODjeq+Ssn6pgPsbu3XX98gcD/SAEKNCmvY08fy3/cpfwFQSwcI+t3l+lIBAACKAgAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAQAAAAZG9jUHJvcHMvYXBwLnhtbJ2QTW/CMAyG7/sVVcS1SSm0QigN2jTthLQdOrRblaUuZMqXkhSVf78AGnCeT/Zr67H90s2kVXYEH6Q1DZrjAmVghO2l2Tfos33LVygLkZueK2ugQScIaMOe6Ie3DnyUELJEMKFBhxjdmpAgDqB5wKltUmewXvOYSr8ndhikgFcrRg0mkrIoagJTBNNDn7sbEF2J62P8L7S34nxf2LUnl3iMtqCd4hEYJfe0tZGrVmpgRZJvBX12TknBY3KEbeW3h/fLClJWuMRLvJhtpRmn7mtVd/Uye5jo0g8/ICKpymL2MkrV5wtKHnFn9u5qNptXuEhxGfjTKLn7yn4BUEsHCF/Olp77AAAAnAEAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAEwAAAFtDb250ZW50X1R5cGVzXS54bWzFVTtPwzAQ3vsrIq8ocdsBIZSkA48RKlFmZOJLYprYlu2W9t9zdqAqpQ+qVrDEiu++x13sSzpatE00B2OFkhkZJH0SgSwUF7LKyPPkPr4io7yXTpYabIS50makdk5fU2qLGlpmE6VBYqRUpmUOX01FNSumrAI67PcvaaGkA+li5zlInt5CyWaNi+4WuN3pIpxEN12el8oI07oRBXMYpj5Kt+IMNHYPcC75hrv401mCyJBja6HtxW4FLasNAdH6yvz+dsSbhu2QEEDMI7bbCA7RmBn3wFpMoIuGvvhi6Lsy01elpglaSs5c3g7hdcnj1FRZigK4KmYtQhKrDTBuawCH5sOatEzIA/oOjxF0z8HJHgLNAUHrlg3Yc5cbSH/R6gCwNCyn1/vdxIr/SB/Df/Jha2aAPzmD4+bsH2Sde5+P7uL9xWVDp2OjtMWRaOD4cr/0PDrWSATGif1nbqWI1Cf3F/yQ48B/avdSGv4Q+QdQSwcI82Bc924BAABQBgAAUEsBAhQAFAAICAgAKFZLW7LO5WzpAAAANgMAABoAAAAAAAAAAAAAAAAAAAAAAHhsL19yZWxzL3dvcmtib29rLnhtbC5yZWxzUEsBAhQAFAAICAgAKFZLW2GY8aIcAgAAtwMAAA8AAAAAAAAAAAAAAAAAMQEAAHhsL3dvcmtib29rLnhtbFBLAQIUABQACAgIAChWS1v2sPGCHgIAANEIAAATAAAAAAAAAAAAAAAAAIoDAAB4bC90aGVtZS90aGVtZTEueG1sUEsBAhQAFAAICAgAKFZLW4ktFQ5QAwAAxSEAAA0AAAAAAAAAAAAAAAAA6QUAAHhsL3N0eWxlcy54bWxQSwECFAAUAAgICAAoVktbaIf8AGYEAADCCwAAGAAAAAAAAAAAAAAAAAB0CQAAeGwvd29ya3NoZWV0cy9zaGVldDEueG1sUEsBAhQAFAAICAgAKFZLWx7/zzKQBAAAWhAAABgAAAAAAAAAAAAAAAAAIA4AAHhsL3dvcmtzaGVldHMvc2hlZXQyLnhtbFBLAQIUABQACAgIAChWS1uh0YAvzgAAAHQBAAAUAAAAAAAAAAAAAAAAAPYSAAB4bC9zaGFyZWRTdHJpbmdzLnhtbFBLAQIUABQACAgIAChWS1tmqoK34AAAADsCAAALAAAAAAAAAAAAAAAAAAYUAABfcmVscy8ucmVsc1BLAQIUABQACAgIAChWS1v63eX6UgEAAIoCAAARAAAAAAAAAAAAAAAAAB8VAABkb2NQcm9wcy9jb3JlLnhtbFBLAQIUABQACAgIAChWS1tfzpae+wAAAJwBAAAQAAAAAAAAAAAAAAAAALAWAABkb2NQcm9wcy9hcHAueG1sUEsBAhQAFAAICAgAKFZLW/NgXPduAQAAUAYAABMAAAAAAAAAAAAAAAAA6RcAAFtDb250ZW50X1R5cGVzXS54bWxQSwUGAAAAAAsACwDGAgAAmBkAAAAA" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "7284" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:39 GMT" - }, - { - "name": "etag", - "value": "W/\"8358afe5b3c08d495ad280e2822044f257db2c761c4adee5df64cb2c957dc80b\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:39 GMT" - }, - { - "name": "source-age", - "value": "137" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "48514c5a5949071b783a65d7886e82c08de9f4b5" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "70F2:2E0B15:7D2488:95C02A:68EA3D75" - }, - { - "name": "x-served-by", - "value": "cache-lis1490034-LIS" - }, - { - "name": "x-timer", - "value": "S1760181759.196638,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 877, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:39.242Z", - "time": 52, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 52 - } - }, - { - "_id": "25af28f3d42b3c31469c3da0dee974ce", - "_order": 1, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 114, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/xlsx/table/fixtures/table.xlsx" - }, - "response": { - "bodySize": 7284, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 7284, - "text": "UEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAaAAAAeGwvX3JlbHMvd29ya2Jvb2sueG1sLnJlbHO9kstqwzAQRff5CjH7WLb7oBTL2ZRCtm36AUIeWya2JEbTR/6+alMaB4LpwnQl5kpz70Ez1eZjHMQbUuy9U1BkOQh0xje96xS87B7Xd7CpV9UTDprTk2j7EEXqcVGBZQ73UkZjcdQx8wFdumk9jZpTSZ0M2ux1h7LM81tJUw+ozzzFtlFA26YAsTsE/Iu3b9ve4IM3ryM6vhAhOfViMtTUISv4Lo9ikSUzkJcZyiUZIh8GjCeIYz0Xf7Vk/LunfbSIfCL4lRLc1zH7F9f/DFPOwdwsOhirCZtnprTp0/lM5R+YVSXP9r/+BFBLBwiyzuVs6QAAADYDAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAAA8AAAB4bC93b3JrYm9vay54bWyNU0tu2zAQ3fcUAve2Pv7UNiwHrmwhAfpDnCZrShpZrClSIGnLbtFFe41epMdIbtQRZaUuWhRdSCLn8+bNzNP86lhy5wBKMylC4vc94oBIZcbENiQf7uLehDjaUJFRLgWE5ASaXC1ezGupdomUOwfzhQ5JYUw1c12dFlBS3ZcVCPTkUpXU4FVtXV0poJkuAEzJ3cDzxm5JmSAtwkz9D4bMc5bCSqb7EoRpQRRwapC9LlilyWKeMw73bUMOraq3tETaEeUpcRfPtN8rJ6Hpbl/FGB2SnHIN2Ggh63fJR0gNdkQ5J05GDfhTb9iF/AYhDUZiGTQ2hnsGtf7lb64W8Voq9kkKQ/kmVZLzkBi1P1dDooalf/NsmkHd0UR3xuMDE5msQ4IrOl2ca3t8YJkpcIHjwWTY2a6BbQsTkok/DYhjaHLbDCokIw/Tcqa0sUUsCsVODoD1mhs25F50ZHfWfR1hB/r4/fHH09enb37DFs03GRa3UjHoPTDNEo6k1YyhQ91kgwb0EsCWdoKL9OAf6UPLqSOCA0txg8yAwvhI7gU24TddKcjfyAwhlkjm7H9e7/m+Am4ottn3PL+BhaN5rY39nrXIJZ7/0CNniYJWgVaMxNkrFpLPL8fBOJqMg16w9Ac931+Peq8Gw1EvXscxjj5aRdP4CwrTos7wiVr62ij8y24h35xQHMdWpEt/fUyBLy0zF4PbtyXodtJa/ARQSwcIYZjxohwCAAC3AwAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAATAAAAeGwvdGhlbWUvdGhlbWUxLnhtbN2VTW/bMAyG7/sVgu6r4rgJ0iBOMSwLdiiwQ7bdGZm21UiyIant8u+nyE7ir6HDMGDofIlIPXxFioy9uv+hJHlGY0WpExrdTChBzctU6Dyh375u3y8osQ50CrLUmNAjWnq/freCpStQIfHh2i4hoYVz1ZIxy70b7E1ZofZ7WWkUOG+anKUGXryskmw6mcyZAqFpE29+J77MMsFxU/InhdrVIgYlOJ+6LURlKdGgfI5fAkjX5yQ/STxF2JODS7PjIfOafRB7g62A9BCdfqzJ9x+lIc8gEzoJD2XrFbsA0g25LDwN1wDpYfqa3rTWG3I9vQAA576U4dnRAuJJ3LAtqF6O5BDP76DLt/TjAQ9xjD39+MrfDviFp3v6t1d+NuD53R2/3EkLqpfzEX4aRdjhA1RIoQ+jN45n+oJkpfw8is9mESz2DX6lWGt86njtOsPUmiMFj6XZeiA018+oJu5YYQbccx+MAElJJRwvtqCEPPoUKeEFGIvON/N0NCwRWjEbfITvT2QH2r4eye2fRbJe4kroN1rFNXHWblRom2obQsqdO0p8sKFIW0qRbr0zGAG7jEVV+CUNiped2uoE/XMFNixL6q5FXhI6j2enq4PKv2l8b/1SVWlCrc4pAZn7zwF3JgxzZazbgC3qFMJJdYeUcGia95N+m8qsfzmYZcjdLzxX0+/VIqO7fx9mY5nt8+3/Ob/9wljnb8sGH/azZ/0TUEsHCPaw8YIeAgAA0QgAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAADQAAAHhsL3N0eWxlcy54bWztWt1u2jAYvd9TWL5fEyjQdQrpOiqmbdJUDTpNmnZhEifx6tiRY1ro08+Ok5CU0FG6rjCFm8Qn3/l8cvwLxjlbxBTcYJESzoawc2RDgJnHfcLCIbyajl+/gSCViPmIcoaHcIlTeOa+clK5pHgSYSyBysDSIYykTN5aVupFOEbpEU8wU08CLmIkVVGEVpoIjPxUk2JqdW17YMWIMOg6bB6PY5kCj8+ZHMLjEgLm8tFX2gY9CEy6EfeVlA+YYYEotBqD+/XgdxvCBvUwW4dZuRzXCThbqRpAA7hOegduEFX0LJyhGJvyuSBGT4BiQpcG7GrAi5BIlVeGllViUu2Q0D4Q+gX+hb7NwQSxtNGTrZJ84ZJnKcDo02cwGT2UKbvoViOU1vqSBlwnQVJiwcaqAPL76TJRjc5UzzZpsrg/RIcCLTvd/vaElFPiaxXhiFMugAhnQzjOPzrNbP3BKHtRq5KzrC27qLecceGrcVu8ZxcWEPAJCjlD9CoZwgDRFMMSuuC3rABdh+JAqmoECSN9lTzRariUPFY3BUcLMZl3qwFkc8UQyoh417Cobw1Vta9hRssa3KQsv1HGeJjSiWZ8D1bu2ErMIlifTlhWULOedjW/NZnyAkoSuhxznUSKOc6B91lIDTqnJGQxvhd4KbjEnsxm1wx2HVQEgogLcqdS644c5rOZnowl8TRk3h4CiRfyK5fIZFGabgVKpgoszSfMzypWz9JIEHY95WNSPlY2JaUMQLl3jf1CZER8Ra1EWovgnlP2yqfOrj7lOu8bVYWrThXd53DEdFsxG8TsPLZaMa2YVkwrphWzi5je8T6tlL3OXqnp7ZWa7j6pOX1hMVZ1+24285V9fOd41338IliXXhX0RO2Htql/im3bt/hBfBHKqc9s2iP6Wuvaf+vaI8Znv9G0btW0ztbj8+/9TuEpAIv97GeD1rInDM3eP14FnmPpNFW1K+fezWbNnr340Nxny9rZrF0y92Aqa4flxk7WbztZo2NW/q29chZXO0UvUaAPevUhr4h1a5emzeaESsJMyVonjHgcoyK+068RjjcSwA/7Z0ka1EiDRtJcCMy8Zck5qXF6D3Fqdb2p8U6aeJdY6BYsKac1ijltXpmpCqs/YLi/AVBLBwiJLRUOUAMAAMUhAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAABgAAAB4bC93b3Jrc2hlZXRzL3NoZWV0MS54bWy9Vt1u2zYUvt9TaMIQbEBq/Th2nER2kTr1Wsytgzhpgd3REmURoUiVpOw4d91r7EX2GO0b7ZCUZMU2hmDI5gvbPCS/853vkDwnev2QU2eFhSScDd2g47sOZjFPCFsO3bvbyauB60iFWIIoZ3jobrB0X49+iNZc3MsMY+UAAJNDN1OqOPc8GWc4R7LDC8xgJuUiRwqGYunJQmCUmE059ULf73s5Isy1COfiORg8TUmMr3hc5pgpCyIwRQroy4wUskZ7SJ6Flwi0hlBrPi2KV3amwQtO9vByEgsueao6Mc8ravtRnnlnT+J8EOG/Qwp6EOqK6EyFNVgePyfKHIn7sngF2AUotSCUqI0J2B1FBv9aOCmhCosPPIEkp4hKDHMFWuI5VneFmVe3/BoM9bQ3irxq8yhKCORDM3METofuZXD+pqtXmAWfCF7L1n9HZnw9AX4lRbKGM8ZfBUmmhOGn1hu+HnP6DsSAczp0lSirid8xqFYbBFlmQHGKU9XsVmgxxxTHCiftfbNSUfAy3+QLThuABKeopEpzAHdc1PYVUB66TOtJAZIX2sUYU6rjdJ1Yr30P+P0T13nkPJ/HiIJKge+3xh/N9l2r1nOKNrw0ulSz+motOL/XJo3r6yyZKLS+BdLXsGLhOgisK2zZgOStsd3qyC8mIzYdXpOD9v86NxNzZCDXlRKgwmeSqGzoDjr907P+4LTXqAQ5eYe14kD6pNOHibiUiue10Ur3CAmqLVVKuNV+ileYAojh2LaBUxuy94TTKAKdpfnWilNUSJ3TCtT6rsha1xlJEswOujU+c/QwdOEe5US/e/qR25isgf4WJgg6Pa3ZC3vs+ZXP7gGfg86g//I+Az8YVE7D3jbUsO22+1+4DXu1vn5w1vjt7vn1bG7tu4sUGkWCrx1hlLHe7TFoHJpj1zXHbofKoVO4R28vRghdu9PXGZ4DuMewV4J1NfIjb6UJVive7K8ImhUesG6oh/8z9dAQ6xlibIeYpW5XhJa6ErAmhecCNUCfprPZb3fXP/90GR4foYLLi7kp8KEd/Kgf9fA4PPZ/ibxUO8BsSYnMDsff/af4Xzr47l7w4U7wdkW/lbdwh7fXOn6FIEzNCtNVOBkUHugEtjVpuVelGgtUy+aOcEEeOVOIjqFVwaJ1o6DfUiTen/Bsyf2AxJKAY2pKmd85NQ+vsNLUQ6gApl/rhafNB275givQ8dBMZurnFiDlXLXGXlPuywKKTIHFnDzax0m2CpppA6q3IKiGTSFwHQ0xE8ZPwtfsNsNsBtFCLgWBYE2fNnQLLpRABMrXgqL4/pIlnzOims7Cga6sVcRjKGZjnuuGT+o6zJ6Ie1UQOGyaWq3q1hLzgugsBTo6G//ERO0kJE1BeaYmRMitq8Y8S5K3q+3xHEU8SWwDMjpCeXExNt9HX0quLm6h95HOR+hrbniO2PENXkJjI+ykWReE3/789tf3r9//gDu5RdKglk8bNAjNz7VZWU1HXps9DJv2e/Q3UEsHCGiH/ABmBAAAwgsAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAGAAAAHhsL3dvcmtzaGVldHMvc2hlZXQyLnhtbL2Y33OjNhDH3/tXMDz07YzBxj9S2zdXQ3o3k2syTa430zcZhNFEICrJdpy/visJMEbnm0wn4xcbvhK7+9kFvOvFx5eCOnvMBWHl0vUHQ9fBZcJSUm6X7ren2w8z1xESlSmirMRL94iF+3H1y+LA+LPIMZYOGCjF0s2lrG48TyQ5LpAYsAqXsJIxXiAJp3zriYpjlOqLCuoFw+HEKxApXWPhhr/FBssykuCIJbsCl9IY4ZgiCeGLnFSisfaSvsleytEBUJt4OiFGZqW1548tewVJOBMsk4OEFXVoNuXcm59xvvDg/1nyQ0DdE1WpoDFWJG+hLBB/3lUfwHYFmdoQSuRRA7urhbb/wJ2MUIn5V5ZCkTNEBYa1Cm3xI5bfKr0un9gDCEtX8h12vdXCq69dLVIC5VCBORxnS/eTfxP7Q7VF7/ib4IPoHDsiZ4dbiG9HkWi8afEPTtI7UuJz9S92WDP6GZIB92ntXi/8gyFrjcDJNocQ73Am26sl2jxiihOJ0zOL9ztJwc3jsdgw2lpIcYZ2VKogwB/jjb6HmJduqRJKwSarlI81plSRuk6i9n4BB5Ox67wyVjwmiEKa/OGwc/6nvryvqoTeoSPb6cTUq+rZ2jD2rCRld6jKpDFUhiuknsM6CtdBoO7xKZrTubnUEf/WNTmVTBnuHjfFudX3DBS7zgRk4TtJZb50Z4PJdD6ZTcM2S1CUz1ilHIIeDcawkOyEZEUjmtS9QoUapc4/M7m/w3tMwYiOsauBU4PsncW0WkCehf5UGaeoEp2iGt91sMZ1TtIUlz90q30W6GXpBuEEjoh69an33FHVbQoVMIb8yWAUqrS9r1N/GIS11yCctn5HZ35HulyG2byQkESrBWcHh+tojXeTntZhXY7AiuRHxbGisxCBXHlT9xU8JTOdCiX83hfWfSHqC3FH8ICiRQmuixLoMObwHMMjDep+5S+8vUp1w2Z2wPtLbRF6y+h8y7re4p9wLSXuKmfAo58Bj9Ud977EI4s46BGbHX7QIR73iEcWsaXEo0vE45+W+J1xx/17ddwPdG0pkaXE40sw4TVhwj5MaMFYSmQpcXgJZnJNmEkfZmLBWEpkKfHkEsz0mjDTPszUgrGUyFLi6SWY2TVhZn2YmQVjKZGlxLNLMPNrwsz7MHMLxlIiS4nnl2DUL8P1aJS33m/v0OKxpciW4jPJIHmd1qLipJT3lR6lnBy6bRh/To341mrNWwVGhLb/YZy8slIiuob5DPNOtwRDpiSJveCZOeMr4lsCjqnu3/26n9dH0Ovq7w2TkEh9mOthADI3CH1/5kNHNZoEwXAMj1PGmDRLQdgYhwh3FXTOFeaP5BWbbq/Tpevhpm7k/Pq07W5dR5m459phyg7lU47Le8CBMnICNHr6XLoV45IjAj35hqLk+VOZfs+JbMcpB2bNzmiSQIe+ZoUaY4UaLsqz7EUVgUZBhdak7aQkrCKqDLqMJhG3mtlJSZZBakt5S7g4uWrl+zSN96c7c7VgaWrGKrgTOsdwaCyufkVF9dtaf/qB/nrQO+vlhdf1D6ft3wKr/wBQSwcIHv/PMpAEAABaEAAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAUAAAAeGwvc2hhcmVkU3RyaW5ncy54bWyNkEFqAjEYhfeeIvx7zShUiiSZheAJ7AHCzO9MYPJnzJ8RPUFv0E3P0L0I3kZ7DtNC6U5m+Xjfew+eKo++EweM7AJpmM8KEEhVqB01Gt62m+krCE6WatsFQg0nZCjNRDEnkaPEGtqU+pWUXLXoLc9Cj5SdXYjepixjI7mPaGtuEZPv5KIoltJbRyCqMFDS8AJiILcfcP2njWJn1O/Eintb5eXcwRgPCMbVSiaj5A/yBCPrcRR4O399f7yPQpGaznE7tvb+eb1dLv+0zLeZB1BLBwih0YAvzgAAAHQBAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAAAsAAABfcmVscy8ucmVsc62Sz0oDMRCH732KkHt3thVEZLO9iNCbSH2AmMz+YTeZMBl1fXuDCFqppQePSX7zzTdDmt0SZvWKnEeKRm+qWiuMjvwYe6OfDvfrG71rV80jzlZKJA9jyqrUxGz0IJJuAbIbMNhcUcJYXjriYKUcuYdk3WR7hG1dXwP/ZOj2iKn23mje+41Wh/eEl7Cp60aHd+ReAkY50eJXopAt9yhGLzO8EU/PRFNVoBpOu2wvd/l7Tggo1lux4IhxnbhUs4yYv3U8uYdynT8T54Su/nM5uAhGj/68kk3py2jVwNEnaD8AUEsHCGaqgrfgAAAAOwIAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAEQAAAGRvY1Byb3BzL2NvcmUueG1sbVLJbsIwEL33KyLfE8fQNUqC1FacilQJqla9ufYQ3MaOZQ8E/r5OgBRUbvMWv/GMnU+2uo424LxqTEFYkpIIjGikMlVB3hbT+J5EHrmRvG4MFGQHnkzKq1zYTDQOXl1jwaECH4Ug4zNhC7JCtBmlXqxAc58EhwnisnGaY4CuopaLH14BHaXpLdWAXHLktAuM7ZBIDpFSDJF27eo+QAoKNWgw6ClLGP3zIjjtLx7olROnVrizcNF6FAf31qvB2LZt0o57a7g/ox+zl3k/aqxMtyoBpMylyIQDjo0rc3oKQi3BC6cshpXvxTMi4Jqbah32U4KJ3+a9ZaC6zdfc4yy80VKBfNyFjAvcYROZPnBRGCHbD3yU3sdPz4spKUfp6CZmaczYgrHs+iFjd59d0/OAvrODjeq+Ssn6pgPsbu3XX98gcD/SAEKNCmvY08fy3/cpfwFQSwcI+t3l+lIBAACKAgAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAQAAAAZG9jUHJvcHMvYXBwLnhtbJ2QTW/CMAyG7/sVVcS1SSm0QigN2jTthLQdOrRblaUuZMqXkhSVf78AGnCeT/Zr67H90s2kVXYEH6Q1DZrjAmVghO2l2Tfos33LVygLkZueK2ugQScIaMOe6Ie3DnyUELJEMKFBhxjdmpAgDqB5wKltUmewXvOYSr8ndhikgFcrRg0mkrIoagJTBNNDn7sbEF2J62P8L7S34nxf2LUnl3iMtqCd4hEYJfe0tZGrVmpgRZJvBX12TknBY3KEbeW3h/fLClJWuMRLvJhtpRmn7mtVd/Uye5jo0g8/ICKpymL2MkrV5wtKHnFn9u5qNptXuEhxGfjTKLn7yn4BUEsHCF/Olp77AAAAnAEAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAEwAAAFtDb250ZW50X1R5cGVzXS54bWzFVTtPwzAQ3vsrIq8ocdsBIZSkA48RKlFmZOJLYprYlu2W9t9zdqAqpQ+qVrDEiu++x13sSzpatE00B2OFkhkZJH0SgSwUF7LKyPPkPr4io7yXTpYabIS50makdk5fU2qLGlpmE6VBYqRUpmUOX01FNSumrAI67PcvaaGkA+li5zlInt5CyWaNi+4WuN3pIpxEN12el8oI07oRBXMYpj5Kt+IMNHYPcC75hrv401mCyJBja6HtxW4FLasNAdH6yvz+dsSbhu2QEEDMI7bbCA7RmBn3wFpMoIuGvvhi6Lsy01elpglaSs5c3g7hdcnj1FRZigK4KmYtQhKrDTBuawCH5sOatEzIA/oOjxF0z8HJHgLNAUHrlg3Yc5cbSH/R6gCwNCyn1/vdxIr/SB/Df/Jha2aAPzmD4+bsH2Sde5+P7uL9xWVDp2OjtMWRaOD4cr/0PDrWSATGif1nbqWI1Cf3F/yQ48B/avdSGv4Q+QdQSwcI82Bc924BAABQBgAAUEsBAhQAFAAICAgAKFZLW7LO5WzpAAAANgMAABoAAAAAAAAAAAAAAAAAAAAAAHhsL19yZWxzL3dvcmtib29rLnhtbC5yZWxzUEsBAhQAFAAICAgAKFZLW2GY8aIcAgAAtwMAAA8AAAAAAAAAAAAAAAAAMQEAAHhsL3dvcmtib29rLnhtbFBLAQIUABQACAgIAChWS1v2sPGCHgIAANEIAAATAAAAAAAAAAAAAAAAAIoDAAB4bC90aGVtZS90aGVtZTEueG1sUEsBAhQAFAAICAgAKFZLW4ktFQ5QAwAAxSEAAA0AAAAAAAAAAAAAAAAA6QUAAHhsL3N0eWxlcy54bWxQSwECFAAUAAgICAAoVktbaIf8AGYEAADCCwAAGAAAAAAAAAAAAAAAAAB0CQAAeGwvd29ya3NoZWV0cy9zaGVldDEueG1sUEsBAhQAFAAICAgAKFZLWx7/zzKQBAAAWhAAABgAAAAAAAAAAAAAAAAAIA4AAHhsL3dvcmtzaGVldHMvc2hlZXQyLnhtbFBLAQIUABQACAgIAChWS1uh0YAvzgAAAHQBAAAUAAAAAAAAAAAAAAAAAPYSAAB4bC9zaGFyZWRTdHJpbmdzLnhtbFBLAQIUABQACAgIAChWS1tmqoK34AAAADsCAAALAAAAAAAAAAAAAAAAAAYUAABfcmVscy8ucmVsc1BLAQIUABQACAgIAChWS1v63eX6UgEAAIoCAAARAAAAAAAAAAAAAAAAAB8VAABkb2NQcm9wcy9jb3JlLnhtbFBLAQIUABQACAgIAChWS1tfzpae+wAAAJwBAAAQAAAAAAAAAAAAAAAAALAWAABkb2NQcm9wcy9hcHAueG1sUEsBAhQAFAAICAgAKFZLW/NgXPduAQAAUAYAABMAAAAAAAAAAAAAAAAA6RcAAFtDb250ZW50X1R5cGVzXS54bWxQSwUGAAAAAAsACwDGAgAAmBkAAAAA" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "7284" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:39 GMT" - }, - { - "name": "etag", - "value": "W/\"8358afe5b3c08d495ad280e2822044f257db2c761c4adee5df64cb2c957dc80b\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:39 GMT" - }, - { - "name": "source-age", - "value": "137" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "1" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "86c5440c6dfc71eff30a9f58d76564ca648b3008" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "70F2:2E0B15:7D2488:95C02A:68EA3D75" - }, - { - "name": "x-served-by", - "value": "cache-lis1490049-LIS" - }, - { - "name": "x-timer", - "value": "S1760181759.276641,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 877, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:39.242Z", - "time": 129, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 129 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/xlxs/table/fixtures/generated/loadXlsxTable-file-variations-should-load-remote-file_504643743/recording.har b/table/plugins/xlxs/table/fixtures/generated/loadXlsxTable-file-variations-should-load-remote-file_504643743/recording.har deleted file mode 100644 index 138fe512..00000000 --- a/table/plugins/xlxs/table/fixtures/generated/loadXlsxTable-file-variations-should-load-remote-file_504643743/recording.har +++ /dev/null @@ -1,153 +0,0 @@ -{ - "log": { - "_recordingName": "loadXlsxTable-file variations-should load remote file", - "creator": { - "comment": "persister:fs", - "name": "Polly.JS", - "version": "6.0.6" - }, - "entries": [ - { - "_id": "25af28f3d42b3c31469c3da0dee974ce", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [], - "headersSize": 114, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/xlsx/table/fixtures/table.xlsx" - }, - "response": { - "bodySize": 7284, - "content": { - "encoding": "base64", - "mimeType": "application/octet-stream", - "size": 7284, - "text": "UEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAaAAAAeGwvX3JlbHMvd29ya2Jvb2sueG1sLnJlbHO9kstqwzAQRff5CjH7WLb7oBTL2ZRCtm36AUIeWya2JEbTR/6+alMaB4LpwnQl5kpz70Ez1eZjHMQbUuy9U1BkOQh0xje96xS87B7Xd7CpV9UTDprTk2j7EEXqcVGBZQ73UkZjcdQx8wFdumk9jZpTSZ0M2ux1h7LM81tJUw+ozzzFtlFA26YAsTsE/Iu3b9ve4IM3ryM6vhAhOfViMtTUISv4Lo9ikSUzkJcZyiUZIh8GjCeIYz0Xf7Vk/LunfbSIfCL4lRLc1zH7F9f/DFPOwdwsOhirCZtnprTp0/lM5R+YVSXP9r/+BFBLBwiyzuVs6QAAADYDAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAAA8AAAB4bC93b3JrYm9vay54bWyNU0tu2zAQ3fcUAve2Pv7UNiwHrmwhAfpDnCZrShpZrClSIGnLbtFFe41epMdIbtQRZaUuWhRdSCLn8+bNzNP86lhy5wBKMylC4vc94oBIZcbENiQf7uLehDjaUJFRLgWE5ASaXC1ezGupdomUOwfzhQ5JYUw1c12dFlBS3ZcVCPTkUpXU4FVtXV0poJkuAEzJ3cDzxm5JmSAtwkz9D4bMc5bCSqb7EoRpQRRwapC9LlilyWKeMw73bUMOraq3tETaEeUpcRfPtN8rJ6Hpbl/FGB2SnHIN2Ggh63fJR0gNdkQ5J05GDfhTb9iF/AYhDUZiGTQ2hnsGtf7lb64W8Voq9kkKQ/kmVZLzkBi1P1dDooalf/NsmkHd0UR3xuMDE5msQ4IrOl2ca3t8YJkpcIHjwWTY2a6BbQsTkok/DYhjaHLbDCokIw/Tcqa0sUUsCsVODoD1mhs25F50ZHfWfR1hB/r4/fHH09enb37DFs03GRa3UjHoPTDNEo6k1YyhQ91kgwb0EsCWdoKL9OAf6UPLqSOCA0txg8yAwvhI7gU24TddKcjfyAwhlkjm7H9e7/m+Am4ottn3PL+BhaN5rY39nrXIJZ7/0CNniYJWgVaMxNkrFpLPL8fBOJqMg16w9Ac931+Peq8Gw1EvXscxjj5aRdP4CwrTos7wiVr62ij8y24h35xQHMdWpEt/fUyBLy0zF4PbtyXodtJa/ARQSwcIYZjxohwCAAC3AwAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAATAAAAeGwvdGhlbWUvdGhlbWUxLnhtbN2VTW/bMAyG7/sVgu6r4rgJ0iBOMSwLdiiwQ7bdGZm21UiyIant8u+nyE7ir6HDMGDofIlIPXxFioy9uv+hJHlGY0WpExrdTChBzctU6Dyh375u3y8osQ50CrLUmNAjWnq/freCpStQIfHh2i4hoYVz1ZIxy70b7E1ZofZ7WWkUOG+anKUGXryskmw6mcyZAqFpE29+J77MMsFxU/InhdrVIgYlOJ+6LURlKdGgfI5fAkjX5yQ/STxF2JODS7PjIfOafRB7g62A9BCdfqzJ9x+lIc8gEzoJD2XrFbsA0g25LDwN1wDpYfqa3rTWG3I9vQAA576U4dnRAuJJ3LAtqF6O5BDP76DLt/TjAQ9xjD39+MrfDviFp3v6t1d+NuD53R2/3EkLqpfzEX4aRdjhA1RIoQ+jN45n+oJkpfw8is9mESz2DX6lWGt86njtOsPUmiMFj6XZeiA018+oJu5YYQbccx+MAElJJRwvtqCEPPoUKeEFGIvON/N0NCwRWjEbfITvT2QH2r4eye2fRbJe4kroN1rFNXHWblRom2obQsqdO0p8sKFIW0qRbr0zGAG7jEVV+CUNiped2uoE/XMFNixL6q5FXhI6j2enq4PKv2l8b/1SVWlCrc4pAZn7zwF3JgxzZazbgC3qFMJJdYeUcGia95N+m8qsfzmYZcjdLzxX0+/VIqO7fx9mY5nt8+3/Ob/9wljnb8sGH/azZ/0TUEsHCPaw8YIeAgAA0QgAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAADQAAAHhsL3N0eWxlcy54bWztWt1u2jAYvd9TWL5fEyjQdQrpOiqmbdJUDTpNmnZhEifx6tiRY1ro08+Ok5CU0FG6rjCFm8Qn3/l8cvwLxjlbxBTcYJESzoawc2RDgJnHfcLCIbyajl+/gSCViPmIcoaHcIlTeOa+clK5pHgSYSyBysDSIYykTN5aVupFOEbpEU8wU08CLmIkVVGEVpoIjPxUk2JqdW17YMWIMOg6bB6PY5kCj8+ZHMLjEgLm8tFX2gY9CEy6EfeVlA+YYYEotBqD+/XgdxvCBvUwW4dZuRzXCThbqRpAA7hOegduEFX0LJyhGJvyuSBGT4BiQpcG7GrAi5BIlVeGllViUu2Q0D4Q+gX+hb7NwQSxtNGTrZJ84ZJnKcDo02cwGT2UKbvoViOU1vqSBlwnQVJiwcaqAPL76TJRjc5UzzZpsrg/RIcCLTvd/vaElFPiaxXhiFMugAhnQzjOPzrNbP3BKHtRq5KzrC27qLecceGrcVu8ZxcWEPAJCjlD9CoZwgDRFMMSuuC3rABdh+JAqmoECSN9lTzRariUPFY3BUcLMZl3qwFkc8UQyoh417Cobw1Vta9hRssa3KQsv1HGeJjSiWZ8D1bu2ErMIlifTlhWULOedjW/NZnyAkoSuhxznUSKOc6B91lIDTqnJGQxvhd4KbjEnsxm1wx2HVQEgogLcqdS644c5rOZnowl8TRk3h4CiRfyK5fIZFGabgVKpgoszSfMzypWz9JIEHY95WNSPlY2JaUMQLl3jf1CZER8Ra1EWovgnlP2yqfOrj7lOu8bVYWrThXd53DEdFsxG8TsPLZaMa2YVkwrphWzi5je8T6tlL3OXqnp7ZWa7j6pOX1hMVZ1+24285V9fOd41338IliXXhX0RO2Htql/im3bt/hBfBHKqc9s2iP6Wuvaf+vaI8Znv9G0btW0ztbj8+/9TuEpAIv97GeD1rInDM3eP14FnmPpNFW1K+fezWbNnr340Nxny9rZrF0y92Aqa4flxk7WbztZo2NW/q29chZXO0UvUaAPevUhr4h1a5emzeaESsJMyVonjHgcoyK+068RjjcSwA/7Z0ka1EiDRtJcCMy8Zck5qXF6D3Fqdb2p8U6aeJdY6BYsKac1ijltXpmpCqs/YLi/AVBLBwiJLRUOUAMAAMUhAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAABgAAAB4bC93b3Jrc2hlZXRzL3NoZWV0MS54bWy9Vt1u2zYUvt9TaMIQbEBq/Th2nER2kTr1Wsytgzhpgd3REmURoUiVpOw4d91r7EX2GO0b7ZCUZMU2hmDI5gvbPCS/853vkDwnev2QU2eFhSScDd2g47sOZjFPCFsO3bvbyauB60iFWIIoZ3jobrB0X49+iNZc3MsMY+UAAJNDN1OqOPc8GWc4R7LDC8xgJuUiRwqGYunJQmCUmE059ULf73s5Isy1COfiORg8TUmMr3hc5pgpCyIwRQroy4wUskZ7SJ6Flwi0hlBrPi2KV3amwQtO9vByEgsueao6Mc8ravtRnnlnT+J8EOG/Qwp6EOqK6EyFNVgePyfKHIn7sngF2AUotSCUqI0J2B1FBv9aOCmhCosPPIEkp4hKDHMFWuI5VneFmVe3/BoM9bQ3irxq8yhKCORDM3METofuZXD+pqtXmAWfCF7L1n9HZnw9AX4lRbKGM8ZfBUmmhOGn1hu+HnP6DsSAczp0lSirid8xqFYbBFlmQHGKU9XsVmgxxxTHCiftfbNSUfAy3+QLThuABKeopEpzAHdc1PYVUB66TOtJAZIX2sUYU6rjdJ1Yr30P+P0T13nkPJ/HiIJKge+3xh/N9l2r1nOKNrw0ulSz+motOL/XJo3r6yyZKLS+BdLXsGLhOgisK2zZgOStsd3qyC8mIzYdXpOD9v86NxNzZCDXlRKgwmeSqGzoDjr907P+4LTXqAQ5eYe14kD6pNOHibiUiue10Ur3CAmqLVVKuNV+ileYAojh2LaBUxuy94TTKAKdpfnWilNUSJ3TCtT6rsha1xlJEswOujU+c/QwdOEe5US/e/qR25isgf4WJgg6Pa3ZC3vs+ZXP7gGfg86g//I+Az8YVE7D3jbUsO22+1+4DXu1vn5w1vjt7vn1bG7tu4sUGkWCrx1hlLHe7TFoHJpj1zXHbofKoVO4R28vRghdu9PXGZ4DuMewV4J1NfIjb6UJVive7K8ImhUesG6oh/8z9dAQ6xlibIeYpW5XhJa6ErAmhecCNUCfprPZb3fXP/90GR4foYLLi7kp8KEd/Kgf9fA4PPZ/ibxUO8BsSYnMDsff/af4Xzr47l7w4U7wdkW/lbdwh7fXOn6FIEzNCtNVOBkUHugEtjVpuVelGgtUy+aOcEEeOVOIjqFVwaJ1o6DfUiTen/Bsyf2AxJKAY2pKmd85NQ+vsNLUQ6gApl/rhafNB275givQ8dBMZurnFiDlXLXGXlPuywKKTIHFnDzax0m2CpppA6q3IKiGTSFwHQ0xE8ZPwtfsNsNsBtFCLgWBYE2fNnQLLpRABMrXgqL4/pIlnzOims7Cga6sVcRjKGZjnuuGT+o6zJ6Ie1UQOGyaWq3q1hLzgugsBTo6G//ERO0kJE1BeaYmRMitq8Y8S5K3q+3xHEU8SWwDMjpCeXExNt9HX0quLm6h95HOR+hrbniO2PENXkJjI+ykWReE3/789tf3r9//gDu5RdKglk8bNAjNz7VZWU1HXps9DJv2e/Q3UEsHCGiH/ABmBAAAwgsAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAGAAAAHhsL3dvcmtzaGVldHMvc2hlZXQyLnhtbL2Y33OjNhDH3/tXMDz07YzBxj9S2zdXQ3o3k2syTa430zcZhNFEICrJdpy/visJMEbnm0wn4xcbvhK7+9kFvOvFx5eCOnvMBWHl0vUHQ9fBZcJSUm6X7ren2w8z1xESlSmirMRL94iF+3H1y+LA+LPIMZYOGCjF0s2lrG48TyQ5LpAYsAqXsJIxXiAJp3zriYpjlOqLCuoFw+HEKxApXWPhhr/FBssykuCIJbsCl9IY4ZgiCeGLnFSisfaSvsleytEBUJt4OiFGZqW1548tewVJOBMsk4OEFXVoNuXcm59xvvDg/1nyQ0DdE1WpoDFWJG+hLBB/3lUfwHYFmdoQSuRRA7urhbb/wJ2MUIn5V5ZCkTNEBYa1Cm3xI5bfKr0un9gDCEtX8h12vdXCq69dLVIC5VCBORxnS/eTfxP7Q7VF7/ib4IPoHDsiZ4dbiG9HkWi8afEPTtI7UuJz9S92WDP6GZIB92ntXi/8gyFrjcDJNocQ73Am26sl2jxiihOJ0zOL9ztJwc3jsdgw2lpIcYZ2VKogwB/jjb6HmJduqRJKwSarlI81plSRuk6i9n4BB5Ox67wyVjwmiEKa/OGwc/6nvryvqoTeoSPb6cTUq+rZ2jD2rCRld6jKpDFUhiuknsM6CtdBoO7xKZrTubnUEf/WNTmVTBnuHjfFudX3DBS7zgRk4TtJZb50Z4PJdD6ZTcM2S1CUz1ilHIIeDcawkOyEZEUjmtS9QoUapc4/M7m/w3tMwYiOsauBU4PsncW0WkCehf5UGaeoEp2iGt91sMZ1TtIUlz90q30W6GXpBuEEjoh69an33FHVbQoVMIb8yWAUqrS9r1N/GIS11yCctn5HZ35HulyG2byQkESrBWcHh+tojXeTntZhXY7AiuRHxbGisxCBXHlT9xU8JTOdCiX83hfWfSHqC3FH8ICiRQmuixLoMObwHMMjDep+5S+8vUp1w2Z2wPtLbRF6y+h8y7re4p9wLSXuKmfAo58Bj9Ud977EI4s46BGbHX7QIR73iEcWsaXEo0vE45+W+J1xx/17ddwPdG0pkaXE40sw4TVhwj5MaMFYSmQpcXgJZnJNmEkfZmLBWEpkKfHkEsz0mjDTPszUgrGUyFLi6SWY2TVhZn2YmQVjKZGlxLNLMPNrwsz7MHMLxlIiS4nnl2DUL8P1aJS33m/v0OKxpciW4jPJIHmd1qLipJT3lR6lnBy6bRh/To341mrNWwVGhLb/YZy8slIiuob5DPNOtwRDpiSJveCZOeMr4lsCjqnu3/26n9dH0Ovq7w2TkEh9mOthADI3CH1/5kNHNZoEwXAMj1PGmDRLQdgYhwh3FXTOFeaP5BWbbq/Tpevhpm7k/Pq07W5dR5m459phyg7lU47Le8CBMnICNHr6XLoV45IjAj35hqLk+VOZfs+JbMcpB2bNzmiSQIe+ZoUaY4UaLsqz7EUVgUZBhdak7aQkrCKqDLqMJhG3mtlJSZZBakt5S7g4uWrl+zSN96c7c7VgaWrGKrgTOsdwaCyufkVF9dtaf/qB/nrQO+vlhdf1D6ft3wKr/wBQSwcIHv/PMpAEAABaEAAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAUAAAAeGwvc2hhcmVkU3RyaW5ncy54bWyNkEFqAjEYhfeeIvx7zShUiiSZheAJ7AHCzO9MYPJnzJ8RPUFv0E3P0L0I3kZ7DtNC6U5m+Xjfew+eKo++EweM7AJpmM8KEEhVqB01Gt62m+krCE6WatsFQg0nZCjNRDEnkaPEGtqU+pWUXLXoLc9Cj5SdXYjepixjI7mPaGtuEZPv5KIoltJbRyCqMFDS8AJiILcfcP2njWJn1O/Eintb5eXcwRgPCMbVSiaj5A/yBCPrcRR4O399f7yPQpGaznE7tvb+eb1dLv+0zLeZB1BLBwih0YAvzgAAAHQBAABQSwMEFAAICAgAKFZLWwAAAAAAAAAAAAAAAAsAAABfcmVscy8ucmVsc62Sz0oDMRCH732KkHt3thVEZLO9iNCbSH2AmMz+YTeZMBl1fXuDCFqppQePSX7zzTdDmt0SZvWKnEeKRm+qWiuMjvwYe6OfDvfrG71rV80jzlZKJA9jyqrUxGz0IJJuAbIbMNhcUcJYXjriYKUcuYdk3WR7hG1dXwP/ZOj2iKn23mje+41Wh/eEl7Cp60aHd+ReAkY50eJXopAt9yhGLzO8EU/PRFNVoBpOu2wvd/l7Tggo1lux4IhxnbhUs4yYv3U8uYdynT8T54Su/nM5uAhGj/68kk3py2jVwNEnaD8AUEsHCGaqgrfgAAAAOwIAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAEQAAAGRvY1Byb3BzL2NvcmUueG1sbVLJbsIwEL33KyLfE8fQNUqC1FacilQJqla9ufYQ3MaOZQ8E/r5OgBRUbvMWv/GMnU+2uo424LxqTEFYkpIIjGikMlVB3hbT+J5EHrmRvG4MFGQHnkzKq1zYTDQOXl1jwaECH4Ug4zNhC7JCtBmlXqxAc58EhwnisnGaY4CuopaLH14BHaXpLdWAXHLktAuM7ZBIDpFSDJF27eo+QAoKNWgw6ClLGP3zIjjtLx7olROnVrizcNF6FAf31qvB2LZt0o57a7g/ox+zl3k/aqxMtyoBpMylyIQDjo0rc3oKQi3BC6cshpXvxTMi4Jqbah32U4KJ3+a9ZaC6zdfc4yy80VKBfNyFjAvcYROZPnBRGCHbD3yU3sdPz4spKUfp6CZmaczYgrHs+iFjd59d0/OAvrODjeq+Ssn6pgPsbu3XX98gcD/SAEKNCmvY08fy3/cpfwFQSwcI+t3l+lIBAACKAgAAUEsDBBQACAgIAChWS1sAAAAAAAAAAAAAAAAQAAAAZG9jUHJvcHMvYXBwLnhtbJ2QTW/CMAyG7/sVVcS1SSm0QigN2jTthLQdOrRblaUuZMqXkhSVf78AGnCeT/Zr67H90s2kVXYEH6Q1DZrjAmVghO2l2Tfos33LVygLkZueK2ugQScIaMOe6Ie3DnyUELJEMKFBhxjdmpAgDqB5wKltUmewXvOYSr8ndhikgFcrRg0mkrIoagJTBNNDn7sbEF2J62P8L7S34nxf2LUnl3iMtqCd4hEYJfe0tZGrVmpgRZJvBX12TknBY3KEbeW3h/fLClJWuMRLvJhtpRmn7mtVd/Uye5jo0g8/ICKpymL2MkrV5wtKHnFn9u5qNptXuEhxGfjTKLn7yn4BUEsHCF/Olp77AAAAnAEAAFBLAwQUAAgICAAoVktbAAAAAAAAAAAAAAAAEwAAAFtDb250ZW50X1R5cGVzXS54bWzFVTtPwzAQ3vsrIq8ocdsBIZSkA48RKlFmZOJLYprYlu2W9t9zdqAqpQ+qVrDEiu++x13sSzpatE00B2OFkhkZJH0SgSwUF7LKyPPkPr4io7yXTpYabIS50makdk5fU2qLGlpmE6VBYqRUpmUOX01FNSumrAI67PcvaaGkA+li5zlInt5CyWaNi+4WuN3pIpxEN12el8oI07oRBXMYpj5Kt+IMNHYPcC75hrv401mCyJBja6HtxW4FLasNAdH6yvz+dsSbhu2QEEDMI7bbCA7RmBn3wFpMoIuGvvhi6Lsy01elpglaSs5c3g7hdcnj1FRZigK4KmYtQhKrDTBuawCH5sOatEzIA/oOjxF0z8HJHgLNAUHrlg3Yc5cbSH/R6gCwNCyn1/vdxIr/SB/Df/Jha2aAPzmD4+bsH2Sde5+P7uL9xWVDp2OjtMWRaOD4cr/0PDrWSATGif1nbqWI1Cf3F/yQ48B/avdSGv4Q+QdQSwcI82Bc924BAABQBgAAUEsBAhQAFAAICAgAKFZLW7LO5WzpAAAANgMAABoAAAAAAAAAAAAAAAAAAAAAAHhsL19yZWxzL3dvcmtib29rLnhtbC5yZWxzUEsBAhQAFAAICAgAKFZLW2GY8aIcAgAAtwMAAA8AAAAAAAAAAAAAAAAAMQEAAHhsL3dvcmtib29rLnhtbFBLAQIUABQACAgIAChWS1v2sPGCHgIAANEIAAATAAAAAAAAAAAAAAAAAIoDAAB4bC90aGVtZS90aGVtZTEueG1sUEsBAhQAFAAICAgAKFZLW4ktFQ5QAwAAxSEAAA0AAAAAAAAAAAAAAAAA6QUAAHhsL3N0eWxlcy54bWxQSwECFAAUAAgICAAoVktbaIf8AGYEAADCCwAAGAAAAAAAAAAAAAAAAAB0CQAAeGwvd29ya3NoZWV0cy9zaGVldDEueG1sUEsBAhQAFAAICAgAKFZLWx7/zzKQBAAAWhAAABgAAAAAAAAAAAAAAAAAIA4AAHhsL3dvcmtzaGVldHMvc2hlZXQyLnhtbFBLAQIUABQACAgIAChWS1uh0YAvzgAAAHQBAAAUAAAAAAAAAAAAAAAAAPYSAAB4bC9zaGFyZWRTdHJpbmdzLnhtbFBLAQIUABQACAgIAChWS1tmqoK34AAAADsCAAALAAAAAAAAAAAAAAAAAAYUAABfcmVscy8ucmVsc1BLAQIUABQACAgIAChWS1v63eX6UgEAAIoCAAARAAAAAAAAAAAAAAAAAB8VAABkb2NQcm9wcy9jb3JlLnhtbFBLAQIUABQACAgIAChWS1tfzpae+wAAAJwBAAAQAAAAAAAAAAAAAAAAALAWAABkb2NQcm9wcy9hcHAueG1sUEsBAhQAFAAICAgAKFZLW/NgXPduAQAAUAYAABMAAAAAAAAAAAAAAAAA6RcAAFtDb250ZW50X1R5cGVzXS54bWxQSwUGAAAAAAsACwDGAgAAmBkAAAAA" - }, - "cookies": [], - "headers": [ - { - "name": "accept-ranges", - "value": "bytes" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "cache-control", - "value": "max-age=300" - }, - { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "7284" - }, - { - "name": "content-security-policy", - "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" - }, - { - "name": "content-type", - "value": "application/octet-stream" - }, - { - "name": "cross-origin-resource-policy", - "value": "cross-origin" - }, - { - "name": "date", - "value": "Sat, 11 Oct 2025 11:22:39 GMT" - }, - { - "name": "etag", - "value": "W/\"8358afe5b3c08d495ad280e2822044f257db2c761c4adee5df64cb2c957dc80b\"" - }, - { - "name": "expires", - "value": "Sat, 11 Oct 2025 11:27:39 GMT" - }, - { - "name": "source-age", - "value": "137" - }, - { - "name": "strict-transport-security", - "value": "max-age=31536000" - }, - { - "name": "vary", - "value": "Authorization,Accept-Encoding" - }, - { - "name": "via", - "value": "1.1 varnish" - }, - { - "name": "x-cache", - "value": "HIT" - }, - { - "name": "x-cache-hits", - "value": "0" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "x-fastly-request-id", - "value": "5503b0dcb8355da577584344e547d168c43d4baf" - }, - { - "name": "x-frame-options", - "value": "deny" - }, - { - "name": "x-github-request-id", - "value": "70F2:2E0B15:7D2488:95C02A:68EA3D75" - }, - { - "name": "x-served-by", - "value": "cache-lis1490034-LIS" - }, - { - "name": "x-timer", - "value": "S1760181759.076650,VS0,VE1" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - } - ], - "headersSize": 877, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2025-10-11T11:22:39.021Z", - "time": 157, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 157 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/table/plugins/xlxs/table/fixtures/table.xlsx b/table/plugins/xlxs/table/fixtures/table.xlsx deleted file mode 100644 index dacbc2ed..00000000 Binary files a/table/plugins/xlxs/table/fixtures/table.xlsx and /dev/null differ diff --git a/table/plugins/xlxs/table/index.ts b/table/plugins/xlxs/table/index.ts deleted file mode 100644 index e300b5a3..00000000 --- a/table/plugins/xlxs/table/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { loadXlsxTable } from "./load.ts" -export { saveXlsxTable } from "./save.ts" diff --git a/table/plugins/xlxs/table/load.spec.ts b/table/plugins/xlxs/table/load.spec.ts deleted file mode 100644 index 16f57539..00000000 --- a/table/plugins/xlxs/table/load.spec.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import { describe, expect, it } from "vitest" -import { useRecording } from "vitest-polly" -import { loadXlsxTable } from "./load.ts" -import { writeTestData } from "./test.ts" - -useRecording() - -const row1 = ["id", "name"] -const row2 = [1, "english"] -const row3 = [2, "中文"] - -const record1 = { id: 1, name: "english" } -const record2 = { id: 2, name: "中文" } - -describe("loadXlsxTable", () => { - describe("file variations", () => { - it("should load local file", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadXlsxTable({ path }) - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should load local file (multipart)", async () => { - const path1 = getTempFilePath() - const path2 = getTempFilePath() - await writeTestData(path1, [row1, row2, row3]) - await writeTestData(path2, [row1, row2, row3]) - - const table = await loadXlsxTable({ path: [path1, path2] }) - expect((await table.collect()).toRecords()).toEqual([ - record1, - record2, - record1, - record2, - ]) - }) - - it("should load remote file", async () => { - const table = await loadXlsxTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/xlsx/table/fixtures/table.xlsx", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - - it("should load multipart remote file", async () => { - const table = await loadXlsxTable({ - path: [ - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/xlsx/table/fixtures/table.xlsx", - "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/xlsx/table/fixtures/table.xlsx", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ]) - }) - }) - - describe("dialect variations", () => { - it("should support sheetNumber", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3], { sheetNumber: 2 }) - - const table = await loadXlsxTable({ - path, - dialect: { sheetNumber: 2 }, - }) - - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should support sheetName", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3], { sheetName: "Sheet2" }) - - const table = await loadXlsxTable({ - path, - dialect: { sheetName: "Sheet2" }, - }) - - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should support no header", async () => { - const path = getTempFilePath() - await writeTestData(path, [row2, row3]) - - const table = await loadXlsxTable({ - path, - dialect: { header: false }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { field1: 1, field2: "english" }, - { field1: 2, field2: "中文" }, - ]) - }) - - it("should support headerRows offset", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadXlsxTable({ - path, - dialect: { headerRows: [2] }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { 1: 2, english: "中文" }, - ]) - }) - - it("should support multiline headerRows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadXlsxTable({ - path, - dialect: { headerRows: [1, 2] }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { "id 1": 2, "name english": "中文" }, - ]) - }) - - it("should support headerJoin", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadXlsxTable({ - path, - dialect: { headerRows: [1, 2], headerJoin: "-" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { "id-1": 2, "name-english": "中文" }, - ]) - }) - - it("should support commentRows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3]) - - const table = await loadXlsxTable({ - path, - dialect: { commentRows: [2] }, - }) - - expect((await table.collect()).toRecords()).toEqual([record2]) - }) - - it("should support commentChar", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3, ["#comment"]]) - - const table = await loadXlsxTable({ - path, - dialect: { commentChar: "#" }, - }) - - expect((await table.collect()).toRecords()).toEqual([record1, record2]) - }) - - it("should handle longer rows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3, [3, "german", "bad"]]) - - const table = await loadXlsxTable({ - path, - dialect: { commentChar: "#" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - record1, - record2, - { id: 3, name: "german" }, - ]) - }) - - it("should handle shorter rows", async () => { - const path = getTempFilePath() - await writeTestData(path, [row1, row2, row3, [3]]) - - const table = await loadXlsxTable({ - path, - dialect: { commentChar: "#" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - record1, - record2, - { id: 3, name: null }, - ]) - }) - }) -}) diff --git a/table/plugins/xlxs/table/load.ts b/table/plugins/xlxs/table/load.ts deleted file mode 100644 index b37f686c..00000000 --- a/table/plugins/xlxs/table/load.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { resolveDialect } from "@dpkit/metadata" -import type { Resource } from "@dpkit/metadata" -import { resolveSchema } from "@dpkit/metadata" -import { loadFile, prefetchFiles } from "@dpkit/dataset" -import type { DataRow } from "../../../data/index.ts" -import { getRecordsFromRows } from "../../../data/index.ts" -import type { LoadTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { normalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" -import * as pl from "nodejs-polars" -import { read, utils } from "xlsx" - -// Currently, we use slow non-rust implementation as in the future -// polars-rust might be able to provide a faster native implementation - -export async function loadXlsxTable( - resource: Partial, - options?: LoadTableOptions, -) { - const paths = await prefetchFiles(resource.path) - if (!paths.length) { - throw new Error("Resource path is not defined") - } - - const dialect = await resolveDialect(resource.dialect) - - const tables: Table[] = [] - for (const path of paths) { - const buffer = await loadFile(path) - - const book = read(buffer, { type: "buffer" }) - const sheetIndex = dialect?.sheetNumber ? dialect.sheetNumber - 1 : 0 - const sheetName = dialect?.sheetName ?? book.SheetNames[sheetIndex] - const sheet = sheetName ? book.Sheets[sheetName] : undefined - - if (sheet) { - const rows = utils.sheet_to_json(sheet, { - header: 1, - raw: true, - }) as DataRow[] - - const records = getRecordsFromRows(rows, dialect) - const table = pl.DataFrame(records).lazy() - - tables.push(table) - } - } - - let table = pl.concat(tables) - - if (!options?.denormalized) { - let schema = await resolveSchema(resource.schema) - if (!schema) schema = await inferSchemaFromTable(table, options) - table = await normalizeTable(table, schema) - } - - return table -} diff --git a/table/plugins/xlxs/table/save.spec.ts b/table/plugins/xlxs/table/save.spec.ts deleted file mode 100644 index 423e1882..00000000 --- a/table/plugins/xlxs/table/save.spec.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { getTempFilePath } from "@dpkit/dataset" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { loadXlsxTable } from "./load.ts" -import { saveXlsxTable } from "./save.ts" -import { readTestData } from "./test.ts" - -const row1 = { id: 1, name: "english" } -const row2 = { id: 2, name: "中文" } -const table = pl.readRecords([row1, row2]).lazy() - -describe("saveXlsxTable", () => { - it("should save table to file", async () => { - const path = getTempFilePath() - await saveXlsxTable(table, { path }) - - const data = await readTestData(path) - expect(data).toEqual([row1, row2]) - }) - - it("should save and load various data types", async () => { - const path = getTempFilePath() - - const source = pl - .DataFrame([ - pl.Series("array", ["[1, 2, 3]"], pl.String), - pl.Series("boolean", [true], pl.Bool), - pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date), - pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime), - pl.Series("duration", ["P23DT23H"], pl.String), - pl.Series("geojson", ['{"value": 1}'], pl.String), - pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)), - pl.Series("integer", [1], pl.Int32), - pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)), - pl.Series("number", [1.1], pl.Float64), - pl.Series("object", ['{"value": 1}']), - pl.Series("string", ["string"], pl.String), - pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time), - pl.Series("year", [2025], pl.Int32), - pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)), - ]) - .lazy() - - await saveXlsxTable(source, { - path, - fieldTypes: { - array: "array", - geojson: "geojson", - geopoint: "geopoint", - list: "list", - object: "object", - // TODO: Remove time after: - // https://github.com/pola-rs/nodejs-polars/issues/364 - time: "time", - year: "year", - yearmonth: "yearmonth", - }, - }) - - const target = await loadXlsxTable({ path }, { denormalized: true }) - expect((await target.collect()).toRecords()).toEqual([ - { - array: "[1, 2, 3]", - boolean: true, - date: "2025-01-01", - datetime: "2025-01-01T00:00:00", - duration: "P23DT23H", - geojson: '{"value": 1}', - geopoint: "40.0,50.0", - integer: 1, - list: "1.0,2.0,3.0", - number: 1.1, - object: '{"value": 1}', - string: "string", - time: "00:00:00", - year: 2025, - yearmonth: "2025-01", - }, - ]) - }) -}) diff --git a/table/plugins/xlxs/table/save.ts b/table/plugins/xlxs/table/save.ts deleted file mode 100644 index eb3c99d5..00000000 --- a/table/plugins/xlxs/table/save.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { saveFile } from "@dpkit/dataset" -import { resolveDialect } from "@dpkit/metadata" -import { utils, write } from "xlsx" -import type { SaveTableOptions } from "../../../plugin.ts" -import { inferSchemaFromTable } from "../../../schema/index.ts" -import { denormalizeTable } from "../../../table/index.ts" -import type { Table } from "../../../table/index.ts" - -// Currently, we use slow non-rust implementation as in the future -// polars-rust might be able to provide a faster native implementation - -export async function saveXlsxTable(table: Table, options: SaveTableOptions) { - const { path, overwrite } = options - - const schema = - options.schema ?? - (await inferSchemaFromTable(table, { - ...options, - keepStrings: true, - })) - - table = await denormalizeTable(table, schema, { - nativeTypes: ["boolean", "integer", "number", "string", "year"], - }) - - const frame = await table.collect() - const dialect = await resolveDialect(options.dialect) - const sheetName = dialect?.sheetName ?? "Sheet1" - - const sheet = utils.json_to_sheet(frame.toRecords()) - const book = utils.book_new() - utils.book_append_sheet(book, sheet, sheetName) - - const buffer = write(book, { type: "buffer", bookType: "xlsx" }) - await saveFile(path, buffer, { overwrite }) - - return path -} diff --git a/table/plugins/xlxs/table/test.ts b/table/plugins/xlxs/table/test.ts deleted file mode 100644 index 8f52e25f..00000000 --- a/table/plugins/xlxs/table/test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { readFile } from "node:fs/promises" -import { writeFile } from "node:fs/promises" -import { read, utils, write } from "xlsx" - -// We intentionally don't use dpkit's function here to isolate the tests - -export async function readTestData(path: string) { - const buffer = await readFile(path) - const book = read(buffer, { type: "buffer" }) - const sheetName = book.SheetNames[0] - const sheet = sheetName ? book.Sheets[sheetName] : undefined - return sheet ? utils.sheet_to_json(sheet) : [] -} - -export async function writeTestData( - path: string, - rows: unknown[][], - options?: { sheetNumber?: number; sheetName?: string }, -) { - const book = utils.book_new() - const sheet = utils.aoa_to_sheet(rows) - const sheetNumber = options?.sheetNumber ?? 1 - - for (let i = 0; i < sheetNumber; i++) { - const sheetName = options?.sheetName ?? `Sheet${i + 1}` - utils.book_append_sheet(book, sheet, sheetName) - } - - const buffer = write(book, { type: "buffer", bookType: "xlsx" }) - await writeFile(path, buffer) -} diff --git a/table/schema/Mapping.ts b/table/schema/Mapping.ts deleted file mode 100644 index 7b6fb541..00000000 --- a/table/schema/Mapping.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import type { PolarsSchema } from "./Schema.ts" - -export interface SchemaMapping { - source: PolarsSchema - target: Schema -} diff --git a/table/schema/Options.ts b/table/schema/Options.ts deleted file mode 100644 index 0dde360b..00000000 --- a/table/schema/Options.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { GeojsonField, GeopointField, ListField } from "@dpkit/metadata" -import type { StringField } from "@dpkit/metadata" -import type { FieldType } from "@dpkit/metadata" - -export interface SchemaOptions { - fieldNames?: string[] - fieldTypes?: Record - missingValues?: string[] - stringFormat?: StringField["format"] - decimalChar?: string - groupChar?: string - bareNumber?: boolean - trueValues?: string[] - falseValues?: string[] - datetimeFormat?: string - dateFormat?: string - timeFormat?: string - arrayType?: "array" | "list" - listDelimiter?: string - listItemType?: ListField["itemType"] - geopointFormat?: GeopointField["format"] - geojsonFormat?: GeojsonField["format"] -} diff --git a/table/schema/Schema.ts b/table/schema/Schema.ts deleted file mode 100644 index 60d6e45c..00000000 --- a/table/schema/Schema.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { PolarsField } from "../field/index.ts" - -export interface PolarsSchema { - fields: PolarsField[] -} diff --git a/table/schema/helpers.ts b/table/schema/helpers.ts deleted file mode 100644 index 5d00efc0..00000000 --- a/table/schema/helpers.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type * as pl from "nodejs-polars" -import type { PolarsSchema } from "./Schema.ts" - -export function getPolarsSchema( - typeMapping: Record, -): PolarsSchema { - const entries = Object.entries(typeMapping) - const fields = entries.map(([name, type]) => ({ name, type })) - - return { fields } -} diff --git a/table/schema/index.ts b/table/schema/index.ts deleted file mode 100644 index 8cd4331e..00000000 --- a/table/schema/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type { PolarsSchema } from "./Schema.ts" -export { getPolarsSchema } from "./helpers.ts" -export type { SchemaOptions } from "./Options.ts" -export { inferSchemaFromTable } from "./infer.ts" -export { inferSchemaFromSample } from "./infer.ts" -export type { InferSchemaOptions } from "./infer.ts" -export type { SchemaMapping } from "./Mapping.ts" -export { matchSchemaField } from "./match.ts" diff --git a/table/schema/infer.spec.ts b/table/schema/infer.spec.ts deleted file mode 100644 index 784ce419..00000000 --- a/table/schema/infer.spec.ts +++ /dev/null @@ -1,362 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inferSchemaFromTable } from "./infer.ts" - -describe("inferSchemaFromTable", () => { - it("should infer from native types", async () => { - const table = pl - .DataFrame({ - integer: pl.Series("integer", [1, 2], pl.Int32), - number: [1.1, 2.2], - }) - .lazy() - - const schema = { - fields: [ - { name: "integer", type: "integer" }, - { name: "number", type: "number" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer integers from floats", async () => { - const table = pl - .DataFrame({ - id: [1.0, 2.0, 3.0], - count: [10.0, 20.0, 30.0], - }) - .lazy() - - const schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "count", type: "integer" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer numeric", async () => { - const table = pl - .DataFrame({ - name1: ["1", "2", "3"], - name2: ["1,000", "2,000", "3,000"], - name3: ["1.1", "2.2", "3.3"], - name4: ["1,000.1", "2,000.2", "3,000.3"], - }) - .lazy() - - const schema = { - fields: [ - { name: "name1", type: "integer" }, - { name: "name2", type: "integer", groupChar: "," }, - { name: "name3", type: "number" }, - { name: "name4", type: "number", groupChar: "," }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer numeric (commaDecimal)", async () => { - const table = pl - .DataFrame({ - name1: ["1.000", "2.000", "3.000"], - name2: ["1.000,5", "2.000,5", "3.000,5"], - }) - .lazy() - - const schema = { - fields: [ - { name: "name1", type: "integer", groupChar: "." }, - { name: "name2", type: "number", decimalChar: ",", groupChar: "." }, - ], - } - - expect(await inferSchemaFromTable(table, { commaDecimal: true })).toEqual( - schema, - ) - }) - - it("should infer booleans", async () => { - const table = pl - .DataFrame({ - name1: ["true", "True", "TRUE"], - name2: ["false", "False", "FALSE"], - }) - .lazy() - - const schema = { - fields: [ - { name: "name1", type: "boolean" }, - { name: "name2", type: "boolean" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer objects", async () => { - const table = pl - .DataFrame({ - name1: ['{"a": 1}'], - name2: ["{}"], - }) - .lazy() - - const schema = { - fields: [ - { name: "name1", type: "object" }, - { name: "name2", type: "object" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer arrays", async () => { - const table = pl - .DataFrame({ - name1: ["[1,2,3]"], - name2: ["[]"], - }) - .lazy() - - const schema = { - fields: [ - { name: "name1", type: "array" }, - { name: "name2", type: "array" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer dates with ISO format", async () => { - const table = pl - .DataFrame({ - name1: ["2023-01-15", "2023-02-20", "2023-03-25"], - }) - .lazy() - - const schema = { - fields: [{ name: "name1", type: "date" }], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer dates with slash format", async () => { - const table = pl - .DataFrame({ - yearFirst: ["2023/01/15", "2023/02/20", "2023/03/25"], - dayMonth: ["15/01/2023", "20/02/2023", "25/03/2023"], - monthDay: ["01/15/2023", "02/20/2023", "03/25/2023"], - }) - .lazy() - - const schemaDefault = { - fields: [ - { name: "yearFirst", type: "date", format: "%Y/%m/%d" }, - { name: "dayMonth", type: "date", format: "%d/%m/%Y" }, - { name: "monthDay", type: "date", format: "%d/%m/%Y" }, - ], - } - - const schemaMonthFirst = { - fields: [ - { name: "yearFirst", type: "date", format: "%Y/%m/%d" }, - { name: "dayMonth", type: "date", format: "%m/%d/%Y" }, - { name: "monthDay", type: "date", format: "%m/%d/%Y" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schemaDefault) - expect(await inferSchemaFromTable(table, { monthFirst: true })).toEqual( - schemaMonthFirst, - ) - }) - - it("should infer dates with hyphen format", async () => { - const table = pl - .DataFrame({ - dayMonth: ["15-01-2023", "20-02-2023", "25-03-2023"], - }) - .lazy() - - const schemaDefault = { - fields: [{ name: "dayMonth", type: "date", format: "%d-%m-%Y" }], - } - - const schemaMonthFirst = { - fields: [{ name: "dayMonth", type: "date", format: "%m-%d-%Y" }], - } - - expect(await inferSchemaFromTable(table)).toEqual(schemaDefault) - expect(await inferSchemaFromTable(table, { monthFirst: true })).toEqual( - schemaMonthFirst, - ) - }) - - it("should infer times with standard format", async () => { - const table = pl - .DataFrame({ - fullTime: ["14:30:45", "08:15:30", "23:59:59"], - shortTime: ["14:30", "08:15", "23:59"], - }) - .lazy() - - const schema = { - fields: [ - { name: "fullTime", type: "time" }, - { name: "shortTime", type: "time", format: "%H:%M" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer times with 12-hour format", async () => { - const table = pl - .DataFrame({ - fullTime: ["2:30:45 PM", "8:15:30 AM", "11:59:59 PM"], - shortTime: ["2:30 PM", "8:15 AM", "11:59 PM"], - }) - .lazy() - - const schema = { - fields: [ - { name: "fullTime", type: "time", format: "%I:%M:%S %p" }, - { name: "shortTime", type: "time", format: "%I:%M %p" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer times with timezone offset", async () => { - const table = pl - .DataFrame({ - name: ["14:30:45+01:00", "08:15:30-05:00", "23:59:59+00:00"], - }) - .lazy() - - const schema = { - fields: [{ name: "name", type: "time" }], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer datetimes with ISO format", async () => { - const table = pl - .DataFrame({ - standard: [ - "2023-01-15T14:30:45", - "2023-02-20T08:15:30", - "2023-03-25T23:59:59", - ], - utc: [ - "2023-01-15T14:30:45Z", - "2023-02-20T08:15:30Z", - "2023-03-25T23:59:59Z", - ], - withTz: [ - "2023-01-15T14:30:45+01:00", - "2023-02-20T08:15:30-05:00", - "2023-03-25T23:59:59+00:00", - ], - withSpace: [ - "2023-01-15 14:30:45", - "2023-02-20 08:15:30", - "2023-03-25 23:59:59", - ], - }) - .lazy() - - const schema = { - fields: [ - { name: "standard", type: "datetime" }, - { name: "utc", type: "datetime" }, - { name: "withTz", type: "datetime" }, - { name: "withSpace", type: "datetime", format: "%Y-%m-%d %H:%M:%S" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) - - it("should infer datetimes with custom formats", async () => { - const table = pl - .DataFrame({ - shortDayMonth: [ - "15/01/2023 14:30", - "20/02/2023 08:15", - "25/03/2023 23:59", - ], - fullDayMonth: [ - "15/01/2023 14:30:45", - "20/02/2023 08:15:30", - "25/03/2023 23:59:59", - ], - shortMonthDay: [ - "01/15/2023 14:30", - "02/20/2023 08:15", - "03/25/2023 23:59", - ], - fullMonthDay: [ - "01/15/2023 14:30:45", - "02/20/2023 08:15:30", - "03/25/2023 23:59:59", - ], - }) - .lazy() - - const schemaDefault = { - fields: [ - { name: "shortDayMonth", type: "datetime", format: "%d/%m/%Y %H:%M" }, - { name: "fullDayMonth", type: "datetime", format: "%d/%m/%Y %H:%M:%S" }, - { name: "shortMonthDay", type: "datetime", format: "%d/%m/%Y %H:%M" }, - { name: "fullMonthDay", type: "datetime", format: "%d/%m/%Y %H:%M:%S" }, - ], - } - - const schemaMonthFirst = { - fields: [ - { name: "shortDayMonth", type: "datetime", format: "%m/%d/%Y %H:%M" }, - { name: "fullDayMonth", type: "datetime", format: "%m/%d/%Y %H:%M:%S" }, - { name: "shortMonthDay", type: "datetime", format: "%m/%d/%Y %H:%M" }, - { name: "fullMonthDay", type: "datetime", format: "%m/%d/%Y %H:%M:%S" }, - ], - } - - expect(await inferSchemaFromTable(table)).toEqual(schemaDefault) - expect(await inferSchemaFromTable(table, { monthFirst: true })).toEqual( - schemaMonthFirst, - ) - }) - - it("should infer lists", async () => { - const table = pl - .DataFrame({ - numericList: ["1.5,2.3", "4.1,5.9", "7.2,8.6"], - integerList: ["1,2", "3,4", "5,6"], - singleValue: ["1.5", "2.3", "4.1"], - }) - .lazy() - - const schema = { - fields: [ - { name: "numericList", type: "list", itemType: "number" }, - { name: "integerList", type: "list", itemType: "integer" }, - { name: "singleValue", type: "number" }, - ], - missingValues: undefined, - } - - expect(await inferSchemaFromTable(table)).toEqual(schema) - }) -}) diff --git a/table/schema/infer.ts b/table/schema/infer.ts deleted file mode 100644 index e9bffbe2..00000000 --- a/table/schema/infer.ts +++ /dev/null @@ -1,241 +0,0 @@ -import type { Field, Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { getPolarsSchema } from "../schema/index.ts" -import type { Table } from "../table/index.ts" -import type { SchemaOptions } from "./Options.ts" - -// TODO: Implement actual options usage for inferring -// TODO: Review default values being {fields: []} vs undefined - -export interface InferSchemaOptions extends SchemaOptions { - sampleRows?: number - confidence?: number - commaDecimal?: boolean - monthFirst?: boolean - keepStrings?: boolean -} - -export async function inferSchemaFromTable( - table: Table, - options?: InferSchemaOptions, -) { - const { sampleRows = 100 } = options ?? {} - - const sample = await table.head(sampleRows).collect() - return inferSchemaFromSample(sample, options) -} - -export function inferSchemaFromSample( - sample: pl.DataFrame, - options?: Exclude, -) { - const { confidence = 0.9, fieldTypes, keepStrings } = options ?? {} - - const typeMapping = createTypeMapping() - const regexMapping = createRegexMapping(options) - - const polarsSchema = getPolarsSchema(sample.schema) - const fieldNames = options?.fieldNames ?? polarsSchema.fields.map(f => f.name) - - const failureThreshold = - sample.height - Math.floor(sample.height * confidence) || 1 - - const schema: Schema = { - fields: [], - } - - for (const name of fieldNames) { - const polarsField = polarsSchema.fields.find(f => f.name === name) - if (!polarsField) { - throw new Error(`Field "${name}" not found in the table`) - } - - // TODO: Remove this workaround once the issue is fixed - // https://github.com/pola-rs/nodejs-polars/issues/372 - let variant = polarsField.type.variant as string - if (!typeMapping[variant]) { - variant = variant.slice(0, -1) - } - - const type = fieldTypes?.[name] ?? typeMapping[variant] ?? "any" - let field = { name, type } - - if (!fieldTypes?.[name]) { - if (type === "array") { - if (options?.arrayType === "list") { - field.type = "list" - } - } - - if (type === "string") { - if (!keepStrings) { - for (const [regex, patch] of Object.entries(regexMapping)) { - const failures = sample - .filter(pl.col(name).str.contains(regex).not()) - .head(failureThreshold).height - - if (failures < failureThreshold) { - field = { ...field, ...patch } - break - } - } - } - } - - if (type === "number") { - const failures = sample - .filter(pl.col(name).eq(pl.col(name).round(0)).not()) - .head(failureThreshold).height - - if (failures < failureThreshold) { - field.type = "integer" - } - } - } - - enhanceField(field, options) - schema.fields.push(field) - } - - enhanceSchema(schema, options) - return schema -} - -function createTypeMapping() { - const mapping: Record = { - Array: "array", - Bool: "boolean", - Categorical: "string", - Date: "date", - Datetime: "datetime", - Decimal: "number", - Float32: "number", - Float64: "number", - Int16: "integer", - Int32: "integer", - Int64: "integer", - Int8: "integer", - List: "array", - Null: "any", - Object: "object", - String: "string", - Struct: "object", - Time: "time", - UInt16: "integer", - UInt32: "integer", - UInt64: "integer", - UInt8: "integer", - Utf8: "string", - } - - return mapping -} - -function createRegexMapping(options?: InferSchemaOptions) { - const { commaDecimal, monthFirst } = options ?? {} - - const mapping: Record> = { - // Numeric - "^\\d+$": { type: "integer" }, - "^\\d{1,3}(,\\d{3})+$": commaDecimal - ? { type: "number" } - : { type: "integer", groupChar: "," }, - "^\\d+\\.\\d+$": commaDecimal - ? { type: "integer", groupChar: "." } - : { type: "number" }, - "^\\d{1,3}(,\\d{3})+\\.\\d+$": { type: "number", groupChar: "," }, - "^\\d{1,3}(\\.\\d{3})+,\\d+$": { - type: "number", - groupChar: ".", - decimalChar: ",", - }, - - // Boolean - "^(true|True|TRUE|false|False|FALSE)$": { type: "boolean" }, - - // Date - "^\\d{4}-\\d{2}-\\d{2}$": { type: "date" }, - "^\\d{4}/\\d{2}/\\d{2}$": { type: "date", format: "%Y/%m/%d" }, - "^\\d{2}/\\d{2}/\\d{4}$": monthFirst - ? { type: "date", format: "%m/%d/%Y" } - : { type: "date", format: "%d/%m/%Y" }, - "^\\d{2}-\\d{2}-\\d{4}$": monthFirst - ? { type: "date", format: "%m-%d-%Y" } - : { type: "date", format: "%d-%m-%Y" }, - "^\\d{2}\\.\\d{2}\\.\\d{4}$": monthFirst - ? { type: "date", format: "%m.%d.%Y" } - : { type: "date", format: "%d.%m.%Y" }, - - // Time - "^\\d{2}:\\d{2}:\\d{2}$": { type: "time" }, - "^\\d{2}:\\d{2}$": { type: "time", format: "%H:%M" }, - "^\\d{1,2}:\\d{2}:\\d{2}\\s*(am|pm|AM|PM)$": { - type: "time", - format: "%I:%M:%S %p", - }, - "^\\d{1,2}:\\d{2}\\s*(am|pm|AM|PM)$": { type: "time", format: "%I:%M %p" }, - "^\\d{2}:\\d{2}:\\d{2}[+-]\\d{2}:?\\d{2}$": { type: "time" }, - - // Datetime - ISO format - "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z?$": { type: "datetime" }, - "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}[+-]\\d{2}:?\\d{2}$": { - type: "datetime", - }, - "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$": { - type: "datetime", - format: "%Y-%m-%d %H:%M:%S", - }, - "^\\d{2}/\\d{2}/\\d{4} \\d{2}:\\d{2}$": monthFirst - ? { type: "datetime", format: "%m/%d/%Y %H:%M" } - : { type: "datetime", format: "%d/%m/%Y %H:%M" }, - "^\\d{2}/\\d{2}/\\d{4} \\d{2}:\\d{2}:\\d{2}$": monthFirst - ? { type: "datetime", format: "%m/%d/%Y %H:%M:%S" } - : { type: "datetime", format: "%d/%m/%Y %H:%M:%S" }, - - // Object - "^\\{": { type: "object" }, - - // Array - "^\\[": { type: "array" }, - - // List - // TODO: Support commaDecimal - "^\\d+,\\d+$": { type: "list", itemType: "integer" }, - "^[\\d.]+,[\\d.]+$": { type: "list", itemType: "number" }, - } - - return mapping -} - -function enhanceField(field: Field, options?: InferSchemaOptions) { - if (field.type === "string") { - field.format = options?.stringFormat ?? field.format - } else if (field.type === "integer") { - field.groupChar = options?.groupChar ?? field.groupChar - field.bareNumber = options?.bareNumber ?? field.bareNumber - } else if (field.type === "number") { - field.decimalChar = options?.decimalChar ?? field.decimalChar - field.groupChar = options?.groupChar ?? field.groupChar - field.bareNumber = options?.bareNumber ?? field.bareNumber - } else if (field.type === "boolean") { - field.trueValues = options?.trueValues ?? field.trueValues - field.falseValues = options?.falseValues ?? field.falseValues - } else if (field.type === "datetime") { - field.format = options?.datetimeFormat ?? field.format - } else if (field.type === "date") { - field.format = options?.dateFormat ?? field.format - } else if (field.type === "time") { - field.format = options?.timeFormat ?? field.format - } else if (field.type === "list") { - field.delimiter = options?.listDelimiter ?? field.delimiter - field.itemType = options?.listItemType ?? field.itemType - } else if (field.type === "geopoint") { - field.format = options?.geopointFormat ?? field.format - } else if (field.type === "geojson") { - field.format = options?.geojsonFormat ?? field.format - } -} - -function enhanceSchema(schema: Schema, options?: InferSchemaOptions) { - schema.missingValues = options?.missingValues ?? schema.missingValues -} diff --git a/table/schema/match.ts b/table/schema/match.ts deleted file mode 100644 index d505664d..00000000 --- a/table/schema/match.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { Field } from "@dpkit/metadata" -import type { SchemaMapping } from "./Mapping.ts" - -export function matchSchemaField( - mapping: SchemaMapping, - field: Field, - index: number, -) { - const fieldsMatch = mapping.target.fieldsMatch ?? "exact" - - const polarsField = - fieldsMatch !== "exact" - ? mapping.source.fields.find(it => it.name === field.name) - : mapping.source.fields[index] - - return polarsField ? { source: polarsField, target: field } : undefined -} diff --git a/table/table/Frame.ts b/table/table/Frame.ts deleted file mode 100644 index e9df756f..00000000 --- a/table/table/Frame.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type * as pl from "nodejs-polars" - -export type Frame = pl.DataFrame diff --git a/table/table/Table.ts b/table/table/Table.ts deleted file mode 100644 index 954d1bcd..00000000 --- a/table/table/Table.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type * as pl from "nodejs-polars" - -export type Table = pl.LazyDataFrame diff --git a/table/table/checks/unique.spec.ts b/table/table/checks/unique.spec.ts deleted file mode 100644 index 6d909c43..00000000 --- a/table/table/checks/unique.spec.ts +++ /dev/null @@ -1,210 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" - -describe("inspectTable (row/unique)", () => { - it("should not errors when all rows are unique for primary key", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 4, 5], - name: ["Alice", "Bob", "Charlie", "David", "Eve"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - primaryKey: ["id"], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for duplicate primary key rows", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 2, 5], - name: ["Alice", "Bob", "Charlie", "Bob2", "Eve"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - primaryKey: ["id"], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 4, - fieldNames: ["id"], - }) - }) - - it("should not errors when all rows are unique for unique key", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 4, 5], - email: [ - "a@test.com", - "b@test.com", - "c@test.com", - "d@test.com", - "e@test.com", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "email", type: "string" }, - ], - uniqueKeys: [["email"]], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should errors for duplicate unique key rows", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 4, 5], - email: [ - "a@test.com", - "b@test.com", - "a@test.com", - "d@test.com", - "b@test.com", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "email", type: "string" }, - ], - uniqueKeys: [["email"]], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 3, - fieldNames: ["email"], - }) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 5, - fieldNames: ["email"], - }) - }) - - it("should handle composite unique keys", async () => { - const table = pl - .DataFrame({ - category: ["A", "A", "B", "A", "B"], - subcategory: ["X", "Y", "X", "X", "Y"], - value: [1, 2, 3, 4, 5], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "category", type: "string" }, - { name: "subcategory", type: "string" }, - { name: "value", type: "number" }, - ], - uniqueKeys: [["category", "subcategory"]], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 4, - fieldNames: ["category", "subcategory"], - }) - }) - - it("should handle both primary key and unique keys", async () => { - const table = pl - .DataFrame({ - id: [1, 2, 3, 2, 5], - email: [ - "a@test.com", - "b@test.com", - "c@test.com", - "d@test.com", - "a@test.com", - ], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "email", type: "string" }, - ], - primaryKey: ["id"], - uniqueKeys: [["email"]], - } - - const errors = await inspectTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 4, - fieldNames: ["id"], - }) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 5, - fieldNames: ["email"], - }) - }) - - it("should handle null values in unique keys correctly", async () => { - const table = pl - .DataFrame({ - id: [1, 2, null, 4, null, 2], - name: ["Alice", "Bob", "Charlie", "David", "Eve", "Bob"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - uniqueKeys: [["id"], ["id", "name"]], - } - - const errors = await inspectTable(table, { schema }) - console.log(errors) - - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 6, - fieldNames: ["id"], - }) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 6, - fieldNames: ["id", "name"], - }) - }) -}) diff --git a/table/table/checks/unique.ts b/table/table/checks/unique.ts deleted file mode 100644 index 6b6b8389..00000000 --- a/table/table/checks/unique.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { RowUniqueError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import type { SchemaMapping } from "../../schema/index.ts" - -export function createChecksRowUnique(mapping: SchemaMapping) { - const uniqueKeys = mapping.target.uniqueKeys ?? [] - - if (mapping.target.primaryKey) { - uniqueKeys.push(mapping.target.primaryKey) - } - - return uniqueKeys.map(createCheckRowUnique) -} - -function createCheckRowUnique(uniqueKey: string[]) { - const isErrorExpr = pl - .concatList(uniqueKey) - .isFirstDistinct() - .not() - // Fold is not available so we use a tricky way to eliminate nulls - .and(pl.concatList(uniqueKey).lst.min().isNotNull()) - - const errorTemplate: RowUniqueError = { - type: "row/unique", - fieldNames: uniqueKey, - rowNumber: 0, - } - - return { isErrorExpr, errorTemplate } -} diff --git a/table/table/denormalize.ts b/table/table/denormalize.ts deleted file mode 100644 index d18278a4..00000000 --- a/table/table/denormalize.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import type * as pl from "nodejs-polars" -import { denormalizeField } from "../field/index.ts" -import type { DenormalizeFieldOptions } from "../field/index.ts" -import type { Table } from "./Table.ts" - -export async function denormalizeTable( - table: Table, - schema: Schema, - options?: DenormalizeFieldOptions, -) { - return table.select(...Object.values(denormalizeFields(schema, options))) -} - -export function denormalizeFields( - schema: Schema, - options?: DenormalizeFieldOptions, -) { - const exprs: Record = {} - - for (const field of schema.fields) { - const missingValues = field.missingValues ?? schema.missingValues - const mergedField = { ...field, missingValues } - - const expr = denormalizeField(mergedField, options) - exprs[field.name] = expr - } - - return exprs -} diff --git a/table/table/helpers.spec.ts b/table/table/helpers.spec.ts deleted file mode 100644 index ca970b12..00000000 --- a/table/table/helpers.spec.ts +++ /dev/null @@ -1,415 +0,0 @@ -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { - joinHeaderRows, - skipCommentRows, - stripInitialSpace, -} from "./helpers.ts" - -describe("joinHeaderRows", () => { - it("should join two header rows with default space separator", async () => { - const table = pl - .DataFrame({ - col1: ["first", "name", "header3", "Alice", "Bob"], - col2: ["last", "name", "header3", "Smith", "Jones"], - col3: [ - "contact", - "email", - "header3", - "alice@example.com", - "bob@example.com", - ], - }) - .lazy() - - const result = await joinHeaderRows(table, { - dialect: { headerRows: [2, 3] }, - }) - - const collected = await result.collect() - expect(collected.columns).toEqual([ - "col1 first", - "col2 last", - "col3 contact", - ]) - expect(collected.height).toBe(4) - expect(collected.row(0)).toEqual(["name", "name", "email"]) - expect(collected.row(1)).toEqual(["header3", "header3", "header3"]) - expect(collected.row(2)).toEqual(["Alice", "Smith", "alice@example.com"]) - expect(collected.row(3)).toEqual(["Bob", "Jones", "bob@example.com"]) - }) - - it("should join two header rows with custom separator", async () => { - const table = pl - .DataFrame({ - col1: ["user", "first", "header3", "Alice", "Bob"], - col2: ["user", "last", "header3", "Smith", "Jones"], - col3: ["meta", "created", "header3", "2023-01-01", "2023-01-02"], - }) - .lazy() - - const result = await joinHeaderRows(table, { - dialect: { headerRows: [2, 3], headerJoin: "_" }, - }) - - const collected = await result.collect() - expect(collected.columns).toEqual(["col1_user", "col2_user", "col3_meta"]) - expect(collected.height).toBe(4) - expect(collected.row(0)).toEqual(["first", "last", "created"]) - expect(collected.row(1)).toEqual(["header3", "header3", "header3"]) - expect(collected.row(2)).toEqual(["Alice", "Smith", "2023-01-01"]) - expect(collected.row(3)).toEqual(["Bob", "Jones", "2023-01-02"]) - }) - - it("should return table unchanged when only one header row", async () => { - const table = pl - .DataFrame({ - name: ["Alice", "Bob"], - age: [30, 25], - city: ["NYC", "LA"], - }) - .lazy() - - const result = await joinHeaderRows(table, { - dialect: { headerRows: [1] }, - }) - - const collected = await result.collect() - expect(collected.columns).toEqual(["name", "age", "city"]) - expect(collected.height).toBe(2) - }) - - it("should return table unchanged when no header rows", async () => { - const table = pl - .DataFrame({ - field1: ["Alice", "Bob"], - field2: [30, 25], - field3: ["NYC", "LA"], - }) - .lazy() - - const result = await joinHeaderRows(table, { - dialect: { header: false }, - }) - - const collected = await result.collect() - expect(collected.columns).toEqual(["field1", "field2", "field3"]) - expect(collected.height).toBe(2) - }) - - it("should join three header rows", async () => { - const table = pl - .DataFrame({ - col1: ["person", "user", "first", "header4", "Alice", "Bob"], - col2: ["person", "user", "last", "header4", "Smith", "Jones"], - col3: ["location", "address", "city", "header4", "NYC", "LA"], - }) - .lazy() - - const result = await joinHeaderRows(table, { - dialect: { headerRows: [2, 3, 4] }, - }) - - const collected = await result.collect() - expect(collected.columns).toEqual([ - "col1 person user", - "col2 person user", - "col3 location address", - ]) - expect(collected.height).toBe(4) - expect(collected.row(0)).toEqual(["first", "last", "city"]) - expect(collected.row(1)).toEqual(["header4", "header4", "header4"]) - expect(collected.row(2)).toEqual(["Alice", "Smith", "NYC"]) - expect(collected.row(3)).toEqual(["Bob", "Jones", "LA"]) - }) - - it("should handle empty strings in header rows", async () => { - const table = pl - .DataFrame({ - col1: ["person", "", "header3", "Alice", "Bob"], - col2: ["", "name", "header3", "Smith", "Jones"], - col3: ["location", "city", "header3", "NYC", "LA"], - }) - .lazy() - - const result = await joinHeaderRows(table, { - dialect: { headerRows: [2, 3] }, - }) - - const collected = await result.collect() - expect(collected.columns).toEqual(["col1 person", "col2 ", "col3 location"]) - expect(collected.height).toBe(4) - expect(collected.row(0)).toEqual(["", "name", "city"]) - expect(collected.row(1)).toEqual(["header3", "header3", "header3"]) - expect(collected.row(2)).toEqual(["Alice", "Smith", "NYC"]) - expect(collected.row(3)).toEqual(["Bob", "Jones", "LA"]) - }) -}) - -describe("skipCommentRows", () => { - it("should skip comment rows by row number", async () => { - const table = pl - .DataFrame({ - name: ["Alice", "# Comment", "Bob", "Charlie"], - age: [30, 0, 25, 35], - city: ["NYC", "ignored", "LA", "SF"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: { commentRows: [2], header: false }, - }) - - const collected = await result.collect() - expect(collected.height).toBe(3) - expect(collected.row(0)).toEqual(["Alice", 30, "NYC"]) - expect(collected.row(1)).toEqual(["Bob", 25, "LA"]) - expect(collected.row(2)).toEqual(["Charlie", 35, "SF"]) - }) - - it("should skip multiple comment rows", async () => { - const table = pl - .DataFrame({ - name: ["Alice", "# Comment 1", "Bob", "# Comment 2", "Charlie"], - age: [30, 0, 25, 0, 35], - city: ["NYC", "ignored", "LA", "ignored", "SF"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: { commentRows: [2, 4], header: false }, - }) - - const collected = await result.collect() - expect(collected.height).toBe(3) - expect(collected.row(0)).toEqual(["Alice", 30, "NYC"]) - expect(collected.row(1)).toEqual(["Bob", 25, "LA"]) - expect(collected.row(2)).toEqual(["Charlie", 35, "SF"]) - }) - - it("should return table unchanged when no commentRows specified", async () => { - const table = pl - .DataFrame({ - name: ["Alice", "Bob", "Charlie"], - age: [30, 25, 35], - city: ["NYC", "LA", "SF"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: {}, - }) - - const collected = await result.collect() - expect(collected.height).toBe(3) - expect(collected.columns).toEqual(["name", "age", "city"]) - }) - - it("should skip rows after header when headerRows specified", async () => { - const table = pl - .DataFrame({ - col1: ["name", "Alice", "# Comment", "Bob"], - col2: ["age", "30", "-1", "25"], - col3: ["city", "NYC", "ignored", "LA"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: { headerRows: [2], commentRows: [5] }, - }) - - const collected = await result.collect() - expect(collected.height).toBe(3) - expect(collected.row(0)).toEqual(["name", "age", "city"]) - expect(collected.row(1)).toEqual(["Alice", "30", "NYC"]) - expect(collected.row(2)).toEqual(["Bob", "25", "LA"]) - }) - - it("should handle commentRows at the beginning", async () => { - const table = pl - .DataFrame({ - name: ["# Skip this", "Alice", "Bob"], - age: [0, 30, 25], - city: ["ignored", "NYC", "LA"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: { commentRows: [1], header: false }, - }) - - const collected = await result.collect() - expect(collected.height).toBe(2) - expect(collected.row(0)).toEqual(["Alice", 30, "NYC"]) - expect(collected.row(1)).toEqual(["Bob", 25, "LA"]) - }) - - it("should handle commentRows at the end", async () => { - const table = pl - .DataFrame({ - name: ["Alice", "Bob", "# Footer comment"], - age: [30, 25, 0], - city: ["NYC", "LA", "ignored"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: { commentRows: [3], header: false }, - }) - - const collected = await result.collect() - expect(collected.height).toBe(2) - expect(collected.row(0)).toEqual(["Alice", 30, "NYC"]) - expect(collected.row(1)).toEqual(["Bob", 25, "LA"]) - }) - - it("should handle multiple header rows with commentRows", async () => { - const table = pl - .DataFrame({ - col1: ["person", "first", "Alice", "# Comment", "Bob"], - col2: ["person", "last", "Smith", "ignored", "Jones"], - col3: ["location", "city", "NYC", "ignored", "LA"], - }) - .lazy() - - const result = skipCommentRows(table, { - dialect: { headerRows: [2, 3], commentRows: [7] }, - }) - - const collected = await result.collect() - expect(collected.height).toBe(4) - expect(collected.row(0)).toEqual(["person", "person", "location"]) - expect(collected.row(1)).toEqual(["first", "last", "city"]) - expect(collected.row(2)).toEqual(["Alice", "Smith", "NYC"]) - expect(collected.row(3)).toEqual(["Bob", "Jones", "LA"]) - }) -}) - -describe("stripInitialSpace", () => { - it("should strip leading and trailing spaces from all columns", async () => { - const table = pl - .DataFrame({ - name: [" Alice ", " Bob", "Charlie "], - age: ["30", " 25 ", "35"], - city: [" NYC", "LA ", " SF "], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: { skipInitialSpace: true }, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual(["Alice", "30", "NYC"]) - expect(collected.row(1)).toEqual(["Bob", "25", "LA"]) - expect(collected.row(2)).toEqual(["Charlie", "35", "SF"]) - }) - - it("should return table unchanged when skipInitialSpace is false", async () => { - const table = pl - .DataFrame({ - name: [" Alice ", " Bob"], - age: ["30", " 25 "], - city: [" NYC", "LA "], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: { skipInitialSpace: false }, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual([" Alice ", "30", " NYC"]) - expect(collected.row(1)).toEqual([" Bob", " 25 ", "LA "]) - }) - - it("should return table unchanged when skipInitialSpace is not specified", async () => { - const table = pl - .DataFrame({ - name: [" Alice ", " Bob"], - age: ["30", " 25 "], - city: [" NYC", "LA "], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: {}, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual([" Alice ", "30", " NYC"]) - expect(collected.row(1)).toEqual([" Bob", " 25 ", "LA "]) - }) - - it("should handle strings with no spaces", async () => { - const table = pl - .DataFrame({ - name: ["Alice", "Bob"], - age: ["30", "25"], - city: ["NYC", "LA"], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: { skipInitialSpace: true }, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual(["Alice", "30", "NYC"]) - expect(collected.row(1)).toEqual(["Bob", "25", "LA"]) - }) - - it("should handle empty strings", async () => { - const table = pl - .DataFrame({ - name: ["Alice", ""], - age: ["30", " "], - city: ["", "LA"], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: { skipInitialSpace: true }, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual(["Alice", "30", ""]) - expect(collected.row(1)).toEqual(["", "", "LA"]) - }) - - it("should handle strings with multiple spaces", async () => { - const table = pl - .DataFrame({ - name: [" Alice ", " Bob"], - age: ["30 ", " 25 "], - city: [" NYC ", " LA "], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: { skipInitialSpace: true }, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual(["Alice", "30", "NYC"]) - expect(collected.row(1)).toEqual(["Bob", "25", "LA"]) - }) - - it("should handle tabs and other whitespace", async () => { - const table = pl - .DataFrame({ - name: ["\tAlice\t", "\nBob"], - age: ["30\n", "\t25\t"], - city: ["\tNYC", "LA\n"], - }) - .lazy() - - const result = stripInitialSpace(table, { - dialect: { skipInitialSpace: true }, - }) - - const collected = await result.collect() - expect(collected.row(0)).toEqual(["Alice", "30", "NYC"]) - expect(collected.row(1)).toEqual(["Bob", "25", "LA"]) - }) -}) diff --git a/table/table/helpers.ts b/table/table/helpers.ts deleted file mode 100644 index f9afb63e..00000000 --- a/table/table/helpers.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { Dialect } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import type { Table } from "../table/index.ts" - -export async function joinHeaderRows( - table: Table, - options: { dialect: Dialect }, -) { - const { dialect } = options - - const headerOffset = getHeaderOffset(dialect) - const headerRows = getHeaderRows(dialect) - const headerJoin = dialect?.headerJoin ?? " " - if (headerRows.length < 2) { - return table - } - - const extraLabelsFrame = await table - .withRowCount() - .withColumn(pl.col("row_nr").add(1)) - .filter(pl.col("row_nr").add(headerOffset).isIn(headerRows)) - .select(...table.columns.map(name => pl.col(name).str.concat(headerJoin))) - .collect() - - const labels = table.columns - const extraLabels = extraLabelsFrame.row(0) - - const mapping = Object.fromEntries( - labels.map((label, index) => [ - label, - [label, extraLabels[index]].join(headerJoin), - ]), - ) - - return table - .withRowCount() - .withColumn(pl.col("row_nr").add(1)) - .filter(pl.col("row_nr").add(headerOffset).isIn(headerRows).not()) - .rename(mapping) - .drop("row_nr") -} - -export function skipCommentRows(table: Table, options: { dialect: Dialect }) { - const { dialect } = options - - const commentOffset = getCommentOffset(dialect) - if (!dialect?.commentRows) { - return table - } - - return table - .withRowCount() - .withColumn(pl.col("row_nr").add(1)) - .filter(pl.col("row_nr").add(commentOffset).isIn(dialect.commentRows).not()) - .drop("row_nr") -} - -export function stripInitialSpace(table: Table, options: { dialect: Dialect }) { - const { dialect } = options - - if (!dialect?.skipInitialSpace) { - return table - } - - return table.select( - // TODO: rebase on stripCharsStart when it's fixed in polars - // https://github.com/pola-rs/nodejs-polars/issues/336 - table.columns.map(name => pl.col(name).str.strip().as(name)), - ) -} - -function getHeaderOffset(dialect?: Dialect) { - const headerRows = getHeaderRows(dialect) - return headerRows.at(0) ?? 0 -} - -function getHeaderRows(dialect?: Dialect) { - return dialect?.header !== false ? (dialect?.headerRows ?? [1]) : [] -} - -function getCommentOffset(dialect?: Dialect) { - const headerRows = getHeaderRows(dialect) - return headerRows.at(-1) ?? 0 -} diff --git a/table/table/index.ts b/table/table/index.ts deleted file mode 100644 index a439bcb2..00000000 --- a/table/table/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { normalizeTable } from "./normalize.ts" -export { denormalizeTable } from "./denormalize.ts" -export { inspectTable } from "./inspect.ts" -export type { Frame } from "./Frame.ts" -export type { Table } from "./Table.ts" -export { skipCommentRows } from "./helpers.ts" -export { joinHeaderRows } from "./helpers.ts" -export { stripInitialSpace } from "./helpers.ts" -export { queryTable } from "./query.ts" diff --git a/table/table/inspect.spec.ts b/table/table/inspect.spec.ts deleted file mode 100644 index 084bd4fc..00000000 --- a/table/table/inspect.spec.ts +++ /dev/null @@ -1,387 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { inspectTable } from "./inspect.ts" - -describe("inspectTable", () => { - describe("fields validation with fieldsMatch='exact'", () => { - it("should pass when fields exactly match", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - - expect(errors).toEqual([]) - }) - - it("should not have fields error when fields same length", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - age: [30, 25], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "exact", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "number" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([ - { - type: "field/name", - fieldName: "name", - actualFieldName: "age", - }, - ]) - }) - }) - - it("should detect extra fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - age: [30, 25], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/extra", - fieldNames: ["age"], - }) - }) - - it("should detect missing fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/missing", - fieldNames: ["name"], - }) - }) - - describe("fields validation with fieldsMatch='equal'", () => { - it("should pass when field names match regardless of order", async () => { - const table = pl - .DataFrame({ - name: ["John", "Jane"], - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - - it("should detect extra fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - age: [30, 25], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/extra", - fieldNames: ["age"], - }) - }) - - it("should detect missing fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "number" }, - { - name: "name", - type: "string", - constraints: { required: true }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/missing", - fieldNames: ["name"], - }) - }) - - it("should pass when non-required fields are missing", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - }) - - describe("fields validation with fieldsMatch='subset'", () => { - it("should pass when data contains all schema fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - age: [30, 25], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - - it("should pass when data contains exact schema fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - - it("should detect missing fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "number" }, - { - name: "name", - type: "string", - constraints: { required: true }, - }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/missing", - fieldNames: ["name"], - }) - }) - - it("should pass when non-required fields are missing", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - }) - - describe("fields validation with fieldsMatch='superset'", () => { - it("should pass when schema contains all data fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "superset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - - it("should pass when schema contains exact data fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "superset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - - it("should detect extra fields", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["John", "Jane"], - age: [30, 25], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "superset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/extra", - fieldNames: ["age"], - }) - }) - }) - - describe("fields validation with fieldsMatch='partial'", () => { - it("should pass when at least one field matches", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - age: [30, 25], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "partial", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toEqual([]) - }) - - it("should detect when no fields match", async () => { - const table = pl - .DataFrame({ - age: [30, 25], - email: ["john@example.com", "jane@example.com"], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "partial", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - } - - const errors = await inspectTable(table, { schema }) - expect(errors).toContainEqual({ - type: "fields/missing", - fieldNames: ["id", "name"], - }) - }) - }) -}) diff --git a/table/table/inspect.ts b/table/table/inspect.ts deleted file mode 100644 index 5978ea9d..00000000 --- a/table/table/inspect.ts +++ /dev/null @@ -1,222 +0,0 @@ -import os from "node:os" -import type { Field, Schema } from "@dpkit/metadata" -import type { RowError } from "@dpkit/metadata" -import type { TableError } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import pAll from "p-all" -import { inspectField } from "../field/index.ts" -import { arrayDiff } from "../helpers.ts" -import { matchSchemaField } from "../schema/index.ts" -import { getPolarsSchema } from "../schema/index.ts" -import type { SchemaMapping } from "../schema/index.ts" -import type { Table } from "./Table.ts" -import { createChecksRowUnique } from "./checks/unique.ts" - -export async function inspectTable( - table: Table, - options?: { - schema?: Schema - sampleRows?: number - maxErrors?: number - }, -) { - const { schema, sampleRows = 100, maxErrors = 1000 } = options ?? {} - const errors: TableError[] = [] - - if (schema) { - const sample = await table.head(sampleRows).collect() - const polarsSchema = getPolarsSchema(sample.schema) - const mapping = { source: polarsSchema, target: schema } - - const matchErrors = inspectFieldsMatch(mapping) - errors.push(...matchErrors) - - const fieldErrors = await inspectFields(mapping, table, { maxErrors }) - errors.push(...fieldErrors) - - const rowErrors = await inspectRows(mapping, table, { maxErrors }) - errors.push(...rowErrors) - } - - return errors.slice(0, maxErrors) -} - -function inspectFieldsMatch(mapping: SchemaMapping) { - const errors: TableError[] = [] - const fieldsMatch = mapping.target.fieldsMatch ?? "exact" - - const fields = mapping.target.fields - const polarsFields = mapping.source.fields - - const names = fields.map(field => field.name) - const polarsNames = polarsFields.map(field => field.name) - - const requiredNames = fields - .filter(field => field.constraints?.required) - .map(field => field.name) - - const extraFields = polarsFields.length - fields.length - const missingFields = fields.length - polarsFields.length - - const extraNames = arrayDiff(polarsNames, names) - const missingNames = arrayDiff(names, polarsNames) - const missingRequiredNames = arrayDiff(requiredNames, polarsNames) - - if (fieldsMatch === "exact") { - if (extraFields > 0) { - errors.push({ - type: "fields/extra", - fieldNames: extraNames, - }) - } - - if (missingFields > 0) { - errors.push({ - type: "fields/missing", - fieldNames: missingNames, - }) - } - } - - if (fieldsMatch === "equal") { - if (extraNames.length > 0) { - errors.push({ - type: "fields/extra", - fieldNames: extraNames, - }) - } - - if (missingRequiredNames.length > 0) { - errors.push({ - type: "fields/missing", - fieldNames: missingRequiredNames, - }) - } - } - - if (fieldsMatch === "subset") { - if (missingRequiredNames.length > 0) { - errors.push({ - type: "fields/missing", - fieldNames: missingRequiredNames, - }) - } - } - - if (fieldsMatch === "superset") { - if (extraNames.length > 0) { - errors.push({ - type: "fields/extra", - fieldNames: extraNames, - }) - } - } - - if (fieldsMatch === "partial") { - if (missingNames.length === fields.length) { - errors.push({ - type: "fields/missing", - fieldNames: missingNames, - }) - } - } - - return errors -} - -async function inspectFields( - mapping: SchemaMapping, - table: Table, - options: { - maxErrors: number - }, -) { - const { maxErrors } = options - const errors: TableError[] = [] - const fields = mapping.target.fields - const concurrency = os.cpus().length - const abortController = new AbortController() - const maxFieldErrors = Math.ceil(maxErrors / fields.length) - - const collectFieldErrors = async (index: number, field: Field) => { - const fieldMapping = matchSchemaField(mapping, field, index) - if (!fieldMapping) return - - const fieldErrors = await inspectField(fieldMapping, table, { - maxErrors: maxFieldErrors, - }) - - errors.push(...fieldErrors) - if (errors.length > maxErrors) { - abortController.abort() - } - } - - try { - await pAll( - fields.map((field, index) => () => collectFieldErrors(index, field)), - { concurrency }, - ) - } catch (error) { - const isAborted = error instanceof Error && error.name === "AbortError" - if (!isAborted) throw error - } - - return errors -} - -async function inspectRows( - mapping: SchemaMapping, - table: Table, - options: { maxErrors: number }, -) { - const { maxErrors } = options - const errors: TableError[] = [] - const fields = mapping.target.fields - const concurrency = os.cpus().length - 1 - const abortController = new AbortController() - const maxRowErrors = Math.ceil(maxErrors / fields.length) - - const collectRowErrors = async (check: any) => { - const rowCheckTable = table - .withRowCount() - .withColumn(pl.col("row_nr").add(1)) - .rename({ row_nr: "dpkit:number" }) - .withColumn( - pl - .when(check.isErrorExpr) - .then(pl.lit(JSON.stringify(check.errorTemplate))) - .otherwise(pl.lit(null)) - .alias("dpkit:error"), - ) - - const rowCheckFrame = await rowCheckTable - .filter(pl.col("dpkit:error").isNotNull()) - .head(maxRowErrors) - .collect() - - for (const row of rowCheckFrame.toRecords() as any[]) { - const errorTemplate = JSON.parse(row["dpkit:error"]) as RowError - errors.push({ - ...errorTemplate, - rowNumber: row["dpkit:number"], - }) - } - - if (errors.length > maxErrors) { - abortController.abort() - } - } - - try { - await pAll( - [...createChecksRowUnique(mapping)].map(it => () => collectRowErrors(it)), - { concurrency }, - ) - } catch (error) { - const isAborted = error instanceof Error && error.name === "AbortError" - if (!isAborted) throw error - } - - return errors -} diff --git a/table/table/normalize.spec.ts b/table/table/normalize.spec.ts deleted file mode 100644 index 5c6c0c1d..00000000 --- a/table/table/normalize.spec.ts +++ /dev/null @@ -1,262 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { normalizeTable } from "./normalize.ts" - -describe("normalizeTable", () => { - it("should work with schema", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["english", "中文"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work with less fields in data", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["english", "中文"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - { name: "other", type: "boolean" }, - ], - } - - const records = [ - { id: 1, name: "english", other: null }, - { id: 2, name: "中文", other: null }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work with more fields in data", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["english", "中文"], - other: [true, false], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work based on fields order", async () => { - const table = pl - .DataFrame({ - field1: [1, 2], - field2: ["english", "中文"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work based on field names (equal)", async () => { - const table = pl - .DataFrame({ - name: ["english", "中文"], - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work based on field names (subset)", async () => { - const table = pl - .DataFrame({ - name: ["english", "中文"], - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work based on field names (superset)", async () => { - const table = pl - .DataFrame({ - name: ["english", "中文"], - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "superset", - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should work based on field names (partial)", async () => { - const table = pl - .DataFrame({ - name: ["english", "中文"], - id: [1, 2], - }) - .lazy() - - const schema: Schema = { - fieldsMatch: "partial", - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should parse string columns", async () => { - const table = pl - .DataFrame({ - id: ["1", "2"], - name: ["english", "中文"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "string" }, - ], - } - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) - - it("should read type errors as nulls", async () => { - const table = pl - .DataFrame({ - id: [1, 2], - name: ["english", "中文"], - }) - .lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "name", type: "integer" }, - ], - } - - const records = [ - { id: 1, name: null }, - { id: 2, name: null }, - ] - - const result = await normalizeTable(table, schema) - const frame = await result.collect() - expect(frame.toRecords()).toEqual(records) - }) -}) diff --git a/table/table/normalize.ts b/table/table/normalize.ts deleted file mode 100644 index 1a90ddbe..00000000 --- a/table/table/normalize.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Schema } from "@dpkit/metadata" -import * as pl from "nodejs-polars" -import { normalizeField } from "../field/index.ts" -import { matchSchemaField } from "../schema/index.ts" -import { getPolarsSchema } from "../schema/index.ts" -import type { SchemaMapping } from "../schema/index.ts" -import type { Table } from "./Table.ts" - -const HEAD_ROWS = 100 - -export async function normalizeTable(table: Table, schema: Schema) { - const head = await table.head(HEAD_ROWS).collect() - const polarsSchema = getPolarsSchema(head.schema) - - const mapping = { source: polarsSchema, target: schema } - return table.select(...Object.values(normalizeFields(mapping))) -} - -export function normalizeFields(mapping: SchemaMapping) { - const exprs: Record = {} - - for (const [index, field] of mapping.target.fields.entries()) { - const fieldMapping = matchSchemaField(mapping, field, index) - let expr = pl.lit(null).alias(field.name) - - if (fieldMapping) { - const missingValues = field.missingValues ?? mapping.target.missingValues - const mergedField = { ...field, missingValues } - - const column = { source: fieldMapping.source, target: mergedField } - expr = normalizeField(column) - } - - exprs[field.name] = expr - } - - return exprs -} diff --git a/table/table/query.ts b/table/table/query.ts deleted file mode 100644 index d3471f83..00000000 --- a/table/table/query.ts +++ /dev/null @@ -1,7 +0,0 @@ -import * as pl from "nodejs-polars" -import type { Table } from "./Table.ts" - -export function queryTable(table: Table, query: string) { - const context = pl.SQLContext({ self: table }) - return context.execute(query) -} diff --git a/table/tsconfig.json b/table/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/table/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/table/typedoc.json b/table/typedoc.json deleted file mode 100644 index f8e49f3a..00000000 --- a/table/typedoc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "entryPoints": ["index.ts"], - "skipErrorChecking": true -} diff --git a/terminal/@compile.ts b/terminal/@compile.ts index 108bc7a9..a5240f44 100644 --- a/terminal/@compile.ts +++ b/terminal/@compile.ts @@ -2,6 +2,8 @@ import { join } from "node:path" import { execa } from "execa" import metadata from "./package.json" with { type: "json" } +// TODO: Merge build/compile folders + function makeShell(...paths: string[]) { return execa({ cwd: join(import.meta.dirname, ...paths), diff --git a/terminal/README.md b/terminal/README.md deleted file mode 100644 index 570260a1..00000000 --- a/terminal/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/terminal - -dpkit CLI is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/terminal/commands/dialect/explore.spec.tsx b/terminal/commands/dialect/explore.spec.tsx index 3a7231fe..152c13a5 100644 --- a/terminal/commands/dialect/explore.spec.tsx +++ b/terminal/commands/dialect/explore.spec.tsx @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { exploreDialectCommand } from "./explore.tsx" diff --git a/terminal/commands/dialect/explore.tsx b/terminal/commands/dialect/explore.tsx index c9725956..b849d056 100644 --- a/terminal/commands/dialect/explore.tsx +++ b/terminal/commands/dialect/explore.tsx @@ -1,7 +1,7 @@ -import { loadDialect } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { resolveDialect } from "@dpkit/library" import { Command } from "commander" +import { loadDialect } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { resolveDialect } from "frictionless-ts" import React from "react" import { Dialect } from "../../components/Dialect/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/dialect/infer.spec.ts b/terminal/commands/dialect/infer.spec.ts index 2d45fdce..61cffb91 100644 --- a/terminal/commands/dialect/infer.spec.ts +++ b/terminal/commands/dialect/infer.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { inferDialectCommand } from "./infer.tsx" diff --git a/terminal/commands/dialect/infer.tsx b/terminal/commands/dialect/infer.tsx index ef2f1f91..7f57c499 100644 --- a/terminal/commands/dialect/infer.tsx +++ b/terminal/commands/dialect/infer.tsx @@ -1,6 +1,6 @@ -import { inferDialect } from "@dpkit/library" -import type { Resource } from "@dpkit/library" import { Command } from "commander" +import { inferDialect } from "frictionless-ts" +import type { Resource } from "frictionless-ts" import React from "react" import { Dialect } from "../../components/Dialect/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/dialect/script.spec.tsx b/terminal/commands/dialect/script.spec.tsx index 26e24f1e..2e24972b 100644 --- a/terminal/commands/dialect/script.spec.tsx +++ b/terminal/commands/dialect/script.spec.tsx @@ -1,6 +1,6 @@ import repl from "node:repl" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { scriptDialectCommand } from "./script.tsx" diff --git a/terminal/commands/dialect/script.tsx b/terminal/commands/dialect/script.tsx index 13edda79..23cf901f 100644 --- a/terminal/commands/dialect/script.tsx +++ b/terminal/commands/dialect/script.tsx @@ -1,9 +1,9 @@ import repl from "node:repl" -import * as dpkit from "@dpkit/library" -import { loadDialect } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { resolveDialect } from "@dpkit/library" import { Command } from "commander" +import * as dpkit from "frictionless-ts" +import { loadDialect } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { resolveDialect } from "frictionless-ts" import pc from "picocolors" import { helpConfiguration } from "../../helpers/help.ts" import { isEmptyObject } from "../../helpers/object.ts" diff --git a/terminal/commands/dialect/validate.spec.ts b/terminal/commands/dialect/validate.spec.ts index b988236f..d9ade06d 100644 --- a/terminal/commands/dialect/validate.spec.ts +++ b/terminal/commands/dialect/validate.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { validateDialectCommand } from "./validate.tsx" diff --git a/terminal/commands/dialect/validate.tsx b/terminal/commands/dialect/validate.tsx index b5a7d5d8..728e1caa 100644 --- a/terminal/commands/dialect/validate.tsx +++ b/terminal/commands/dialect/validate.tsx @@ -1,7 +1,7 @@ -import { loadDescriptor, validateDialect } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { resolveDialect } from "@dpkit/library" import { Command } from "commander" +import { loadDescriptor, validateDialect } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { resolveDialect } from "frictionless-ts" import React from "react" import { Report } from "../../components/Report/index.ts" import { selectErrorType } from "../../helpers/error.ts" diff --git a/terminal/commands/file/copy.spec.ts b/terminal/commands/file/copy.spec.ts index b6a4fcee..44435fa1 100644 --- a/terminal/commands/file/copy.spec.ts +++ b/terminal/commands/file/copy.spec.ts @@ -1,6 +1,6 @@ import { existsSync } from "node:fs" -import { getTempFilePath, writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { getTempFilePath, writeTempFile } from "frictionless-ts" import { describe, expect, it } from "vitest" import { useRecording } from "vitest-polly" import { copyFileCommand } from "./copy.ts" diff --git a/terminal/commands/file/copy.ts b/terminal/commands/file/copy.ts index 5e242ad3..2dfb3a03 100644 --- a/terminal/commands/file/copy.ts +++ b/terminal/commands/file/copy.ts @@ -1,5 +1,5 @@ -import { copyFile } from "@dpkit/library" import { Command } from "commander" +import { copyFile } from "frictionless-ts" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import * as params from "../../params/index.ts" diff --git a/terminal/commands/file/describe.spec.ts b/terminal/commands/file/describe.spec.ts index c7225902..19648ca5 100644 --- a/terminal/commands/file/describe.spec.ts +++ b/terminal/commands/file/describe.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { describeFileCommand } from "./describe.tsx" diff --git a/terminal/commands/file/describe.tsx b/terminal/commands/file/describe.tsx index 0e93bc0a..96c58b17 100644 --- a/terminal/commands/file/describe.tsx +++ b/terminal/commands/file/describe.tsx @@ -1,5 +1,5 @@ -import { describeFile } from "@dpkit/library" import { Command } from "commander" +import { describeFile } from "frictionless-ts" import React from "react" import { Datagrid } from "../../components/Datagrid/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/file/validate.spec.ts b/terminal/commands/file/validate.spec.ts index a220a75d..a54cfc1b 100644 --- a/terminal/commands/file/validate.spec.ts +++ b/terminal/commands/file/validate.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { validateFileCommand } from "./validate.tsx" diff --git a/terminal/commands/file/validate.tsx b/terminal/commands/file/validate.tsx index 5652d500..19d13f1a 100644 --- a/terminal/commands/file/validate.tsx +++ b/terminal/commands/file/validate.tsx @@ -1,5 +1,5 @@ -import { validateFile } from "@dpkit/library" import { Command } from "commander" +import { validateFile } from "frictionless-ts" import React from "react" import { Report } from "../../components/Report/index.ts" import { selectErrorType } from "../../helpers/error.ts" diff --git a/terminal/commands/package/copy.spec.ts b/terminal/commands/package/copy.spec.ts index cb3b26bf..100f9b35 100644 --- a/terminal/commands/package/copy.spec.ts +++ b/terminal/commands/package/copy.spec.ts @@ -1,8 +1,8 @@ import { existsSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { copyPackageCommand } from "./copy.ts" diff --git a/terminal/commands/package/copy.ts b/terminal/commands/package/copy.ts index 92f06fc0..390a74c6 100644 --- a/terminal/commands/package/copy.ts +++ b/terminal/commands/package/copy.ts @@ -1,5 +1,5 @@ -import { loadPackage, savePackage } from "@dpkit/library" import { Command } from "commander" +import { loadPackage, savePackage } from "frictionless-ts" import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" import { Session } from "../../session.ts" diff --git a/terminal/commands/package/explore.spec.tsx b/terminal/commands/package/explore.spec.tsx index 74b90f46..4a84cd67 100644 --- a/terminal/commands/package/explore.spec.tsx +++ b/terminal/commands/package/explore.spec.tsx @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { explorePackageCommand } from "./explore.tsx" diff --git a/terminal/commands/package/explore.tsx b/terminal/commands/package/explore.tsx index aa4af147..33e095bf 100644 --- a/terminal/commands/package/explore.tsx +++ b/terminal/commands/package/explore.tsx @@ -1,5 +1,5 @@ -import { loadPackage } from "@dpkit/library" import { Command } from "commander" +import { loadPackage } from "frictionless-ts" import React from "react" import { Package } from "../../components/Package/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/package/infer.spec.ts b/terminal/commands/package/infer.spec.ts index d7fff950..ce245d16 100644 --- a/terminal/commands/package/infer.spec.ts +++ b/terminal/commands/package/infer.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { inferPackageCommand } from "./infer.tsx" diff --git a/terminal/commands/package/infer.tsx b/terminal/commands/package/infer.tsx index 0e5650a6..cd0a9246 100644 --- a/terminal/commands/package/infer.tsx +++ b/terminal/commands/package/infer.tsx @@ -1,5 +1,5 @@ -import { inferPackage } from "@dpkit/library" import { Command } from "commander" +import { inferPackage } from "frictionless-ts" import React from "react" import { Package } from "../../components/Package/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/package/publish/ckan.spec.ts b/terminal/commands/package/publish/ckan.spec.ts index d7840fea..cede8b6a 100644 --- a/terminal/commands/package/publish/ckan.spec.ts +++ b/terminal/commands/package/publish/ckan.spec.ts @@ -1,6 +1,6 @@ -import { writeTempFile } from "@dpkit/dataset" -import * as lib from "@dpkit/library" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" +import * as lib from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { ckanPublishPackageCommand } from "./ckan.ts" @@ -8,7 +8,7 @@ import { ckanPublishPackageCommand } from "./ckan.ts" useRecording() describe("package publish ckan", () => { - it("should attempt to publish package to CKAN", async () => { + it.skip("should attempt to publish package to CKAN", async () => { const csvPath = await writeTempFile("id,name\n1,alice\n2,bob") const packageContent = JSON.stringify({ name: "test-package", diff --git a/terminal/commands/package/publish/ckan.ts b/terminal/commands/package/publish/ckan.ts index 6b8acb0b..eb15ccb1 100644 --- a/terminal/commands/package/publish/ckan.ts +++ b/terminal/commands/package/publish/ckan.ts @@ -1,5 +1,5 @@ -import { loadPackage, savePackageToCkan } from "@dpkit/library" import { Command } from "commander" +import { loadPackage, savePackageToCkan } from "frictionless-ts" import { helpConfiguration } from "../../../helpers/help.ts" import * as params from "../../../params/index.ts" import { Session } from "../../../session.ts" diff --git a/terminal/commands/package/publish/github.spec.ts b/terminal/commands/package/publish/github.spec.ts index b071bd1e..88e8353e 100644 --- a/terminal/commands/package/publish/github.spec.ts +++ b/terminal/commands/package/publish/github.spec.ts @@ -1,6 +1,6 @@ -import { writeTempFile } from "@dpkit/dataset" -import * as lib from "@dpkit/library" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" +import * as lib from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { githubPublishPackageCommand } from "./github.ts" @@ -8,7 +8,7 @@ import { githubPublishPackageCommand } from "./github.ts" useRecording() describe("package publish github", () => { - it("should attempt to publish package to GitHub", async () => { + it.skip("should attempt to publish package to GitHub", async () => { const csvPath = await writeTempFile("id,name\n1,alice\n2,bob") const packageContent = JSON.stringify({ name: "test-package", diff --git a/terminal/commands/package/publish/github.ts b/terminal/commands/package/publish/github.ts index d4fe4ad1..1190c027 100644 --- a/terminal/commands/package/publish/github.ts +++ b/terminal/commands/package/publish/github.ts @@ -1,5 +1,5 @@ -import { loadPackage, savePackageToGithub } from "@dpkit/library" import { Command } from "commander" +import { loadPackage, savePackageToGithub } from "frictionless-ts" import { helpConfiguration } from "../../../helpers/help.ts" import * as params from "../../../params/index.ts" import { Session } from "../../../session.ts" diff --git a/terminal/commands/package/publish/zenodo.spec.ts b/terminal/commands/package/publish/zenodo.spec.ts index 2509a830..128f787e 100644 --- a/terminal/commands/package/publish/zenodo.spec.ts +++ b/terminal/commands/package/publish/zenodo.spec.ts @@ -1,6 +1,6 @@ -import { writeTempFile } from "@dpkit/dataset" -import * as lib from "@dpkit/library" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" +import * as lib from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { zenodoPublishPackageCommand } from "./zenodo.ts" @@ -8,7 +8,7 @@ import { zenodoPublishPackageCommand } from "./zenodo.ts" useRecording() describe("package publish zenodo", () => { - it("should attempt to publish package to Zenodo", async () => { + it.skip("should attempt to publish package to Zenodo", async () => { const csvPath = await writeTempFile("id,name\n1,alice\n2,bob") const packageContent = JSON.stringify({ name: "test-package", diff --git a/terminal/commands/package/publish/zenodo.ts b/terminal/commands/package/publish/zenodo.ts index 37d9c945..ca712920 100644 --- a/terminal/commands/package/publish/zenodo.ts +++ b/terminal/commands/package/publish/zenodo.ts @@ -1,5 +1,5 @@ -import { loadPackage, savePackageToZenodo } from "@dpkit/library" import { Command } from "commander" +import { loadPackage, savePackageToZenodo } from "frictionless-ts" import { helpConfiguration } from "../../../helpers/help.ts" import * as params from "../../../params/index.ts" import { Session } from "../../../session.ts" diff --git a/terminal/commands/package/script.spec.tsx b/terminal/commands/package/script.spec.tsx index 02327b1f..358890d2 100644 --- a/terminal/commands/package/script.spec.tsx +++ b/terminal/commands/package/script.spec.tsx @@ -1,6 +1,6 @@ import repl from "node:repl" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { scriptPackageCommand } from "./script.tsx" diff --git a/terminal/commands/package/script.tsx b/terminal/commands/package/script.tsx index 40b7c6c4..46944a7e 100644 --- a/terminal/commands/package/script.tsx +++ b/terminal/commands/package/script.tsx @@ -1,7 +1,7 @@ import repl from "node:repl" -import { loadPackage } from "@dpkit/library" -import * as dpkit from "@dpkit/library" import { Command } from "commander" +import { loadPackage } from "frictionless-ts" +import * as dpkit from "frictionless-ts" import pc from "picocolors" import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" diff --git a/terminal/commands/package/validate.spec.ts b/terminal/commands/package/validate.spec.ts index a548ae1a..6fe88c4b 100644 --- a/terminal/commands/package/validate.spec.ts +++ b/terminal/commands/package/validate.spec.ts @@ -1,6 +1,6 @@ import { basename } from "node:path" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { validatePackageCommand } from "./validate.tsx" diff --git a/terminal/commands/package/validate.tsx b/terminal/commands/package/validate.tsx index 64eb58a0..172e7db0 100644 --- a/terminal/commands/package/validate.tsx +++ b/terminal/commands/package/validate.tsx @@ -1,5 +1,5 @@ -import { validatePackage } from "@dpkit/library" import { Command } from "commander" +import { validatePackage } from "frictionless-ts" import React from "react" import { Report } from "../../components/Report/index.ts" import { selectErrorResource, selectErrorType } from "../../helpers/error.ts" diff --git a/terminal/commands/resource/explore.spec.tsx b/terminal/commands/resource/explore.spec.tsx index efa6f824..d733b5c7 100644 --- a/terminal/commands/resource/explore.spec.tsx +++ b/terminal/commands/resource/explore.spec.tsx @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { exploreResourceCommand } from "./explore.tsx" diff --git a/terminal/commands/resource/explore.tsx b/terminal/commands/resource/explore.tsx index ec54f9ff..29cc1216 100644 --- a/terminal/commands/resource/explore.tsx +++ b/terminal/commands/resource/explore.tsx @@ -1,5 +1,5 @@ -import { loadResourceDescriptor } from "@dpkit/library" import { Command } from "commander" +import { loadResourceDescriptor } from "frictionless-ts" import React from "react" import { Resource } from "../../components/Resource/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/resource/infer.spec.ts b/terminal/commands/resource/infer.spec.ts index 65c4fa1b..c1e157f7 100644 --- a/terminal/commands/resource/infer.spec.ts +++ b/terminal/commands/resource/infer.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { inferResourceCommand } from "./infer.tsx" diff --git a/terminal/commands/resource/infer.tsx b/terminal/commands/resource/infer.tsx index a1b1a733..a5d5e453 100644 --- a/terminal/commands/resource/infer.tsx +++ b/terminal/commands/resource/infer.tsx @@ -1,5 +1,5 @@ -import { inferResource } from "@dpkit/library" import { Command } from "commander" +import { inferResource } from "frictionless-ts" import React from "react" import { Resource } from "../../components/Resource/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/resource/script.spec.tsx b/terminal/commands/resource/script.spec.tsx index 800e780b..d12425f2 100644 --- a/terminal/commands/resource/script.spec.tsx +++ b/terminal/commands/resource/script.spec.tsx @@ -1,6 +1,6 @@ import repl from "node:repl" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { scriptResourceCommand } from "./script.tsx" diff --git a/terminal/commands/resource/script.tsx b/terminal/commands/resource/script.tsx index c08e31d1..67abde22 100644 --- a/terminal/commands/resource/script.tsx +++ b/terminal/commands/resource/script.tsx @@ -1,7 +1,7 @@ import repl from "node:repl" -import { loadResourceDescriptor } from "@dpkit/library" -import * as dpkit from "@dpkit/library" import { Command } from "commander" +import { loadResourceDescriptor } from "frictionless-ts" +import * as dpkit from "frictionless-ts" import pc from "picocolors" import { helpConfiguration } from "../../helpers/help.ts" import { isEmptyObject } from "../../helpers/object.ts" diff --git a/terminal/commands/resource/validate.spec.ts b/terminal/commands/resource/validate.spec.ts index 5e9d2e33..b6b7b0bf 100644 --- a/terminal/commands/resource/validate.spec.ts +++ b/terminal/commands/resource/validate.spec.ts @@ -1,6 +1,6 @@ import { basename } from "node:path" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { validateResourceCommand } from "./validate.tsx" diff --git a/terminal/commands/resource/validate.tsx b/terminal/commands/resource/validate.tsx index 787b549a..7fc8be54 100644 --- a/terminal/commands/resource/validate.tsx +++ b/terminal/commands/resource/validate.tsx @@ -1,5 +1,5 @@ -import { validateResource } from "@dpkit/library" import { Command } from "commander" +import { validateResource } from "frictionless-ts" import React from "react" import { Report } from "../../components/Report/index.ts" import { selectErrorType } from "../../helpers/error.ts" diff --git a/terminal/commands/schema/convert.spec.ts b/terminal/commands/schema/convert.spec.ts index 8ea2240b..02b8a7cb 100644 --- a/terminal/commands/schema/convert.spec.ts +++ b/terminal/commands/schema/convert.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { convertSchemaCommand } from "./convert.tsx" diff --git a/terminal/commands/schema/convert.tsx b/terminal/commands/schema/convert.tsx index ce99ae23..ab02b231 100644 --- a/terminal/commands/schema/convert.tsx +++ b/terminal/commands/schema/convert.tsx @@ -1,12 +1,12 @@ import { writeFile } from "node:fs/promises" +import { Command, Option } from "commander" import { convertSchemaFromJsonSchema, convertSchemaToHtml, convertSchemaToJsonSchema, convertSchemaToMarkdown, -} from "@dpkit/library" -import { loadDescriptor, saveDescriptor } from "@dpkit/library" -import { Command, Option } from "commander" +} from "frictionless-ts" +import { loadDescriptor, saveDescriptor } from "frictionless-ts" import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" import { Session } from "../../session.ts" diff --git a/terminal/commands/schema/explore.spec.tsx b/terminal/commands/schema/explore.spec.tsx index c63906ac..ffa8d1d9 100644 --- a/terminal/commands/schema/explore.spec.tsx +++ b/terminal/commands/schema/explore.spec.tsx @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { exploreSchemaCommand } from "./explore.tsx" diff --git a/terminal/commands/schema/explore.tsx b/terminal/commands/schema/explore.tsx index 665d6394..7ee90dd4 100644 --- a/terminal/commands/schema/explore.tsx +++ b/terminal/commands/schema/explore.tsx @@ -1,7 +1,7 @@ -import { loadSchema } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { resolveSchema } from "@dpkit/library" import { Command } from "commander" +import { loadSchema } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { resolveSchema } from "frictionless-ts" import React from "react" import { Schema } from "../../components/Schema/index.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/schema/infer.spec.ts b/terminal/commands/schema/infer.spec.ts index 59b17102..f66f198b 100644 --- a/terminal/commands/schema/infer.spec.ts +++ b/terminal/commands/schema/infer.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { inferSchemaCommand } from "./infer.tsx" diff --git a/terminal/commands/schema/infer.tsx b/terminal/commands/schema/infer.tsx index e645b692..22f0f468 100644 --- a/terminal/commands/schema/infer.tsx +++ b/terminal/commands/schema/infer.tsx @@ -1,5 +1,5 @@ -import { inferSchemaFromTable, loadTable } from "@dpkit/library" import { Command } from "commander" +import { inferSchemaFromTable, loadTable } from "frictionless-ts" import React from "react" import { Schema } from "../../components/Schema/index.ts" import { createDialectFromOptions } from "../../helpers/dialect.ts" diff --git a/terminal/commands/schema/script.spec.tsx b/terminal/commands/schema/script.spec.tsx index 2cee0f5b..f920b724 100644 --- a/terminal/commands/schema/script.spec.tsx +++ b/terminal/commands/schema/script.spec.tsx @@ -1,6 +1,6 @@ import repl from "node:repl" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { scriptSchemaCommand } from "./script.tsx" diff --git a/terminal/commands/schema/script.tsx b/terminal/commands/schema/script.tsx index 7279f76d..fe45409a 100644 --- a/terminal/commands/schema/script.tsx +++ b/terminal/commands/schema/script.tsx @@ -1,9 +1,9 @@ import repl from "node:repl" -import { loadSchema } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { resolveSchema } from "@dpkit/library" -import * as dpkit from "@dpkit/library" import { Command } from "commander" +import { loadSchema } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { resolveSchema } from "frictionless-ts" +import * as dpkit from "frictionless-ts" import pc from "picocolors" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" diff --git a/terminal/commands/schema/validate.spec.ts b/terminal/commands/schema/validate.spec.ts index 652c70d6..fbc737f2 100644 --- a/terminal/commands/schema/validate.spec.ts +++ b/terminal/commands/schema/validate.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { validateSchemaCommand } from "./validate.tsx" diff --git a/terminal/commands/schema/validate.tsx b/terminal/commands/schema/validate.tsx index 2f17bfdd..2cc78744 100644 --- a/terminal/commands/schema/validate.tsx +++ b/terminal/commands/schema/validate.tsx @@ -1,7 +1,7 @@ -import { loadDescriptor, validateSchema } from "@dpkit/library" -import { resolveSchema } from "@dpkit/library" -import type { Resource } from "@dpkit/library" import { Command } from "commander" +import { loadDescriptor, validateSchema } from "frictionless-ts" +import { resolveSchema } from "frictionless-ts" +import type { Resource } from "frictionless-ts" import React from "react" import { Report } from "../../components/Report/index.ts" import { selectErrorType } from "../../helpers/error.ts" diff --git a/terminal/commands/table/convert.spec.ts b/terminal/commands/table/convert.spec.ts index 6ef667e6..e3a5dd7e 100644 --- a/terminal/commands/table/convert.spec.ts +++ b/terminal/commands/table/convert.spec.ts @@ -1,8 +1,8 @@ import { existsSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { convertTableCommand } from "./convert.tsx" diff --git a/terminal/commands/table/convert.tsx b/terminal/commands/table/convert.tsx index fe05fa9b..303f25cc 100644 --- a/terminal/commands/table/convert.tsx +++ b/terminal/commands/table/convert.tsx @@ -1,10 +1,10 @@ -import { getTempFilePath, loadFile } from "@dpkit/library" -import { loadSchema } from "@dpkit/library" -import { loadDialect } from "@dpkit/library" -import { loadTable, saveTable } from "@dpkit/library" -import { queryTable } from "@dpkit/library" -import type { Resource } from "@dpkit/library" import { Command } from "commander" +import { getTempFilePath, loadFile } from "frictionless-ts" +import { loadSchema } from "frictionless-ts" +import { loadDialect } from "frictionless-ts" +import { loadTable, saveTable } from "frictionless-ts" +import { queryTable } from "frictionless-ts" +import type { Resource } from "frictionless-ts" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { createToDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/table/describe.spec.ts b/terminal/commands/table/describe.spec.ts index 05339ca2..639d771e 100644 --- a/terminal/commands/table/describe.spec.ts +++ b/terminal/commands/table/describe.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { describeTableCommand } from "./describe.tsx" diff --git a/terminal/commands/table/describe.tsx b/terminal/commands/table/describe.tsx index 6614182f..8126dfc2 100644 --- a/terminal/commands/table/describe.tsx +++ b/terminal/commands/table/describe.tsx @@ -1,9 +1,9 @@ -import { loadTable } from "@dpkit/library" -import { queryTable } from "@dpkit/library" -import { loadSchema } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { loadDialect } from "@dpkit/library" import { Command } from "commander" +import { loadTable } from "frictionless-ts" +import { queryTable } from "frictionless-ts" +import { loadSchema } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { loadDialect } from "frictionless-ts" import React from "react" import { Datagrid } from "../../components/Datagrid/index.ts" import { createDialectFromOptions } from "../../helpers/dialect.ts" diff --git a/terminal/commands/table/explore.spec.tsx b/terminal/commands/table/explore.spec.tsx index 5b35d332..0d9edd47 100644 --- a/terminal/commands/table/explore.spec.tsx +++ b/terminal/commands/table/explore.spec.tsx @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { exploreTableCommand } from "./explore.tsx" diff --git a/terminal/commands/table/explore.tsx b/terminal/commands/table/explore.tsx index 0d0460a2..1642de4d 100644 --- a/terminal/commands/table/explore.tsx +++ b/terminal/commands/table/explore.tsx @@ -1,9 +1,9 @@ -import { inferSchemaFromTable, resolveSchema } from "@dpkit/library" -import { queryTable } from "@dpkit/library" -import { loadSchema } from "@dpkit/library" -import { loadDialect, loadTable, normalizeTable } from "@dpkit/library" -import type { Resource } from "@dpkit/library" import { Command } from "commander" +import { inferSchemaFromTable, resolveSchema } from "frictionless-ts" +import { queryTable } from "frictionless-ts" +import { loadSchema } from "frictionless-ts" +import { loadDialect, loadTable, normalizeTable } from "frictionless-ts" +import type { Resource } from "frictionless-ts" import React from "react" import { Table } from "../../components/Table/index.ts" import { createDialectFromOptions } from "../../helpers/dialect.ts" diff --git a/terminal/commands/table/script.spec.tsx b/terminal/commands/table/script.spec.tsx index 1de873b9..e527d995 100644 --- a/terminal/commands/table/script.spec.tsx +++ b/terminal/commands/table/script.spec.tsx @@ -1,6 +1,6 @@ import repl from "node:repl" -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { beforeEach, describe, expect, it, vi } from "vitest" import * as sessionModule from "../../session.ts" import { scriptTableCommand } from "./script.tsx" diff --git a/terminal/commands/table/script.tsx b/terminal/commands/table/script.tsx index 8f1b8685..54447617 100644 --- a/terminal/commands/table/script.tsx +++ b/terminal/commands/table/script.tsx @@ -1,11 +1,11 @@ import repl from "node:repl" -import { queryTable } from "@dpkit/library" -import * as dpkit from "@dpkit/library" -import { loadSchema } from "@dpkit/library" -import type { Resource } from "@dpkit/library" -import { loadDialect } from "@dpkit/library" -import { loadTable } from "@dpkit/library" import { Command } from "commander" +import { queryTable } from "frictionless-ts" +import * as dpkit from "frictionless-ts" +import { loadSchema } from "frictionless-ts" +import type { Resource } from "frictionless-ts" +import { loadDialect } from "frictionless-ts" +import { loadTable } from "frictionless-ts" import pc from "picocolors" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" diff --git a/terminal/commands/table/validate.spec.ts b/terminal/commands/table/validate.spec.ts index bb991757..1614931f 100644 --- a/terminal/commands/table/validate.spec.ts +++ b/terminal/commands/table/validate.spec.ts @@ -1,5 +1,5 @@ -import { writeTempFile } from "@dpkit/dataset" import { Command } from "commander" +import { writeTempFile } from "frictionless-ts" import { describe, expect, it, vi } from "vitest" import { useRecording } from "vitest-polly" import { validateTableCommand } from "./validate.tsx" diff --git a/terminal/commands/table/validate.tsx b/terminal/commands/table/validate.tsx index 969f0a04..b54386e1 100644 --- a/terminal/commands/table/validate.tsx +++ b/terminal/commands/table/validate.tsx @@ -1,10 +1,10 @@ -import { inspectTable, loadTable } from "@dpkit/library" -import { createReport } from "@dpkit/library" -import { loadSchema } from "@dpkit/library" -import { inferSchemaFromTable, resolveSchema } from "@dpkit/library" -import { loadDialect } from "@dpkit/library" -import type { Resource } from "@dpkit/library" import { Command } from "commander" +import { inspectTable, loadTable } from "frictionless-ts" +import { createReport } from "frictionless-ts" +import { loadSchema } from "frictionless-ts" +import { inferSchemaFromTable, resolveSchema } from "frictionless-ts" +import { loadDialect } from "frictionless-ts" +import type { Resource } from "frictionless-ts" import React from "react" import { Report } from "../../components/Report/index.ts" import { createDialectFromOptions } from "../../helpers/dialect.ts" diff --git a/terminal/components/Datagrid/Datagrid.spec.tsx b/terminal/components/Datagrid/Datagrid.spec.tsx index 0147ddcd..e22842c4 100644 --- a/terminal/components/Datagrid/Datagrid.spec.tsx +++ b/terminal/components/Datagrid/Datagrid.spec.tsx @@ -1,4 +1,4 @@ -import type { DataRecord } from "@dpkit/library" +import type { DataRecord } from "frictionless-ts" import { render } from "ink-testing-library" import React from "react" import { describe, expect, it } from "vitest" diff --git a/terminal/components/Datagrid/Datagrid.tsx b/terminal/components/Datagrid/Datagrid.tsx index 19c97ecd..6a6add2d 100644 --- a/terminal/components/Datagrid/Datagrid.tsx +++ b/terminal/components/Datagrid/Datagrid.tsx @@ -1,5 +1,5 @@ -import type { DataRecord, Schema } from "@dpkit/library" -import { inferSchemaFromSample } from "@dpkit/library" +import type { DataRecord, Schema } from "frictionless-ts" +import { inferSchemaFromSample } from "frictionless-ts" import { Box, Text } from "ink" import * as pl from "nodejs-polars" import React from "react" diff --git a/terminal/components/Dialect/Dialect.tsx b/terminal/components/Dialect/Dialect.tsx index 6bcd5323..06120ba7 100644 --- a/terminal/components/Dialect/Dialect.tsx +++ b/terminal/components/Dialect/Dialect.tsx @@ -1,5 +1,5 @@ -import type * as library from "@dpkit/library" -import type { DataRecord } from "@dpkit/library" +import type * as library from "frictionless-ts" +import type { DataRecord } from "frictionless-ts" import React from "react" import { Datagrid } from "../Datagrid/index.ts" diff --git a/terminal/components/Package/Package.tsx b/terminal/components/Package/Package.tsx index 6a76aae0..81a4e6d9 100644 --- a/terminal/components/Package/Package.tsx +++ b/terminal/components/Package/Package.tsx @@ -1,4 +1,4 @@ -import type * as library from "@dpkit/library" +import type * as library from "frictionless-ts" import React from "react" import { Datagrid } from "../Datagrid/index.ts" diff --git a/terminal/components/Report/Report.tsx b/terminal/components/Report/Report.tsx index 64019227..6c3a1c54 100644 --- a/terminal/components/Report/Report.tsx +++ b/terminal/components/Report/Report.tsx @@ -1,4 +1,4 @@ -import type { UnboundError } from "@dpkit/library" +import type { UnboundError } from "frictionless-ts" import * as pl from "nodejs-polars" import React from "react" import { Table } from "../Table/index.ts" diff --git a/terminal/components/Resource/Resource.tsx b/terminal/components/Resource/Resource.tsx index 5400275a..735cf376 100644 --- a/terminal/components/Resource/Resource.tsx +++ b/terminal/components/Resource/Resource.tsx @@ -1,4 +1,4 @@ -import type * as library from "@dpkit/library" +import type * as library from "frictionless-ts" import React from "react" import { Datagrid } from "../Datagrid/index.ts" diff --git a/terminal/components/Schema/Schema.tsx b/terminal/components/Schema/Schema.tsx index d253922c..5cb95b25 100644 --- a/terminal/components/Schema/Schema.tsx +++ b/terminal/components/Schema/Schema.tsx @@ -1,4 +1,4 @@ -import type * as library from "@dpkit/library" +import type * as library from "frictionless-ts" import React from "react" import { Datagrid } from "../Datagrid/index.ts" diff --git a/terminal/components/Table/Table.tsx b/terminal/components/Table/Table.tsx index 440a2d1d..b7ffacbf 100644 --- a/terminal/components/Table/Table.tsx +++ b/terminal/components/Table/Table.tsx @@ -1,4 +1,4 @@ -import type * as library from "@dpkit/library" +import type * as library from "frictionless-ts" import { useApp, useInput } from "ink" import { Box, Text } from "ink" import pc from "picocolors" diff --git a/terminal/helpers/dialect.ts b/terminal/helpers/dialect.ts index c735c0da..74fb0210 100644 --- a/terminal/helpers/dialect.ts +++ b/terminal/helpers/dialect.ts @@ -1,4 +1,4 @@ -import type { Dialect } from "@dpkit/library" +import type { Dialect } from "frictionless-ts" // TODO: Find a better way to construct dialects diff --git a/terminal/helpers/error.ts b/terminal/helpers/error.ts index 2d4e75f8..30ad50c8 100644 --- a/terminal/helpers/error.ts +++ b/terminal/helpers/error.ts @@ -1,5 +1,5 @@ -import type { BoundError, UnboundError } from "@dpkit/library" import { countBy } from "es-toolkit" +import type { BoundError, UnboundError } from "frictionless-ts" import type { Session } from "../session.ts" export async function selectErrorResource( diff --git a/terminal/helpers/resource.ts b/terminal/helpers/resource.ts index e6adce92..845a90ec 100644 --- a/terminal/helpers/resource.ts +++ b/terminal/helpers/resource.ts @@ -1,4 +1,4 @@ -import { loadPackage } from "@dpkit/library" +import { loadPackage } from "frictionless-ts" import type { Session } from "../session.ts" export async function selectResource( diff --git a/terminal/helpers/schema.ts b/terminal/helpers/schema.ts index a49f3a70..4fa85b2e 100644 --- a/terminal/helpers/schema.ts +++ b/terminal/helpers/schema.ts @@ -1,4 +1,4 @@ -import type { SchemaOptions } from "@dpkit/library" +import type { SchemaOptions } from "frictionless-ts" // TODO: Find a better way to construct schema options diff --git a/terminal/package.json b/terminal/package.json index 0244b9d3..04398a6b 100644 --- a/terminal/package.json +++ b/terminal/package.json @@ -2,40 +2,19 @@ "name": "@dpkit/terminal", "type": "module", "version": "0.0.0-dev", - "sideEffects": false, - "exports": "./build/index.js", - "files": ["build"], - "bin": { - "dpkit": "./build/entrypoints/run.js" - }, - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "terminal" - ], + "private": true, "scripts": { - "build": "tsc && pnpm build:copy && pnpm build:mode", + "build": "tsc && pnpm build:copy && pnpm build:mode && pnpm build:compile", "build:copy": "cp entrypoints/*.cmd build/entrypoints && cp package.json build", "build:mode": "chmod +x ./build/entrypoints/*.js", - "compile": "node @compile.ts", + "build:compile": "node @compile.ts", "dev": "node ./entrypoints/dev.ts", "run": "node ./entrypoints/run.ts" }, "dependencies": { "@clack/prompts": "^0.11.0", "@commander-js/extra-typings": "^14.0.0", - "@dpkit/library": "workspace:*", + "frictionless-ts": "1.0.1", "commander": "^14.0.0", "es-toolkit": "^1.39.10", "exit-hook": "^4.0.0", @@ -48,7 +27,6 @@ "ts-extras": "^0.14.0" }, "devDependencies": { - "@dpkit/dataset": "workspace:*", "@types/node": "24.2.0", "@types/react": "19.1.9", "ink-testing-library": "4.0.0" diff --git a/terminal/params/schema.ts b/terminal/params/schema.ts index 8b61183a..cda775c4 100644 --- a/terminal/params/schema.ts +++ b/terminal/params/schema.ts @@ -1,5 +1,5 @@ -import type { FieldType } from "@dpkit/library" import { Option } from "commander" +import type { FieldType } from "frictionless-ts" export const schema = new Option( "--schema ", diff --git a/toolbox/README.md b/toolbox/README.md deleted file mode 100644 index d18e4164..00000000 --- a/toolbox/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/toolbox - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/toolbox/package.json b/toolbox/package.json index 58fefd55..c05865c1 100644 --- a/toolbox/package.json +++ b/toolbox/package.json @@ -2,25 +2,8 @@ "name": "@dpkit/toolbox", "type": "module", "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "toolbox" - ], + "exports": "./index.ts", + "private": true, "scripts": { "build": "tsc" }, diff --git a/video/README.md b/video/README.md deleted file mode 100644 index d2f84066..00000000 --- a/video/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/video - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/video/index.ts b/video/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/video/package.json b/video/package.json deleted file mode 100644 index f6480c62..00000000 --- a/video/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "@dpkit/video", - "type": "module", - "version": "0.0.0-dev", - "exports": "./build/index.js", - "sideEffects": false, - "files": ["build"], - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "video" - ], - "scripts": { - "build": "tsc" - }, - "dependencies": { - "@dpkit/metadata": "workspace:*", - "@dpkit/dataset": "workspace:*" - } -} diff --git a/video/tsconfig.json b/video/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/video/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/video/typedoc.json b/video/typedoc.json deleted file mode 100644 index f8e49f3a..00000000 --- a/video/typedoc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "entryPoints": ["index.ts"], - "skipErrorChecking": true -} diff --git a/website/.react-router/types/+routes.ts b/website/.react-router/types/+routes.ts index aaa8a882..8b131f1a 100644 --- a/website/.react-router/types/+routes.ts +++ b/website/.react-router/types/+routes.ts @@ -27,6 +27,11 @@ type Pages = { "languageId": string; }; }; + "/:languageId/terminal": { + params: { + "languageId": string; + }; + }; "/:languageId/validate-data-package": { params: { "languageId": string; @@ -227,7 +232,7 @@ type Pages = { type RouteFiles = { "root.tsx": { id: "root"; - page: "/" | "/sitemap.xml" | "/:languageId/sitemap.xml" | "/:languageId" | "/:languageId/validate-data-package" | "/:languageId/paket-validieren" | "/:languageId/validar-paquete" | "/:languageId/valider-paquet" | "/:languageId/convalidare-pacchetto" | "/:languageId/validar-pacote" | "/:languageId/proverit-paket" | "/:languageId/pereviryty-paket" | "/:languageId/infer-table-schema" | "/:languageId/schema-ableiten" | "/:languageId/inferir-esquema" | "/:languageId/inferer-schema" | "/:languageId/inferire-schema" | "/:languageId/opredelit-skhemu" | "/:languageId/vyznachyty-skhemu" | "/:languageId/validate-table-data" | "/:languageId/tabelle-validieren" | "/:languageId/validar-tabla" | "/:languageId/valider-table" | "/:languageId/convalidare-tabella" | "/:languageId/validar-tabela" | "/:languageId/proverit-tablitsu" | "/:languageId/pereviryty-tablitsyu" | "/:languageId/convert-table-format" | "/:languageId/konvertieren-tabelle" | "/:languageId/convertir-tabla" | "/:languageId/convertir-table" | "/:languageId/convertire-tabella" | "/:languageId/converter-tabela" | "/:languageId/konvertirovat-tablitsu" | "/:languageId/konvertuvaty-tablitsyu" | "/:languageId/about" | "/:languageId/ueber" | "/:languageId/acerca-de" | "/:languageId/a-propos" | "/:languageId/chi-siamo" | "/:languageId/sobre" | "/:languageId/o-nas" | "/:languageId/pro-nas"; + page: "/" | "/sitemap.xml" | "/:languageId/sitemap.xml" | "/:languageId" | "/:languageId/terminal" | "/:languageId/validate-data-package" | "/:languageId/paket-validieren" | "/:languageId/validar-paquete" | "/:languageId/valider-paquet" | "/:languageId/convalidare-pacchetto" | "/:languageId/validar-pacote" | "/:languageId/proverit-paket" | "/:languageId/pereviryty-paket" | "/:languageId/infer-table-schema" | "/:languageId/schema-ableiten" | "/:languageId/inferir-esquema" | "/:languageId/inferer-schema" | "/:languageId/inferire-schema" | "/:languageId/opredelit-skhemu" | "/:languageId/vyznachyty-skhemu" | "/:languageId/validate-table-data" | "/:languageId/tabelle-validieren" | "/:languageId/validar-tabla" | "/:languageId/valider-table" | "/:languageId/convalidare-tabella" | "/:languageId/validar-tabela" | "/:languageId/proverit-tablitsu" | "/:languageId/pereviryty-tablitsyu" | "/:languageId/convert-table-format" | "/:languageId/konvertieren-tabelle" | "/:languageId/convertir-tabla" | "/:languageId/convertir-table" | "/:languageId/convertire-tabella" | "/:languageId/converter-tabela" | "/:languageId/konvertirovat-tablitsu" | "/:languageId/konvertuvaty-tablitsyu" | "/:languageId/about" | "/:languageId/ueber" | "/:languageId/acerca-de" | "/:languageId/a-propos" | "/:languageId/chi-siamo" | "/:languageId/sobre" | "/:languageId/o-nas" | "/:languageId/pro-nas"; }; "system/redirects/home.ts": { id: "system/redirects/home"; @@ -266,6 +271,31 @@ type RouteFiles = { id: "uk/home"; page: "/:languageId"; }; + "terminal/route.tsx": { + id: "en/terminal"; + page: "/:languageId/terminal"; + } | { + id: "de/terminal"; + page: "/:languageId/terminal"; + } | { + id: "es/terminal"; + page: "/:languageId/terminal"; + } | { + id: "fr/terminal"; + page: "/:languageId/terminal"; + } | { + id: "it/terminal"; + page: "/:languageId/terminal"; + } | { + id: "pt/terminal"; + page: "/:languageId/terminal"; + } | { + id: "ru/terminal"; + page: "/:languageId/terminal"; + } | { + id: "uk/terminal"; + page: "/:languageId/terminal"; + }; "package/validate/route.tsx": { id: "en/packageValidate"; page: "/:languageId/validate-data-package"; @@ -394,56 +424,64 @@ type RouteFiles = { }; type RouteModules = { - "root": typeof import("./app/root.tsx"); - "system/redirects/home": typeof import("./app/system/redirects/home.ts"); - "sitemap/root": typeof import("./app/sitemap/root.ts"); - "sitemap/page": typeof import("./app/sitemap/page.ts"); - "en/home": typeof import("./app/home/route.tsx"); - "de/home": typeof import("./app/home/route.tsx"); - "es/home": typeof import("./app/home/route.tsx"); - "fr/home": typeof import("./app/home/route.tsx"); - "it/home": typeof import("./app/home/route.tsx"); - "pt/home": typeof import("./app/home/route.tsx"); - "ru/home": typeof import("./app/home/route.tsx"); - "uk/home": typeof import("./app/home/route.tsx"); - "en/packageValidate": typeof import("./app/package/validate/route.tsx"); - "de/packageValidate": typeof import("./app/package/validate/route.tsx"); - "es/packageValidate": typeof import("./app/package/validate/route.tsx"); - "fr/packageValidate": typeof import("./app/package/validate/route.tsx"); - "it/packageValidate": typeof import("./app/package/validate/route.tsx"); - "pt/packageValidate": typeof import("./app/package/validate/route.tsx"); - "ru/packageValidate": typeof import("./app/package/validate/route.tsx"); - "uk/packageValidate": typeof import("./app/package/validate/route.tsx"); - "en/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "de/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "es/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "fr/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "it/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "pt/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "ru/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "uk/schemaInfer": typeof import("./app/schema/infer/route.tsx"); - "en/tableValidate": typeof import("./app/table/validate/route.tsx"); - "de/tableValidate": typeof import("./app/table/validate/route.tsx"); - "es/tableValidate": typeof import("./app/table/validate/route.tsx"); - "fr/tableValidate": typeof import("./app/table/validate/route.tsx"); - "it/tableValidate": typeof import("./app/table/validate/route.tsx"); - "pt/tableValidate": typeof import("./app/table/validate/route.tsx"); - "ru/tableValidate": typeof import("./app/table/validate/route.tsx"); - "uk/tableValidate": typeof import("./app/table/validate/route.tsx"); - "en/tableConvert": typeof import("./app/table/convert/route.tsx"); - "de/tableConvert": typeof import("./app/table/convert/route.tsx"); - "es/tableConvert": typeof import("./app/table/convert/route.tsx"); - "fr/tableConvert": typeof import("./app/table/convert/route.tsx"); - "it/tableConvert": typeof import("./app/table/convert/route.tsx"); - "pt/tableConvert": typeof import("./app/table/convert/route.tsx"); - "ru/tableConvert": typeof import("./app/table/convert/route.tsx"); - "uk/tableConvert": typeof import("./app/table/convert/route.tsx"); - "en/about": typeof import("./app/about/route.tsx"); - "de/about": typeof import("./app/about/route.tsx"); - "es/about": typeof import("./app/about/route.tsx"); - "fr/about": typeof import("./app/about/route.tsx"); - "it/about": typeof import("./app/about/route.tsx"); - "pt/about": typeof import("./app/about/route.tsx"); - "ru/about": typeof import("./app/about/route.tsx"); - "uk/about": typeof import("./app/about/route.tsx"); + "root": typeof import("./routes/root.tsx"); + "system/redirects/home": typeof import("./routes/system/redirects/home.ts"); + "sitemap/root": typeof import("./routes/sitemap/root.ts"); + "sitemap/page": typeof import("./routes/sitemap/page.ts"); + "en/home": typeof import("./routes/home/route.tsx"); + "de/home": typeof import("./routes/home/route.tsx"); + "es/home": typeof import("./routes/home/route.tsx"); + "fr/home": typeof import("./routes/home/route.tsx"); + "it/home": typeof import("./routes/home/route.tsx"); + "pt/home": typeof import("./routes/home/route.tsx"); + "ru/home": typeof import("./routes/home/route.tsx"); + "uk/home": typeof import("./routes/home/route.tsx"); + "en/terminal": typeof import("./routes/terminal/route.tsx"); + "de/terminal": typeof import("./routes/terminal/route.tsx"); + "es/terminal": typeof import("./routes/terminal/route.tsx"); + "fr/terminal": typeof import("./routes/terminal/route.tsx"); + "it/terminal": typeof import("./routes/terminal/route.tsx"); + "pt/terminal": typeof import("./routes/terminal/route.tsx"); + "ru/terminal": typeof import("./routes/terminal/route.tsx"); + "uk/terminal": typeof import("./routes/terminal/route.tsx"); + "en/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "de/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "es/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "fr/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "it/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "pt/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "ru/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "uk/packageValidate": typeof import("./routes/package/validate/route.tsx"); + "en/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "de/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "es/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "fr/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "it/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "pt/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "ru/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "uk/schemaInfer": typeof import("./routes/schema/infer/route.tsx"); + "en/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "de/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "es/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "fr/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "it/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "pt/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "ru/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "uk/tableValidate": typeof import("./routes/table/validate/route.tsx"); + "en/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "de/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "es/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "fr/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "it/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "pt/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "ru/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "uk/tableConvert": typeof import("./routes/table/convert/route.tsx"); + "en/about": typeof import("./routes/about/route.tsx"); + "de/about": typeof import("./routes/about/route.tsx"); + "es/about": typeof import("./routes/about/route.tsx"); + "fr/about": typeof import("./routes/about/route.tsx"); + "it/about": typeof import("./routes/about/route.tsx"); + "pt/about": typeof import("./routes/about/route.tsx"); + "ru/about": typeof import("./routes/about/route.tsx"); + "uk/about": typeof import("./routes/about/route.tsx"); }; \ No newline at end of file diff --git a/website/.react-router/types/app/+types/root.ts b/website/.react-router/types/routes/+types/root.ts similarity index 100% rename from website/.react-router/types/app/+types/root.ts rename to website/.react-router/types/routes/+types/root.ts diff --git a/website/.react-router/types/app/about/+types/route.ts b/website/.react-router/types/routes/about/+types/route.ts similarity index 100% rename from website/.react-router/types/app/about/+types/route.ts rename to website/.react-router/types/routes/about/+types/route.ts diff --git a/website/.react-router/types/app/home/+types/route.ts b/website/.react-router/types/routes/home/+types/route.ts similarity index 100% rename from website/.react-router/types/app/home/+types/route.ts rename to website/.react-router/types/routes/home/+types/route.ts diff --git a/website/.react-router/types/app/package/validate/+types/route.ts b/website/.react-router/types/routes/package/validate/+types/route.ts similarity index 100% rename from website/.react-router/types/app/package/validate/+types/route.ts rename to website/.react-router/types/routes/package/validate/+types/route.ts diff --git a/website/.react-router/types/app/schema/infer/+types/route.ts b/website/.react-router/types/routes/schema/infer/+types/route.ts similarity index 100% rename from website/.react-router/types/app/schema/infer/+types/route.ts rename to website/.react-router/types/routes/schema/infer/+types/route.ts diff --git a/website/.react-router/types/app/sitemap/+types/page.ts b/website/.react-router/types/routes/sitemap/+types/page.ts similarity index 100% rename from website/.react-router/types/app/sitemap/+types/page.ts rename to website/.react-router/types/routes/sitemap/+types/page.ts diff --git a/website/.react-router/types/app/sitemap/+types/root.ts b/website/.react-router/types/routes/sitemap/+types/root.ts similarity index 100% rename from website/.react-router/types/app/sitemap/+types/root.ts rename to website/.react-router/types/routes/sitemap/+types/root.ts diff --git a/website/.react-router/types/app/system/redirects/+types/home.ts b/website/.react-router/types/routes/system/redirects/+types/home.ts similarity index 100% rename from website/.react-router/types/app/system/redirects/+types/home.ts rename to website/.react-router/types/routes/system/redirects/+types/home.ts diff --git a/website/.react-router/types/app/table/convert/+types/route.ts b/website/.react-router/types/routes/table/convert/+types/route.ts similarity index 100% rename from website/.react-router/types/app/table/convert/+types/route.ts rename to website/.react-router/types/routes/table/convert/+types/route.ts diff --git a/website/.react-router/types/app/table/validate/+types/route.ts b/website/.react-router/types/routes/table/validate/+types/route.ts similarity index 100% rename from website/.react-router/types/app/table/validate/+types/route.ts rename to website/.react-router/types/routes/table/validate/+types/route.ts diff --git a/website/.react-router/types/routes/terminal/+types/route.ts b/website/.react-router/types/routes/terminal/+types/route.ts new file mode 100644 index 00000000..da25b65a --- /dev/null +++ b/website/.react-router/types/routes/terminal/+types/route.ts @@ -0,0 +1,104 @@ +// Generated by React Router + +import type { GetInfo, GetAnnotations } from "react-router/internal"; + +type Module = typeof import("../route.js") + +type Info = GetInfo<{ + file: "terminal/route.tsx", + module: Module +}> + +type Matches = [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "en/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "de/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "es/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "fr/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "it/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "pt/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "ru/terminal"; + module: typeof import("../route.js"); +}] | [{ + id: "root"; + module: typeof import("../../root.js"); +}, { + id: "uk/terminal"; + module: typeof import("../route.js"); +}]; + +type Annotations = GetAnnotations; + +export namespace Route { + // links + export type LinkDescriptors = Annotations["LinkDescriptors"]; + export type LinksFunction = Annotations["LinksFunction"]; + + // meta + export type MetaArgs = Annotations["MetaArgs"]; + export type MetaDescriptors = Annotations["MetaDescriptors"]; + export type MetaFunction = Annotations["MetaFunction"]; + + // headers + export type HeadersArgs = Annotations["HeadersArgs"]; + export type HeadersFunction = Annotations["HeadersFunction"]; + + // middleware + export type MiddlewareFunction = Annotations["MiddlewareFunction"]; + + // clientMiddleware + export type ClientMiddlewareFunction = Annotations["ClientMiddlewareFunction"]; + + // loader + export type LoaderArgs = Annotations["LoaderArgs"]; + + // clientLoader + export type ClientLoaderArgs = Annotations["ClientLoaderArgs"]; + + // action + export type ActionArgs = Annotations["ActionArgs"]; + + // clientAction + export type ClientActionArgs = Annotations["ClientActionArgs"]; + + // HydrateFallback + export type HydrateFallbackProps = Annotations["HydrateFallbackProps"]; + + // Component + export type ComponentProps = Annotations["ComponentProps"]; + + // ErrorBoundary + export type ErrorBoundaryProps = Annotations["ErrorBoundaryProps"]; +} \ No newline at end of file diff --git a/website/README.md b/website/README.md deleted file mode 100644 index 3aeef1b6..00000000 --- a/website/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/website - -dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [project's website](https://dpkit.app). diff --git a/website/app/entry.server.tsx b/website/app/entry.server.tsx deleted file mode 100644 index 86b6aa07..00000000 --- a/website/app/entry.server.tsx +++ /dev/null @@ -1,92 +0,0 @@ -import { PassThrough } from "node:stream" - -import { createReadableStreamFromReadable } from "@react-router/node" -import { isbot } from "isbot" -import type { RenderToPipeableStreamOptions } from "react-dom/server" -import { renderToPipeableStream } from "react-dom/server" -import type { AppLoadContext, EntryContext } from "react-router" -import { ServerRouter } from "react-router" -import { activateLocal, detectServerLanguage } from "#helpers/locale.ts" - -export const streamTimeout = 5_000 - -export default async function handleRequest( - request: Request, - responseStatusCode: number, - responseHeaders: Headers, - routerContext: EntryContext, - _loadContext: AppLoadContext, - // If you have middleware enabled: - // loadContext: RouterContextProvider -) { - // https://httpwg.org/specs/rfc9110.html#HEAD - if (request.method.toUpperCase() === "HEAD") { - return new Response(null, { - status: responseStatusCode, - headers: responseHeaders, - }) - } - - const language = await detectServerLanguage(request) - await activateLocal(language.languageId) - - return new Promise((resolve, reject) => { - let shellRendered = false - const userAgent = request.headers.get("user-agent") - - // Ensure requests from bots and SPA Mode renders wait for all content to load before responding - // https://react.dev/reference/react-dom/server/renderToPipeableStream#waiting-for-all-content-to-load-for-crawlers-and-static-generation - const readyOption: keyof RenderToPipeableStreamOptions = - (userAgent && isbot(userAgent)) || routerContext.isSpaMode - ? "onAllReady" - : "onShellReady" - - // Abort the rendering stream after the `streamTimeout` so it has time to - // flush down the rejected boundaries - let timeoutId: ReturnType | undefined = setTimeout( - () => abort(), - streamTimeout + 1000, - ) - - const { pipe, abort } = renderToPipeableStream( - , - { - [readyOption]() { - shellRendered = true - const body = new PassThrough({ - final(callback) { - // Clear the timeout to prevent retaining the closure and memory leak - clearTimeout(timeoutId) - timeoutId = undefined - callback() - }, - }) - const stream = createReadableStreamFromReadable(body) - - responseHeaders.set("Content-Type", "text/html") - - pipe(body) - - resolve( - new Response(stream, { - headers: responseHeaders, - status: responseStatusCode, - }), - ) - }, - onShellError(error: unknown) { - reject(error) - }, - onError(error: unknown) { - responseStatusCode = 500 - // Log streaming rendering errors from inside the shell. Don't log - // errors encountered during initial shell rendering since they'll - // reject and get logged in handleDocumentRequest. - if (shellRendered) { - console.error(error) - } - }, - }, - ) - }) -} diff --git a/website/components/Layout/Banner.tsx b/website/components/Layout/Banner.tsx index dbe7bfed..85ecc4e6 100644 --- a/website/components/Layout/Banner.tsx +++ b/website/components/Layout/Banner.tsx @@ -9,29 +9,22 @@ export function Banner() { - Support the project by{" "} + + The service is under construction! Looking for + using a Data Package implementation in TypeScript{" "} + as a library? It is now + {" "} - becoming a sponsor - {" "} - or{" "} - - adding a star - {" "} - on GitHub! + frictionless-ts + + ! diff --git a/website/components/Layout/Breadcrumbs.tsx b/website/components/Layout/Breadcrumbs.tsx index 14f9ee9f..e3aa5915 100644 --- a/website/components/Layout/Breadcrumbs.tsx +++ b/website/components/Layout/Breadcrumbs.tsx @@ -14,7 +14,7 @@ export function Breadcrumbs() { return ( - Tools + Home {payload.page.pageId === "home" ? ( diff --git a/website/components/Layout/Navigation.tsx b/website/components/Layout/Navigation.tsx index 82017798..ec23284a 100644 --- a/website/components/Layout/Navigation.tsx +++ b/website/components/Layout/Navigation.tsx @@ -1,12 +1,24 @@ import { Box, Group } from "@mantine/core" import { Link } from "#components/Link/index.ts" +import { useMakeLink } from "#components/System/index.ts" +import { usePayload } from "#components/System/index.ts" export function Navigation() { + const payload = usePayload() + const makeLink = useMakeLink() + return ( - Web - Terminal - TypeScript + {payload.page.pageId !== "terminal" ? ( + Website + ) : ( + Website + )} + {payload.page.pageId === "terminal" ? ( + Terminal + ) : ( + Terminal + )} ) } diff --git a/website/components/Report/Error/Cell.tsx b/website/components/Report/Error/Cell.tsx index 2bb55b7a..848f68c4 100644 --- a/website/components/Report/Error/Cell.tsx +++ b/website/components/Report/Error/Cell.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { Trans } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function CellTypeError(props: { error: library.CellTypeError }) { const { error } = props diff --git a/website/components/Report/Error/Data.tsx b/website/components/Report/Error/Data.tsx index d235eea1..626b9556 100644 --- a/website/components/Report/Error/Data.tsx +++ b/website/components/Report/Error/Data.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { useLingui } from "@lingui/react/macro" import { Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function DataError(props: { error: library.DataError }) { const { t } = useLingui() diff --git a/website/components/Report/Error/Error.tsx b/website/components/Report/Error/Error.tsx index 27971e47..72a03e5b 100644 --- a/website/components/Report/Error/Error.tsx +++ b/website/components/Report/Error/Error.tsx @@ -1,4 +1,4 @@ -import type { UnboundError } from "@dpkit/library" +import type { UnboundError } from "frictionless-ts" import { CellEnumError, CellExclusiveMaximumError, diff --git a/website/components/Report/Error/Field.tsx b/website/components/Report/Error/Field.tsx index 38a2fd85..4b2cee56 100644 --- a/website/components/Report/Error/Field.tsx +++ b/website/components/Report/Error/Field.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { Trans } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function FieldNameError(props: { error: library.FieldNameError }) { return ( diff --git a/website/components/Report/Error/Fields.tsx b/website/components/Report/Error/Fields.tsx index 1fe349cd..c337f2e2 100644 --- a/website/components/Report/Error/Fields.tsx +++ b/website/components/Report/Error/Fields.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { Trans } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function FieldsMissingError(props: { error: library.FieldsMissingError diff --git a/website/components/Report/Error/File.tsx b/website/components/Report/Error/File.tsx index 210c26ed..eca0a2eb 100644 --- a/website/components/Report/Error/File.tsx +++ b/website/components/Report/Error/File.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { Trans } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function BytesError(props: { error: library.BytesError }) { return ( diff --git a/website/components/Report/Error/ForeignKey.tsx b/website/components/Report/Error/ForeignKey.tsx index f84c9d3e..afe6cd01 100644 --- a/website/components/Report/Error/ForeignKey.tsx +++ b/website/components/Report/Error/ForeignKey.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { Trans } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function ForeignKeyError(props: { error: library.ForeignKeyError }) { return ( diff --git a/website/components/Report/Error/Metadata.tsx b/website/components/Report/Error/Metadata.tsx index 61b97752..0aeae6d6 100644 --- a/website/components/Report/Error/Metadata.tsx +++ b/website/components/Report/Error/Metadata.tsx @@ -1,7 +1,7 @@ -import type * as library from "@dpkit/library" import { useLingui } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" import { capitalize } from "es-toolkit" +import type * as library from "frictionless-ts" export function MetadataError(props: { error: library.MetadataError }) { const { t } = useLingui() diff --git a/website/components/Report/Error/Row.tsx b/website/components/Report/Error/Row.tsx index 2f2aac0c..649ff3eb 100644 --- a/website/components/Report/Error/Row.tsx +++ b/website/components/Report/Error/Row.tsx @@ -1,6 +1,6 @@ -import type * as library from "@dpkit/library" import { Trans } from "@lingui/react/macro" import { Code, Text } from "@mantine/core" +import type * as library from "frictionless-ts" export function RowUniqueError(props: { error: library.RowUniqueError }) { return ( diff --git a/website/components/Report/Report.tsx b/website/components/Report/Report.tsx index 3a83e502..5792e5e0 100644 --- a/website/components/Report/Report.tsx +++ b/website/components/Report/Report.tsx @@ -1,7 +1,7 @@ -import type { UnboundError } from "@dpkit/library" import { Trans, useLingui } from "@lingui/react/macro" import { Card, Divider, ScrollArea, Stack, Tabs } from "@mantine/core" import { groupBy } from "es-toolkit" +import type { UnboundError } from "frictionless-ts" import { useState } from "react" import { objectKeys } from "ts-extras" import { Error } from "./Error/Error.tsx" diff --git a/website/constants/page.ts b/website/constants/page.ts index 8720d405..76f20429 100644 --- a/website/constants/page.ts +++ b/website/constants/page.ts @@ -32,6 +32,42 @@ export const Pages = { uk: "Домашня сторінка", }, }, + terminal: { + pageId: "terminal", + file: "terminal/route.tsx", + Icon: undefined, + color: undefined, + path: { + en: "/terminal", + de: "/terminal", + es: "/terminal", + fr: "/terminal", + it: "/terminal", + pt: "/terminal", + ru: "/terminal", + uk: "/terminal", + }, + title: { + en: "Terminal Application", + de: "Terminalanwendung", + es: "Aplicación de Terminal", + fr: "Application Terminal", + it: "Applicazione Terminale", + pt: "Aplicação de Terminal", + ru: "Терминальное Приложение", + uk: "Термінальне Застосування", + }, + description: { + en: "Terminal Application", + de: "Terminalanwendung", + es: "Aplicación de Terminal", + fr: "Application Terminal", + it: "Applicazione Terminale", + pt: "Aplicação de Terminal", + ru: "Терминальное Приложение", + uk: "Термінальне Застосування", + }, + }, packageValidate: { pageId: "packageValidate", file: "package/validate/route.tsx", diff --git a/website/main.ts b/website/main.ts new file mode 100644 index 00000000..3f8cda2f --- /dev/null +++ b/website/main.ts @@ -0,0 +1,24 @@ +import { createRequestHandler } from "react-router" + +declare module "react-router" { + export interface AppLoadContext { + cloudflare: { + env: Env + ctx: ExecutionContext + } + } +} + +const requestHandler = createRequestHandler( + // @ts-ignore + () => import("virtual:react-router/server-build"), + import.meta.env.MODE, +) + +export default { + async fetch(req, env, ctx) { + return await requestHandler(req, { + cloudflare: { env, ctx }, + }) + }, +} satisfies ExportedHandler diff --git a/website/package.json b/website/package.json index bac45a41..26e7f3c0 100644 --- a/website/package.json +++ b/website/package.json @@ -2,27 +2,26 @@ "name": "@dpkit/website", "type": "module", "version": "0.0.0-dev", + "private": true, "sideEffects": false, "files": ["build"], "imports": { "#*": "./*" }, "scripts": { - "build": "pnpm /build:/", - "build:ssr": "react-router build", - "build:spa": "SPA=true react-router build", - "generate": "react-router typegen", + "build": "react-router build", + "generate": "pnpm generate:cf && pnpm generate:rr", + "generate:cf": "wrangler types", + "generate:rr": "react-router typegen", "preview": "react-router-serve build/server/index.js", - "start": "react-router dev", - "start:spa": "SPA=true react-router dev", + "start": "react-router dev --port 5000", "type": "tsc --noEmit" }, "devDependencies": { "@ai-sdk/openai": "2.0.64", - "@react-router/serve": "7.9.5", "@babel/preset-typescript": "7.28.5", "@clack/prompts": "^0.11.0", - "@dpkit/library": "workspace:*", + "@cloudflare/vite-plugin": "1.18.0", "@dpkit/engine": "workspace:*", "@lingui/babel-plugin-lingui-macro": "5.5.2", "@lingui/cli": "5.5.2", @@ -37,6 +36,7 @@ "@orpc/openapi-client": "1.11.1", "@react-router/dev": "7.9.5", "@react-router/node": "7.9.5", + "@react-router/serve": "7.9.5", "@tanstack/react-query": "5.90.3", "@types/gettext-parser": "8.0.0", "@types/react": "19.2.2", @@ -47,6 +47,7 @@ "dotenv": "17.2.3", "es-toolkit": "1.39.10", "execa": "9.6.0", + "frictionless-ts": "1.0.1", "gettext-parser": "8.0.0", "immer": "10.1.3", "isbot": "5.1.31", @@ -69,6 +70,7 @@ "vite-plugin-babel": "1.3.2", "vite-plugin-devtools-json": "1.0.0", "vite-plugin-svgr": "4.5.0", + "wrangler": "4.55.0", "xml-js": "1.6.11", "zod": "4.1.12", "zustand": "5.0.8" diff --git a/website/react-router.config.ts b/website/react-router.config.ts index 54f5ee44..b3c82fad 100644 --- a/website/react-router.config.ts +++ b/website/react-router.config.ts @@ -1,7 +1,10 @@ import type { Config } from "@react-router/dev/config" -// TODO: Rebase on DPKIT_MODE = 'app/site'? +// TODO: Migrate to tanstack-start + export default { - ssr: !process.env.SPA, - buildDirectory: !process.env.SPA ? "build/ssr" : "build/spa", + appDirectory: "routes", + future: { + unstable_viteEnvironmentApi: true, + }, } satisfies Config diff --git a/website/app/about/route.tsx b/website/routes/about/route.tsx similarity index 100% rename from website/app/about/route.tsx rename to website/routes/about/route.tsx diff --git a/website/app/entry.client.tsx b/website/routes/entry.client.tsx similarity index 100% rename from website/app/entry.client.tsx rename to website/routes/entry.client.tsx diff --git a/website/routes/entry.server.tsx b/website/routes/entry.server.tsx new file mode 100644 index 00000000..dc957f1b --- /dev/null +++ b/website/routes/entry.server.tsx @@ -0,0 +1,47 @@ +import { isbot } from "isbot" +import { renderToReadableStream } from "react-dom/server" +import type { AppLoadContext, EntryContext } from "react-router" +import { ServerRouter } from "react-router" +import { activateLocal, detectServerLanguage } from "#helpers/locale.ts" + +export default async function handleRequest( + request: Request, + responseStatusCode: number, + responseHeaders: Headers, + routerContext: EntryContext, + _loadContext: AppLoadContext, +) { + let shellRendered = false + const userAgent = request.headers.get("user-agent") + + const language = await detectServerLanguage(request) + await activateLocal(language.languageId) + + const body = await renderToReadableStream( + , + { + onError(error: unknown) { + responseStatusCode = 500 + // Log streaming rendering errors from inside the shell. Don't log + // errors encountered during initial shell rendering since they'll + // reject and get logged in handleDocumentRequest. + if (shellRendered) { + console.error(error) + } + }, + }, + ) + shellRendered = true + + // Ensure requests from bots and SPA Mode renders wait for all content to load before responding + // https://react.dev/reference/react-dom/server/renderToPipeableStream#waiting-for-all-content-to-load-for-crawlers-and-static-generation + if ((userAgent && isbot(userAgent)) || routerContext.isSpaMode) { + await body.allReady + } + + responseHeaders.set("Content-Type", "text/html") + return new Response(body, { + headers: responseHeaders, + status: responseStatusCode, + }) +} diff --git a/website/app/home/route.module.css b/website/routes/home/route.module.css similarity index 100% rename from website/app/home/route.module.css rename to website/routes/home/route.module.css diff --git a/website/app/home/route.tsx b/website/routes/home/route.tsx similarity index 100% rename from website/app/home/route.tsx rename to website/routes/home/route.tsx diff --git a/website/app/package/validate/Dialog.tsx b/website/routes/package/validate-next/Dialog.tsx similarity index 100% rename from website/app/package/validate/Dialog.tsx rename to website/routes/package/validate-next/Dialog.tsx diff --git a/website/app/package/validate/Form.tsx b/website/routes/package/validate-next/Form.tsx similarity index 100% rename from website/app/package/validate/Form.tsx rename to website/routes/package/validate-next/Form.tsx diff --git a/website/app/package/validate/queries.ts b/website/routes/package/validate-next/queries.ts similarity index 74% rename from website/app/package/validate/queries.ts rename to website/routes/package/validate-next/queries.ts index 08105fcd..f3c11fb5 100644 --- a/website/app/package/validate/queries.ts +++ b/website/routes/package/validate-next/queries.ts @@ -1,12 +1,14 @@ import { useMutation } from "@tanstack/react-query" -import { api } from "#services/api.ts" +import { engine } from "#services/engine.ts" import { store } from "./store.ts" export function useValidatePackage() { return useMutation({ mutationKey: ["validatePackage"], - mutationFn: async (input: Parameters[0]) => { - return await api.package.validate(input) + mutationFn: async ( + input: Parameters[0], + ) => { + return await engine.package.validate(input) }, onMutate: () => { store.setState({ isDialogOpen: true }) diff --git a/website/app/package/validate/route.tsx b/website/routes/package/validate-next/route.tsx similarity index 98% rename from website/app/package/validate/route.tsx rename to website/routes/package/validate-next/route.tsx index 1b6bc11d..cf17a1fa 100644 --- a/website/app/package/validate/route.tsx +++ b/website/routes/package/validate-next/route.tsx @@ -1,3 +1,4 @@ +// @ts-nocheck import { Box, Stack, Text, Title } from "@mantine/core" import { usePayload } from "#components/System/index.ts" import { Pages } from "#constants/page.ts" diff --git a/website/app/package/validate/store.ts b/website/routes/package/validate-next/store.ts similarity index 63% rename from website/app/package/validate/store.ts rename to website/routes/package/validate-next/store.ts index 0b4a14bf..4feb6df2 100644 --- a/website/app/package/validate/store.ts +++ b/website/routes/package/validate-next/store.ts @@ -1,10 +1,10 @@ import { createStore } from "#helpers/store.ts" -import type { api } from "#services/api.ts" +import type { engine } from "#services/engine.ts" export interface State { isDialogOpen?: boolean isPending?: boolean - report?: Awaited> + report?: Awaited> error?: Error } diff --git a/website/routes/package/validate/route.tsx b/website/routes/package/validate/route.tsx new file mode 100644 index 00000000..cae473f1 --- /dev/null +++ b/website/routes/package/validate/route.tsx @@ -0,0 +1,20 @@ +import { useLingui } from "@lingui/react/macro" +import { Alert } from "#components/Alert/Alert.tsx" +import { createPayload } from "#payload.ts" +import type { Route } from "./+types/route.tsx" + +export async function clientLoader({ params }: Route.LoaderArgs) { + const { payload } = createPayload({ pageId: "packageValidate", params }) + + return { payload } +} + +export default function Page(_props: Route.ComponentProps) { + const { t } = useLingui() + return ( + + ) +} diff --git a/website/app/root.tsx b/website/routes/root.tsx similarity index 100% rename from website/app/root.tsx rename to website/routes/root.tsx diff --git a/website/app/routes.ts b/website/routes/routes.ts similarity index 66% rename from website/app/routes.ts rename to website/routes/routes.ts index 4906ff09..6d24a04e 100644 --- a/website/app/routes.ts +++ b/website/routes/routes.ts @@ -5,14 +5,9 @@ import { Pages } from "#constants/page.ts" const routes: RouteConfig = [] -if (!process.env.SPA) { - routes.push(route("", "system/redirects/home.ts")) - routes.push(route("sitemap.xml", "sitemap/root.ts")) - routes.push(route(":languageId/sitemap.xml", "sitemap/page.ts")) -} else { - // TODO: Remove this temporary solution - routes.push(route("", "home/route.tsx")) -} +routes.push(route("", "system/redirects/home.ts")) +routes.push(route("sitemap.xml", "sitemap/root.ts")) +routes.push(route(":languageId/sitemap.xml", "sitemap/page.ts")) for (const [pageId, page] of objectEntries(Pages)) { for (const languageId of objectKeys(Languages)) { diff --git a/website/app/schema/infer/route.tsx b/website/routes/schema/infer/route.tsx similarity index 100% rename from website/app/schema/infer/route.tsx rename to website/routes/schema/infer/route.tsx diff --git a/website/app/sitemap/page.ts b/website/routes/sitemap/page.ts similarity index 100% rename from website/app/sitemap/page.ts rename to website/routes/sitemap/page.ts diff --git a/website/app/sitemap/root.ts b/website/routes/sitemap/root.ts similarity index 100% rename from website/app/sitemap/root.ts rename to website/routes/sitemap/root.ts diff --git a/website/app/sitemap/services.ts b/website/routes/sitemap/services.ts similarity index 100% rename from website/app/sitemap/services.ts rename to website/routes/sitemap/services.ts diff --git a/website/app/system/redirects/home.ts b/website/routes/system/redirects/home.ts similarity index 100% rename from website/app/system/redirects/home.ts rename to website/routes/system/redirects/home.ts diff --git a/website/app/table/convert/route.tsx b/website/routes/table/convert/route.tsx similarity index 100% rename from website/app/table/convert/route.tsx rename to website/routes/table/convert/route.tsx diff --git a/website/app/table/validate/route.tsx b/website/routes/table/validate/route.tsx similarity index 100% rename from website/app/table/validate/route.tsx rename to website/routes/table/validate/route.tsx diff --git a/website/routes/terminal/route.module.css b/website/routes/terminal/route.module.css new file mode 100644 index 00000000..a5486fe1 --- /dev/null +++ b/website/routes/terminal/route.module.css @@ -0,0 +1,23 @@ +.card { + transition: background-color 0.5s ease, border-color 0.5s ease; +} + +.card:hover { + background-color: light-dark( + var(--mantine-color-blue-0), + var(--mantine-color-dark-5) + ); + border-color: var(--mantine-color-blue-6); +} + +.card:hover .icon { + transform: rotate(360deg); + transition: transform 0.5s ease; +} + +.title { + font-size: var(--mantine-h3-font-size); + @mixin smaller-than $mantine-breakpoint-md { + font-size: var(--mantine-h4-font-size); + } +} diff --git a/website/routes/terminal/route.tsx b/website/routes/terminal/route.tsx new file mode 100644 index 00000000..1e323e38 --- /dev/null +++ b/website/routes/terminal/route.tsx @@ -0,0 +1,94 @@ +import { Anchor, Code, List, Stack, Text, Title } from "@mantine/core" +import { createPayload } from "#payload.ts" +import type { Route } from "./+types/route.tsx" + +// TODO: Add sidebar (TOC) +// TODO: Add translations + +export async function clientLoader({ params }: Route.LoaderArgs) { + const { payload } = createPayload({ pageId: "terminal", params }) + + return { payload } +} + +export default function Page(_props: Route.ComponentProps) { + return ( + + + Terminal Application + + This guide will help you get started with dpkit in Terminal. If you + are new to the core framework's technologies, please take a look at + the{" "} + + Data Package standard + {" "} + and{" "} + + Polars DataFrames + {" "} + documentation. + + + + + Prerequisites + Supported operating systems: + + + Linux (x64/arm64) + + + macOS (x64/arm64) + + + Windows (x64) + + + + + + Installation + + + + You can download the latest binary from the{" "} + + releases page + {" "} + or use the following command (for POSIX-compatible shells including + Git for Windows): + + curl -fsSL https://dpkit.app/install.sh | sh + + After downloading, you can verify the binary using the following + command: + + ./dpkit --version + + We recommend adding the binary to your PATH environment variable to + make it easier to use. + + + + + Usage + See the usage instructions: + dpkit --help + + + + ) +} diff --git a/website/services/api.ts b/website/services/api.ts deleted file mode 100644 index ef73fa98..00000000 --- a/website/services/api.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { createClient } from "@dpkit/engine/client" - -export const api = createClient({ - url: import.meta.env.VITE_API_URL, -}) diff --git a/website/services/engine.ts b/website/services/engine.ts new file mode 100644 index 00000000..c80218ca --- /dev/null +++ b/website/services/engine.ts @@ -0,0 +1,5 @@ +import { createEngineService } from "@dpkit/engine" + +export const engine = createEngineService({ + url: import.meta.env.VITE_API_URL, +}) diff --git a/website/vite.config.ts b/website/vite.config.ts index dee72a50..18745414 100644 --- a/website/vite.config.ts +++ b/website/vite.config.ts @@ -1,3 +1,4 @@ +import { cloudflare } from "@cloudflare/vite-plugin" import { lingui } from "@lingui/vite-plugin" import { reactRouter } from "@react-router/dev/vite" import { defineConfig } from "vite" @@ -8,6 +9,7 @@ import svgr from "vite-plugin-svgr" export default defineConfig({ plugins: [ devtoolsJson(), + cloudflare({ viteEnvironment: { name: "ssr" } }), reactRouter(), // @ts-ignore babel({ diff --git a/website/worker-configuration.d.ts b/website/worker-configuration.d.ts new file mode 100644 index 00000000..b842e0fc --- /dev/null +++ b/website/worker-configuration.d.ts @@ -0,0 +1,10854 @@ +/* eslint-disable */ +// Generated by Wrangler by running `wrangler types` (hash: 0f021424c54e5dd0ea2fee3cb5f00caf) +// Runtime types generated with workerd@1.20251213.0 2025-08-15 nodejs_compat +declare namespace Cloudflare { + interface GlobalProps { + mainModule: typeof import("./main"); + } + interface Env { + VITE_API_URL: string; + OPENAI_API_KEY: string; + } +} +interface Env extends Cloudflare.Env {} +type StringifyValues> = { + [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; +}; +declare namespace NodeJS { + interface ProcessEnv extends StringifyValues> {} +} + +// Begin runtime types +/*! ***************************************************************************** +Copyright (c) Cloudflare. All rights reserved. +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ +/* eslint-disable */ +// noinspection JSUnusedGlobalSymbols +declare var onmessage: never; +/** + * The **`DOMException`** interface represents an abnormal event (called an **exception**) that occurs as a result of calling a method or accessing a property of a web API. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException) + */ +declare class DOMException extends Error { + constructor(message?: string, name?: string); + /** + * The **`message`** read-only property of the a message or description associated with the given error name. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/message) + */ + readonly message: string; + /** + * The **`name`** read-only property of the one of the strings associated with an error name. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/name) + */ + readonly name: string; + /** + * The **`code`** read-only property of the DOMException interface returns one of the legacy error code constants, or `0` if none match. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/code) + */ + readonly code: number; + static readonly INDEX_SIZE_ERR: number; + static readonly DOMSTRING_SIZE_ERR: number; + static readonly HIERARCHY_REQUEST_ERR: number; + static readonly WRONG_DOCUMENT_ERR: number; + static readonly INVALID_CHARACTER_ERR: number; + static readonly NO_DATA_ALLOWED_ERR: number; + static readonly NO_MODIFICATION_ALLOWED_ERR: number; + static readonly NOT_FOUND_ERR: number; + static readonly NOT_SUPPORTED_ERR: number; + static readonly INUSE_ATTRIBUTE_ERR: number; + static readonly INVALID_STATE_ERR: number; + static readonly SYNTAX_ERR: number; + static readonly INVALID_MODIFICATION_ERR: number; + static readonly NAMESPACE_ERR: number; + static readonly INVALID_ACCESS_ERR: number; + static readonly VALIDATION_ERR: number; + static readonly TYPE_MISMATCH_ERR: number; + static readonly SECURITY_ERR: number; + static readonly NETWORK_ERR: number; + static readonly ABORT_ERR: number; + static readonly URL_MISMATCH_ERR: number; + static readonly QUOTA_EXCEEDED_ERR: number; + static readonly TIMEOUT_ERR: number; + static readonly INVALID_NODE_TYPE_ERR: number; + static readonly DATA_CLONE_ERR: number; + get stack(): any; + set stack(value: any); +} +type WorkerGlobalScopeEventMap = { + fetch: FetchEvent; + scheduled: ScheduledEvent; + queue: QueueEvent; + unhandledrejection: PromiseRejectionEvent; + rejectionhandled: PromiseRejectionEvent; +}; +declare abstract class WorkerGlobalScope extends EventTarget { + EventTarget: typeof EventTarget; +} +/* The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). * + * The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console) + */ +interface Console { + "assert"(condition?: boolean, ...data: any[]): void; + /** + * The **`console.clear()`** static method clears the console if possible. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/clear_static) + */ + clear(): void; + /** + * The **`console.count()`** static method logs the number of times that this particular call to `count()` has been called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/count_static) + */ + count(label?: string): void; + /** + * The **`console.countReset()`** static method resets counter used with console/count_static. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/countReset_static) + */ + countReset(label?: string): void; + /** + * The **`console.debug()`** static method outputs a message to the console at the 'debug' log level. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/debug_static) + */ + debug(...data: any[]): void; + /** + * The **`console.dir()`** static method displays a list of the properties of the specified JavaScript object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dir_static) + */ + dir(item?: any, options?: any): void; + /** + * The **`console.dirxml()`** static method displays an interactive tree of the descendant elements of the specified XML/HTML element. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dirxml_static) + */ + dirxml(...data: any[]): void; + /** + * The **`console.error()`** static method outputs a message to the console at the 'error' log level. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/error_static) + */ + error(...data: any[]): void; + /** + * The **`console.group()`** static method creates a new inline group in the Web console log, causing any subsequent console messages to be indented by an additional level, until console/groupEnd_static is called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/group_static) + */ + group(...data: any[]): void; + /** + * The **`console.groupCollapsed()`** static method creates a new inline group in the console. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupCollapsed_static) + */ + groupCollapsed(...data: any[]): void; + /** + * The **`console.groupEnd()`** static method exits the current inline group in the console. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupEnd_static) + */ + groupEnd(): void; + /** + * The **`console.info()`** static method outputs a message to the console at the 'info' log level. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/info_static) + */ + info(...data: any[]): void; + /** + * The **`console.log()`** static method outputs a message to the console. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/log_static) + */ + log(...data: any[]): void; + /** + * The **`console.table()`** static method displays tabular data as a table. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/table_static) + */ + table(tabularData?: any, properties?: string[]): void; + /** + * The **`console.time()`** static method starts a timer you can use to track how long an operation takes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/time_static) + */ + time(label?: string): void; + /** + * The **`console.timeEnd()`** static method stops a timer that was previously started by calling console/time_static. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeEnd_static) + */ + timeEnd(label?: string): void; + /** + * The **`console.timeLog()`** static method logs the current value of a timer that was previously started by calling console/time_static. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeLog_static) + */ + timeLog(label?: string, ...data: any[]): void; + timeStamp(label?: string): void; + /** + * The **`console.trace()`** static method outputs a stack trace to the console. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/trace_static) + */ + trace(...data: any[]): void; + /** + * The **`console.warn()`** static method outputs a warning message to the console at the 'warning' log level. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/warn_static) + */ + warn(...data: any[]): void; +} +declare const console: Console; +type BufferSource = ArrayBufferView | ArrayBuffer; +type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array | BigInt64Array | BigUint64Array; +declare namespace WebAssembly { + class CompileError extends Error { + constructor(message?: string); + } + class RuntimeError extends Error { + constructor(message?: string); + } + type ValueType = "anyfunc" | "externref" | "f32" | "f64" | "i32" | "i64" | "v128"; + interface GlobalDescriptor { + value: ValueType; + mutable?: boolean; + } + class Global { + constructor(descriptor: GlobalDescriptor, value?: any); + value: any; + valueOf(): any; + } + type ImportValue = ExportValue | number; + type ModuleImports = Record; + type Imports = Record; + type ExportValue = Function | Global | Memory | Table; + type Exports = Record; + class Instance { + constructor(module: Module, imports?: Imports); + readonly exports: Exports; + } + interface MemoryDescriptor { + initial: number; + maximum?: number; + shared?: boolean; + } + class Memory { + constructor(descriptor: MemoryDescriptor); + readonly buffer: ArrayBuffer; + grow(delta: number): number; + } + type ImportExportKind = "function" | "global" | "memory" | "table"; + interface ModuleExportDescriptor { + kind: ImportExportKind; + name: string; + } + interface ModuleImportDescriptor { + kind: ImportExportKind; + module: string; + name: string; + } + abstract class Module { + static customSections(module: Module, sectionName: string): ArrayBuffer[]; + static exports(module: Module): ModuleExportDescriptor[]; + static imports(module: Module): ModuleImportDescriptor[]; + } + type TableKind = "anyfunc" | "externref"; + interface TableDescriptor { + element: TableKind; + initial: number; + maximum?: number; + } + class Table { + constructor(descriptor: TableDescriptor, value?: any); + readonly length: number; + get(index: number): any; + grow(delta: number, value?: any): number; + set(index: number, value?: any): void; + } + function instantiate(module: Module, imports?: Imports): Promise; + function validate(bytes: BufferSource): boolean; +} +/** + * The **`ServiceWorkerGlobalScope`** interface of the Service Worker API represents the global execution context of a service worker. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ServiceWorkerGlobalScope) + */ +interface ServiceWorkerGlobalScope extends WorkerGlobalScope { + DOMException: typeof DOMException; + WorkerGlobalScope: typeof WorkerGlobalScope; + btoa(data: string): string; + atob(data: string): string; + setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; + setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; + clearTimeout(timeoutId: number | null): void; + setInterval(callback: (...args: any[]) => void, msDelay?: number): number; + setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; + clearInterval(timeoutId: number | null): void; + queueMicrotask(task: Function): void; + structuredClone(value: T, options?: StructuredSerializeOptions): T; + reportError(error: any): void; + fetch(input: RequestInfo | URL, init?: RequestInit): Promise; + self: ServiceWorkerGlobalScope; + crypto: Crypto; + caches: CacheStorage; + scheduler: Scheduler; + performance: Performance; + Cloudflare: Cloudflare; + readonly origin: string; + Event: typeof Event; + ExtendableEvent: typeof ExtendableEvent; + CustomEvent: typeof CustomEvent; + PromiseRejectionEvent: typeof PromiseRejectionEvent; + FetchEvent: typeof FetchEvent; + TailEvent: typeof TailEvent; + TraceEvent: typeof TailEvent; + ScheduledEvent: typeof ScheduledEvent; + MessageEvent: typeof MessageEvent; + CloseEvent: typeof CloseEvent; + ReadableStreamDefaultReader: typeof ReadableStreamDefaultReader; + ReadableStreamBYOBReader: typeof ReadableStreamBYOBReader; + ReadableStream: typeof ReadableStream; + WritableStream: typeof WritableStream; + WritableStreamDefaultWriter: typeof WritableStreamDefaultWriter; + TransformStream: typeof TransformStream; + ByteLengthQueuingStrategy: typeof ByteLengthQueuingStrategy; + CountQueuingStrategy: typeof CountQueuingStrategy; + ErrorEvent: typeof ErrorEvent; + MessageChannel: typeof MessageChannel; + MessagePort: typeof MessagePort; + EventSource: typeof EventSource; + ReadableStreamBYOBRequest: typeof ReadableStreamBYOBRequest; + ReadableStreamDefaultController: typeof ReadableStreamDefaultController; + ReadableByteStreamController: typeof ReadableByteStreamController; + WritableStreamDefaultController: typeof WritableStreamDefaultController; + TransformStreamDefaultController: typeof TransformStreamDefaultController; + CompressionStream: typeof CompressionStream; + DecompressionStream: typeof DecompressionStream; + TextEncoderStream: typeof TextEncoderStream; + TextDecoderStream: typeof TextDecoderStream; + Headers: typeof Headers; + Body: typeof Body; + Request: typeof Request; + Response: typeof Response; + WebSocket: typeof WebSocket; + WebSocketPair: typeof WebSocketPair; + WebSocketRequestResponsePair: typeof WebSocketRequestResponsePair; + AbortController: typeof AbortController; + AbortSignal: typeof AbortSignal; + TextDecoder: typeof TextDecoder; + TextEncoder: typeof TextEncoder; + navigator: Navigator; + Navigator: typeof Navigator; + URL: typeof URL; + URLSearchParams: typeof URLSearchParams; + URLPattern: typeof URLPattern; + Blob: typeof Blob; + File: typeof File; + FormData: typeof FormData; + Crypto: typeof Crypto; + SubtleCrypto: typeof SubtleCrypto; + CryptoKey: typeof CryptoKey; + CacheStorage: typeof CacheStorage; + Cache: typeof Cache; + FixedLengthStream: typeof FixedLengthStream; + IdentityTransformStream: typeof IdentityTransformStream; + HTMLRewriter: typeof HTMLRewriter; +} +declare function addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; +declare function removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; +/** + * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) + */ +declare function dispatchEvent(event: WorkerGlobalScopeEventMap[keyof WorkerGlobalScopeEventMap]): boolean; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/btoa) */ +declare function btoa(data: string): string; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/atob) */ +declare function atob(data: string): string; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ +declare function setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ +declare function setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearTimeout) */ +declare function clearTimeout(timeoutId: number | null): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ +declare function setInterval(callback: (...args: any[]) => void, msDelay?: number): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ +declare function setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearInterval) */ +declare function clearInterval(timeoutId: number | null): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/queueMicrotask) */ +declare function queueMicrotask(task: Function): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/structuredClone) */ +declare function structuredClone(value: T, options?: StructuredSerializeOptions): T; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/reportError) */ +declare function reportError(error: any): void; +/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/fetch) */ +declare function fetch(input: RequestInfo | URL, init?: RequestInit): Promise; +declare const self: ServiceWorkerGlobalScope; +/** +* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. +* The Workers runtime implements the full surface of this API, but with some differences in +* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) +* compared to those implemented in most browsers. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) +*/ +declare const crypto: Crypto; +/** +* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) +*/ +declare const caches: CacheStorage; +declare const scheduler: Scheduler; +/** +* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, +* as well as timing of subrequests and other operations. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) +*/ +declare const performance: Performance; +declare const Cloudflare: Cloudflare; +declare const origin: string; +declare const navigator: Navigator; +interface TestController { +} +interface ExecutionContext { + waitUntil(promise: Promise): void; + passThroughOnException(): void; + readonly props: Props; +} +type ExportedHandlerFetchHandler = (request: Request>, env: Env, ctx: ExecutionContext) => Response | Promise; +type ExportedHandlerTailHandler = (events: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTraceHandler = (traces: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTailStreamHandler = (event: TailStream.TailEvent, env: Env, ctx: ExecutionContext) => TailStream.TailEventHandlerType | Promise; +type ExportedHandlerScheduledHandler = (controller: ScheduledController, env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerQueueHandler = (batch: MessageBatch, env: Env, ctx: ExecutionContext) => void | Promise; +type ExportedHandlerTestHandler = (controller: TestController, env: Env, ctx: ExecutionContext) => void | Promise; +interface ExportedHandler { + fetch?: ExportedHandlerFetchHandler; + tail?: ExportedHandlerTailHandler; + trace?: ExportedHandlerTraceHandler; + tailStream?: ExportedHandlerTailStreamHandler; + scheduled?: ExportedHandlerScheduledHandler; + test?: ExportedHandlerTestHandler; + email?: EmailExportedHandler; + queue?: ExportedHandlerQueueHandler; +} +interface StructuredSerializeOptions { + transfer?: any[]; +} +declare abstract class Navigator { + sendBeacon(url: string, body?: BodyInit): boolean; + readonly userAgent: string; + readonly hardwareConcurrency: number; + readonly language: string; + readonly languages: string[]; +} +interface AlarmInvocationInfo { + readonly isRetry: boolean; + readonly retryCount: number; +} +interface Cloudflare { + readonly compatibilityFlags: Record; +} +interface DurableObject { + fetch(request: Request): Response | Promise; + alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; + webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; + webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; + webSocketError?(ws: WebSocket, error: unknown): void | Promise; +} +type DurableObjectStub = Fetcher & { + readonly id: DurableObjectId; + readonly name?: string; +}; +interface DurableObjectId { + toString(): string; + equals(other: DurableObjectId): boolean; + readonly name?: string; +} +declare abstract class DurableObjectNamespace { + newUniqueId(options?: DurableObjectNamespaceNewUniqueIdOptions): DurableObjectId; + idFromName(name: string): DurableObjectId; + idFromString(id: string): DurableObjectId; + get(id: DurableObjectId, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; + getByName(name: string, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; + jurisdiction(jurisdiction: DurableObjectJurisdiction): DurableObjectNamespace; +} +type DurableObjectJurisdiction = "eu" | "fedramp" | "fedramp-high"; +interface DurableObjectNamespaceNewUniqueIdOptions { + jurisdiction?: DurableObjectJurisdiction; +} +type DurableObjectLocationHint = "wnam" | "enam" | "sam" | "weur" | "eeur" | "apac" | "oc" | "afr" | "me"; +interface DurableObjectNamespaceGetDurableObjectOptions { + locationHint?: DurableObjectLocationHint; +} +interface DurableObjectClass<_T extends Rpc.DurableObjectBranded | undefined = undefined> { +} +interface DurableObjectState { + waitUntil(promise: Promise): void; + readonly props: Props; + readonly id: DurableObjectId; + readonly storage: DurableObjectStorage; + container?: Container; + blockConcurrencyWhile(callback: () => Promise): Promise; + acceptWebSocket(ws: WebSocket, tags?: string[]): void; + getWebSockets(tag?: string): WebSocket[]; + setWebSocketAutoResponse(maybeReqResp?: WebSocketRequestResponsePair): void; + getWebSocketAutoResponse(): WebSocketRequestResponsePair | null; + getWebSocketAutoResponseTimestamp(ws: WebSocket): Date | null; + setHibernatableWebSocketEventTimeout(timeoutMs?: number): void; + getHibernatableWebSocketEventTimeout(): number | null; + getTags(ws: WebSocket): string[]; + abort(reason?: string): void; +} +interface DurableObjectTransaction { + get(key: string, options?: DurableObjectGetOptions): Promise; + get(keys: string[], options?: DurableObjectGetOptions): Promise>; + list(options?: DurableObjectListOptions): Promise>; + put(key: string, value: T, options?: DurableObjectPutOptions): Promise; + put(entries: Record, options?: DurableObjectPutOptions): Promise; + delete(key: string, options?: DurableObjectPutOptions): Promise; + delete(keys: string[], options?: DurableObjectPutOptions): Promise; + rollback(): void; + getAlarm(options?: DurableObjectGetAlarmOptions): Promise; + setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; + deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; +} +interface DurableObjectStorage { + get(key: string, options?: DurableObjectGetOptions): Promise; + get(keys: string[], options?: DurableObjectGetOptions): Promise>; + list(options?: DurableObjectListOptions): Promise>; + put(key: string, value: T, options?: DurableObjectPutOptions): Promise; + put(entries: Record, options?: DurableObjectPutOptions): Promise; + delete(key: string, options?: DurableObjectPutOptions): Promise; + delete(keys: string[], options?: DurableObjectPutOptions): Promise; + deleteAll(options?: DurableObjectPutOptions): Promise; + transaction(closure: (txn: DurableObjectTransaction) => Promise): Promise; + getAlarm(options?: DurableObjectGetAlarmOptions): Promise; + setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; + deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; + sync(): Promise; + sql: SqlStorage; + kv: SyncKvStorage; + transactionSync(closure: () => T): T; + getCurrentBookmark(): Promise; + getBookmarkForTime(timestamp: number | Date): Promise; + onNextSessionRestoreBookmark(bookmark: string): Promise; +} +interface DurableObjectListOptions { + start?: string; + startAfter?: string; + end?: string; + prefix?: string; + reverse?: boolean; + limit?: number; + allowConcurrency?: boolean; + noCache?: boolean; +} +interface DurableObjectGetOptions { + allowConcurrency?: boolean; + noCache?: boolean; +} +interface DurableObjectGetAlarmOptions { + allowConcurrency?: boolean; +} +interface DurableObjectPutOptions { + allowConcurrency?: boolean; + allowUnconfirmed?: boolean; + noCache?: boolean; +} +interface DurableObjectSetAlarmOptions { + allowConcurrency?: boolean; + allowUnconfirmed?: boolean; +} +declare class WebSocketRequestResponsePair { + constructor(request: string, response: string); + get request(): string; + get response(): string; +} +interface AnalyticsEngineDataset { + writeDataPoint(event?: AnalyticsEngineDataPoint): void; +} +interface AnalyticsEngineDataPoint { + indexes?: ((ArrayBuffer | string) | null)[]; + doubles?: number[]; + blobs?: ((ArrayBuffer | string) | null)[]; +} +/** + * The **`Event`** interface represents an event which takes place on an `EventTarget`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event) + */ +declare class Event { + constructor(type: string, init?: EventInit); + /** + * The **`type`** read-only property of the Event interface returns a string containing the event's type. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/type) + */ + get type(): string; + /** + * The **`eventPhase`** read-only property of the being evaluated. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/eventPhase) + */ + get eventPhase(): number; + /** + * The read-only **`composed`** property of the or not the event will propagate across the shadow DOM boundary into the standard DOM. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composed) + */ + get composed(): boolean; + /** + * The **`bubbles`** read-only property of the Event interface indicates whether the event bubbles up through the DOM tree or not. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/bubbles) + */ + get bubbles(): boolean; + /** + * The **`cancelable`** read-only property of the Event interface indicates whether the event can be canceled, and therefore prevented as if the event never happened. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelable) + */ + get cancelable(): boolean; + /** + * The **`defaultPrevented`** read-only property of the Event interface returns a boolean value indicating whether or not the call to Event.preventDefault() canceled the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/defaultPrevented) + */ + get defaultPrevented(): boolean; + /** + * The Event property **`returnValue`** indicates whether the default action for this event has been prevented or not. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/returnValue) + */ + get returnValue(): boolean; + /** + * The **`currentTarget`** read-only property of the Event interface identifies the element to which the event handler has been attached. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/currentTarget) + */ + get currentTarget(): EventTarget | undefined; + /** + * The read-only **`target`** property of the dispatched. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/target) + */ + get target(): EventTarget | undefined; + /** + * The deprecated **`Event.srcElement`** is an alias for the Event.target property. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/srcElement) + */ + get srcElement(): EventTarget | undefined; + /** + * The **`timeStamp`** read-only property of the Event interface returns the time (in milliseconds) at which the event was created. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/timeStamp) + */ + get timeStamp(): number; + /** + * The **`isTrusted`** read-only property of the when the event was generated by the user agent (including via user actions and programmatic methods such as HTMLElement.focus()), and `false` when the event was dispatched via The only exception is the `click` event, which initializes the `isTrusted` property to `false` in user agents. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/isTrusted) + */ + get isTrusted(): boolean; + /** + * The **`cancelBubble`** property of the Event interface is deprecated. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) + */ + get cancelBubble(): boolean; + /** + * The **`cancelBubble`** property of the Event interface is deprecated. + * @deprecated + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) + */ + set cancelBubble(value: boolean); + /** + * The **`stopImmediatePropagation()`** method of the If several listeners are attached to the same element for the same event type, they are called in the order in which they were added. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopImmediatePropagation) + */ + stopImmediatePropagation(): void; + /** + * The **`preventDefault()`** method of the Event interface tells the user agent that if the event does not get explicitly handled, its default action should not be taken as it normally would be. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/preventDefault) + */ + preventDefault(): void; + /** + * The **`stopPropagation()`** method of the Event interface prevents further propagation of the current event in the capturing and bubbling phases. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopPropagation) + */ + stopPropagation(): void; + /** + * The **`composedPath()`** method of the Event interface returns the event's path which is an array of the objects on which listeners will be invoked. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composedPath) + */ + composedPath(): EventTarget[]; + static readonly NONE: number; + static readonly CAPTURING_PHASE: number; + static readonly AT_TARGET: number; + static readonly BUBBLING_PHASE: number; +} +interface EventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; +} +type EventListener = (event: EventType) => void; +interface EventListenerObject { + handleEvent(event: EventType): void; +} +type EventListenerOrEventListenerObject = EventListener | EventListenerObject; +/** + * The **`EventTarget`** interface is implemented by objects that can receive events and may have listeners for them. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget) + */ +declare class EventTarget = Record> { + constructor(); + /** + * The **`addEventListener()`** method of the EventTarget interface sets up a function that will be called whenever the specified event is delivered to the target. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/addEventListener) + */ + addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; + /** + * The **`removeEventListener()`** method of the EventTarget interface removes an event listener previously registered with EventTarget.addEventListener() from the target. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/removeEventListener) + */ + removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; + /** + * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) + */ + dispatchEvent(event: EventMap[keyof EventMap]): boolean; +} +interface EventTargetEventListenerOptions { + capture?: boolean; +} +interface EventTargetAddEventListenerOptions { + capture?: boolean; + passive?: boolean; + once?: boolean; + signal?: AbortSignal; +} +interface EventTargetHandlerObject { + handleEvent: (event: Event) => any | undefined; +} +/** + * The **`AbortController`** interface represents a controller object that allows you to abort one or more Web requests as and when desired. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController) + */ +declare class AbortController { + constructor(); + /** + * The **`signal`** read-only property of the AbortController interface returns an AbortSignal object instance, which can be used to communicate with/abort an asynchronous operation as desired. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/signal) + */ + get signal(): AbortSignal; + /** + * The **`abort()`** method of the AbortController interface aborts an asynchronous operation before it has completed. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/abort) + */ + abort(reason?: any): void; +} +/** + * The **`AbortSignal`** interface represents a signal object that allows you to communicate with an asynchronous operation (such as a fetch request) and abort it if required via an AbortController object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal) + */ +declare abstract class AbortSignal extends EventTarget { + /** + * The **`AbortSignal.abort()`** static method returns an AbortSignal that is already set as aborted (and which does not trigger an AbortSignal/abort_event event). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_static) + */ + static abort(reason?: any): AbortSignal; + /** + * The **`AbortSignal.timeout()`** static method returns an AbortSignal that will automatically abort after a specified time. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/timeout_static) + */ + static timeout(delay: number): AbortSignal; + /** + * The **`AbortSignal.any()`** static method takes an iterable of abort signals and returns an AbortSignal. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/any_static) + */ + static any(signals: AbortSignal[]): AbortSignal; + /** + * The **`aborted`** read-only property returns a value that indicates whether the asynchronous operations the signal is communicating with are aborted (`true`) or not (`false`). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/aborted) + */ + get aborted(): boolean; + /** + * The **`reason`** read-only property returns a JavaScript value that indicates the abort reason. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/reason) + */ + get reason(): any; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ + get onabort(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ + set onabort(value: any | null); + /** + * The **`throwIfAborted()`** method throws the signal's abort AbortSignal.reason if the signal has been aborted; otherwise it does nothing. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/throwIfAborted) + */ + throwIfAborted(): void; +} +interface Scheduler { + wait(delay: number, maybeOptions?: SchedulerWaitOptions): Promise; +} +interface SchedulerWaitOptions { + signal?: AbortSignal; +} +/** + * The **`ExtendableEvent`** interface extends the lifetime of the `install` and `activate` events dispatched on the global scope as part of the service worker lifecycle. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent) + */ +declare abstract class ExtendableEvent extends Event { + /** + * The **`ExtendableEvent.waitUntil()`** method tells the event dispatcher that work is ongoing. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent/waitUntil) + */ + waitUntil(promise: Promise): void; +} +/** + * The **`CustomEvent`** interface represents events initialized by an application for any purpose. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent) + */ +declare class CustomEvent extends Event { + constructor(type: string, init?: CustomEventCustomEventInit); + /** + * The read-only **`detail`** property of the CustomEvent interface returns any data passed when initializing the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent/detail) + */ + get detail(): T; +} +interface CustomEventCustomEventInit { + bubbles?: boolean; + cancelable?: boolean; + composed?: boolean; + detail?: any; +} +/** + * The **`Blob`** interface represents a blob, which is a file-like object of immutable, raw data; they can be read as text or binary data, or converted into a ReadableStream so its methods can be used for processing the data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob) + */ +declare class Blob { + constructor(type?: ((ArrayBuffer | ArrayBufferView) | string | Blob)[], options?: BlobOptions); + /** + * The **`size`** read-only property of the Blob interface returns the size of the Blob or File in bytes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) + */ + get size(): number; + /** + * The **`type`** read-only property of the Blob interface returns the MIME type of the file. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) + */ + get type(): string; + /** + * The **`slice()`** method of the Blob interface creates and returns a new `Blob` object which contains data from a subset of the blob on which it's called. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) + */ + slice(start?: number, end?: number, type?: string): Blob; + /** + * The **`arrayBuffer()`** method of the Blob interface returns a Promise that resolves with the contents of the blob as binary data contained in an ArrayBuffer. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/arrayBuffer) + */ + arrayBuffer(): Promise; + /** + * The **`bytes()`** method of the Blob interface returns a Promise that resolves with a Uint8Array containing the contents of the blob as an array of bytes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/bytes) + */ + bytes(): Promise; + /** + * The **`text()`** method of the string containing the contents of the blob, interpreted as UTF-8. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) + */ + text(): Promise; + /** + * The **`stream()`** method of the Blob interface returns a ReadableStream which upon reading returns the data contained within the `Blob`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/stream) + */ + stream(): ReadableStream; +} +interface BlobOptions { + type?: string; +} +/** + * The **`File`** interface provides information about files and allows JavaScript in a web page to access their content. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File) + */ +declare class File extends Blob { + constructor(bits: ((ArrayBuffer | ArrayBufferView) | string | Blob)[] | undefined, name: string, options?: FileOptions); + /** + * The **`name`** read-only property of the File interface returns the name of the file represented by a File object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) + */ + get name(): string; + /** + * The **`lastModified`** read-only property of the File interface provides the last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) + */ + get lastModified(): number; +} +interface FileOptions { + type?: string; + lastModified?: number; +} +/** +* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) +*/ +declare abstract class CacheStorage { + /** + * The **`open()`** method of the the Cache object matching the `cacheName`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CacheStorage/open) + */ + open(cacheName: string): Promise; + readonly default: Cache; +} +/** +* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) +*/ +declare abstract class Cache { + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#delete) */ + delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#match) */ + match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#put) */ + put(request: RequestInfo | URL, response: Response): Promise; +} +interface CacheQueryOptions { + ignoreMethod?: boolean; +} +/** +* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. +* The Workers runtime implements the full surface of this API, but with some differences in +* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) +* compared to those implemented in most browsers. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) +*/ +declare abstract class Crypto { + /** + * The **`Crypto.subtle`** read-only property returns a cryptographic operations. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/subtle) + */ + get subtle(): SubtleCrypto; + /** + * The **`Crypto.getRandomValues()`** method lets you get cryptographically strong random values. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) + */ + getRandomValues(buffer: T): T; + /** + * The **`randomUUID()`** method of the Crypto interface is used to generate a v4 UUID using a cryptographically secure random number generator. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID) + */ + randomUUID(): string; + DigestStream: typeof DigestStream; +} +/** + * The **`SubtleCrypto`** interface of the Web Crypto API provides a number of low-level cryptographic functions. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto) + */ +declare abstract class SubtleCrypto { + /** + * The **`encrypt()`** method of the SubtleCrypto interface encrypts data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/encrypt) + */ + encrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, plainText: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`decrypt()`** method of the SubtleCrypto interface decrypts some encrypted data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/decrypt) + */ + decrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, cipherText: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`sign()`** method of the SubtleCrypto interface generates a digital signature. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/sign) + */ + sign(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, data: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`verify()`** method of the SubtleCrypto interface verifies a digital signature. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/verify) + */ + verify(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, signature: ArrayBuffer | ArrayBufferView, data: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`digest()`** method of the SubtleCrypto interface generates a _digest_ of the given data, using the specified hash function. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/digest) + */ + digest(algorithm: string | SubtleCryptoHashAlgorithm, data: ArrayBuffer | ArrayBufferView): Promise; + /** + * The **`generateKey()`** method of the SubtleCrypto interface is used to generate a new key (for symmetric algorithms) or key pair (for public-key algorithms). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/generateKey) + */ + generateKey(algorithm: string | SubtleCryptoGenerateKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + /** + * The **`deriveKey()`** method of the SubtleCrypto interface can be used to derive a secret key from a master key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveKey) + */ + deriveKey(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, derivedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + /** + * The **`deriveBits()`** method of the key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveBits) + */ + deriveBits(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, length?: number | null): Promise; + /** + * The **`importKey()`** method of the SubtleCrypto interface imports a key: that is, it takes as input a key in an external, portable format and gives you a CryptoKey object that you can use in the Web Crypto API. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/importKey) + */ + importKey(format: string, keyData: (ArrayBuffer | ArrayBufferView) | JsonWebKey, algorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + /** + * The **`exportKey()`** method of the SubtleCrypto interface exports a key: that is, it takes as input a CryptoKey object and gives you the key in an external, portable format. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/exportKey) + */ + exportKey(format: string, key: CryptoKey): Promise; + /** + * The **`wrapKey()`** method of the SubtleCrypto interface 'wraps' a key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/wrapKey) + */ + wrapKey(format: string, key: CryptoKey, wrappingKey: CryptoKey, wrapAlgorithm: string | SubtleCryptoEncryptAlgorithm): Promise; + /** + * The **`unwrapKey()`** method of the SubtleCrypto interface 'unwraps' a key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/unwrapKey) + */ + unwrapKey(format: string, wrappedKey: ArrayBuffer | ArrayBufferView, unwrappingKey: CryptoKey, unwrapAlgorithm: string | SubtleCryptoEncryptAlgorithm, unwrappedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; + timingSafeEqual(a: ArrayBuffer | ArrayBufferView, b: ArrayBuffer | ArrayBufferView): boolean; +} +/** + * The **`CryptoKey`** interface of the Web Crypto API represents a cryptographic key obtained from one of the SubtleCrypto methods SubtleCrypto.generateKey, SubtleCrypto.deriveKey, SubtleCrypto.importKey, or SubtleCrypto.unwrapKey. + * Available only in secure contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey) + */ +declare abstract class CryptoKey { + /** + * The read-only **`type`** property of the CryptoKey interface indicates which kind of key is represented by the object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/type) + */ + readonly type: string; + /** + * The read-only **`extractable`** property of the CryptoKey interface indicates whether or not the key may be extracted using `SubtleCrypto.exportKey()` or `SubtleCrypto.wrapKey()`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/extractable) + */ + readonly extractable: boolean; + /** + * The read-only **`algorithm`** property of the CryptoKey interface returns an object describing the algorithm for which this key can be used, and any associated extra parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/algorithm) + */ + readonly algorithm: CryptoKeyKeyAlgorithm | CryptoKeyAesKeyAlgorithm | CryptoKeyHmacKeyAlgorithm | CryptoKeyRsaKeyAlgorithm | CryptoKeyEllipticKeyAlgorithm | CryptoKeyArbitraryKeyAlgorithm; + /** + * The read-only **`usages`** property of the CryptoKey interface indicates what can be done with the key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/usages) + */ + readonly usages: string[]; +} +interface CryptoKeyPair { + publicKey: CryptoKey; + privateKey: CryptoKey; +} +interface JsonWebKey { + kty: string; + use?: string; + key_ops?: string[]; + alg?: string; + ext?: boolean; + crv?: string; + x?: string; + y?: string; + d?: string; + n?: string; + e?: string; + p?: string; + q?: string; + dp?: string; + dq?: string; + qi?: string; + oth?: RsaOtherPrimesInfo[]; + k?: string; +} +interface RsaOtherPrimesInfo { + r?: string; + d?: string; + t?: string; +} +interface SubtleCryptoDeriveKeyAlgorithm { + name: string; + salt?: (ArrayBuffer | ArrayBufferView); + iterations?: number; + hash?: (string | SubtleCryptoHashAlgorithm); + $public?: CryptoKey; + info?: (ArrayBuffer | ArrayBufferView); +} +interface SubtleCryptoEncryptAlgorithm { + name: string; + iv?: (ArrayBuffer | ArrayBufferView); + additionalData?: (ArrayBuffer | ArrayBufferView); + tagLength?: number; + counter?: (ArrayBuffer | ArrayBufferView); + length?: number; + label?: (ArrayBuffer | ArrayBufferView); +} +interface SubtleCryptoGenerateKeyAlgorithm { + name: string; + hash?: (string | SubtleCryptoHashAlgorithm); + modulusLength?: number; + publicExponent?: (ArrayBuffer | ArrayBufferView); + length?: number; + namedCurve?: string; +} +interface SubtleCryptoHashAlgorithm { + name: string; +} +interface SubtleCryptoImportKeyAlgorithm { + name: string; + hash?: (string | SubtleCryptoHashAlgorithm); + length?: number; + namedCurve?: string; + compressed?: boolean; +} +interface SubtleCryptoSignAlgorithm { + name: string; + hash?: (string | SubtleCryptoHashAlgorithm); + dataLength?: number; + saltLength?: number; +} +interface CryptoKeyKeyAlgorithm { + name: string; +} +interface CryptoKeyAesKeyAlgorithm { + name: string; + length: number; +} +interface CryptoKeyHmacKeyAlgorithm { + name: string; + hash: CryptoKeyKeyAlgorithm; + length: number; +} +interface CryptoKeyRsaKeyAlgorithm { + name: string; + modulusLength: number; + publicExponent: ArrayBuffer | ArrayBufferView; + hash?: CryptoKeyKeyAlgorithm; +} +interface CryptoKeyEllipticKeyAlgorithm { + name: string; + namedCurve: string; +} +interface CryptoKeyArbitraryKeyAlgorithm { + name: string; + hash?: CryptoKeyKeyAlgorithm; + namedCurve?: string; + length?: number; +} +declare class DigestStream extends WritableStream { + constructor(algorithm: string | SubtleCryptoHashAlgorithm); + readonly digest: Promise; + get bytesWritten(): number | bigint; +} +/** + * The **`TextDecoder`** interface represents a decoder for a specific text encoding, such as `UTF-8`, `ISO-8859-2`, `KOI8-R`, `GBK`, etc. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder) + */ +declare class TextDecoder { + constructor(label?: string, options?: TextDecoderConstructorOptions); + /** + * The **`TextDecoder.decode()`** method returns a string containing text decoded from the buffer passed as a parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder/decode) + */ + decode(input?: (ArrayBuffer | ArrayBufferView), options?: TextDecoderDecodeOptions): string; + get encoding(): string; + get fatal(): boolean; + get ignoreBOM(): boolean; +} +/** + * The **`TextEncoder`** interface takes a stream of code points as input and emits a stream of UTF-8 bytes. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder) + */ +declare class TextEncoder { + constructor(); + /** + * The **`TextEncoder.encode()`** method takes a string as input, and returns a Global_Objects/Uint8Array containing the text given in parameters encoded with the specific method for that TextEncoder object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encode) + */ + encode(input?: string): Uint8Array; + /** + * The **`TextEncoder.encodeInto()`** method takes a string to encode and a destination Uint8Array to put resulting UTF-8 encoded text into, and returns a dictionary object indicating the progress of the encoding. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encodeInto) + */ + encodeInto(input: string, buffer: Uint8Array): TextEncoderEncodeIntoResult; + get encoding(): string; +} +interface TextDecoderConstructorOptions { + fatal: boolean; + ignoreBOM: boolean; +} +interface TextDecoderDecodeOptions { + stream: boolean; +} +interface TextEncoderEncodeIntoResult { + read: number; + written: number; +} +/** + * The **`ErrorEvent`** interface represents events providing information related to errors in scripts or in files. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent) + */ +declare class ErrorEvent extends Event { + constructor(type: string, init?: ErrorEventErrorEventInit); + /** + * The **`filename`** read-only property of the ErrorEvent interface returns a string containing the name of the script file in which the error occurred. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/filename) + */ + get filename(): string; + /** + * The **`message`** read-only property of the ErrorEvent interface returns a string containing a human-readable error message describing the problem. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/message) + */ + get message(): string; + /** + * The **`lineno`** read-only property of the ErrorEvent interface returns an integer containing the line number of the script file on which the error occurred. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/lineno) + */ + get lineno(): number; + /** + * The **`colno`** read-only property of the ErrorEvent interface returns an integer containing the column number of the script file on which the error occurred. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/colno) + */ + get colno(): number; + /** + * The **`error`** read-only property of the ErrorEvent interface returns a JavaScript value, such as an Error or DOMException, representing the error associated with this event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/error) + */ + get error(): any; +} +interface ErrorEventErrorEventInit { + message?: string; + filename?: string; + lineno?: number; + colno?: number; + error?: any; +} +/** + * The **`MessageEvent`** interface represents a message received by a target object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent) + */ +declare class MessageEvent extends Event { + constructor(type: string, initializer: MessageEventInit); + /** + * The **`data`** read-only property of the The data sent by the message emitter; this can be any data type, depending on what originated this event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/data) + */ + readonly data: any; + /** + * The **`origin`** read-only property of the origin of the message emitter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/origin) + */ + readonly origin: string | null; + /** + * The **`lastEventId`** read-only property of the unique ID for the event. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/lastEventId) + */ + readonly lastEventId: string; + /** + * The **`source`** read-only property of the a WindowProxy, MessagePort, or a `MessageEventSource` (which can be a WindowProxy, message emitter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/source) + */ + readonly source: MessagePort | null; + /** + * The **`ports`** read-only property of the containing all MessagePort objects sent with the message, in order. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/ports) + */ + readonly ports: MessagePort[]; +} +interface MessageEventInit { + data: ArrayBuffer | string; +} +/** + * The **`PromiseRejectionEvent`** interface represents events which are sent to the global script context when JavaScript Promises are rejected. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent) + */ +declare abstract class PromiseRejectionEvent extends Event { + /** + * The PromiseRejectionEvent interface's **`promise`** read-only property indicates the JavaScript rejected. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/promise) + */ + readonly promise: Promise; + /** + * The PromiseRejectionEvent **`reason`** read-only property is any JavaScript value or Object which provides the reason passed into Promise.reject(). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/reason) + */ + readonly reason: any; +} +/** + * The **`FormData`** interface provides a way to construct a set of key/value pairs representing form fields and their values, which can be sent using the Window/fetch, XMLHttpRequest.send() or navigator.sendBeacon() methods. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData) + */ +declare class FormData { + constructor(); + /** + * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) + */ + append(name: string, value: string): void; + /** + * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) + */ + append(name: string, value: Blob, filename?: string): void; + /** + * The **`delete()`** method of the FormData interface deletes a key and its value(s) from a `FormData` object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/delete) + */ + delete(name: string): void; + /** + * The **`get()`** method of the FormData interface returns the first value associated with a given key from within a `FormData` object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/get) + */ + get(name: string): (File | string) | null; + /** + * The **`getAll()`** method of the FormData interface returns all the values associated with a given key from within a `FormData` object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/getAll) + */ + getAll(name: string): (File | string)[]; + /** + * The **`has()`** method of the FormData interface returns whether a `FormData` object contains a certain key. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/has) + */ + has(name: string): boolean; + /** + * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) + */ + set(name: string, value: string): void; + /** + * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) + */ + set(name: string, value: Blob, filename?: string): void; + /* Returns an array of key, value pairs for every entry in the list. */ + entries(): IterableIterator<[ + key: string, + value: File | string + ]>; + /* Returns a list of keys in the list. */ + keys(): IterableIterator; + /* Returns a list of values in the list. */ + values(): IterableIterator<(File | string)>; + forEach(callback: (this: This, value: File | string, key: string, parent: FormData) => void, thisArg?: This): void; + [Symbol.iterator](): IterableIterator<[ + key: string, + value: File | string + ]>; +} +interface ContentOptions { + html?: boolean; +} +declare class HTMLRewriter { + constructor(); + on(selector: string, handlers: HTMLRewriterElementContentHandlers): HTMLRewriter; + onDocument(handlers: HTMLRewriterDocumentContentHandlers): HTMLRewriter; + transform(response: Response): Response; +} +interface HTMLRewriterElementContentHandlers { + element?(element: Element): void | Promise; + comments?(comment: Comment): void | Promise; + text?(element: Text): void | Promise; +} +interface HTMLRewriterDocumentContentHandlers { + doctype?(doctype: Doctype): void | Promise; + comments?(comment: Comment): void | Promise; + text?(text: Text): void | Promise; + end?(end: DocumentEnd): void | Promise; +} +interface Doctype { + readonly name: string | null; + readonly publicId: string | null; + readonly systemId: string | null; +} +interface Element { + tagName: string; + readonly attributes: IterableIterator; + readonly removed: boolean; + readonly namespaceURI: string; + getAttribute(name: string): string | null; + hasAttribute(name: string): boolean; + setAttribute(name: string, value: string): Element; + removeAttribute(name: string): Element; + before(content: string | ReadableStream | Response, options?: ContentOptions): Element; + after(content: string | ReadableStream | Response, options?: ContentOptions): Element; + prepend(content: string | ReadableStream | Response, options?: ContentOptions): Element; + append(content: string | ReadableStream | Response, options?: ContentOptions): Element; + replace(content: string | ReadableStream | Response, options?: ContentOptions): Element; + remove(): Element; + removeAndKeepContent(): Element; + setInnerContent(content: string | ReadableStream | Response, options?: ContentOptions): Element; + onEndTag(handler: (tag: EndTag) => void | Promise): void; +} +interface EndTag { + name: string; + before(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; + after(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; + remove(): EndTag; +} +interface Comment { + text: string; + readonly removed: boolean; + before(content: string, options?: ContentOptions): Comment; + after(content: string, options?: ContentOptions): Comment; + replace(content: string, options?: ContentOptions): Comment; + remove(): Comment; +} +interface Text { + readonly text: string; + readonly lastInTextNode: boolean; + readonly removed: boolean; + before(content: string | ReadableStream | Response, options?: ContentOptions): Text; + after(content: string | ReadableStream | Response, options?: ContentOptions): Text; + replace(content: string | ReadableStream | Response, options?: ContentOptions): Text; + remove(): Text; +} +interface DocumentEnd { + append(content: string, options?: ContentOptions): DocumentEnd; +} +/** + * This is the event type for `fetch` events dispatched on the ServiceWorkerGlobalScope. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent) + */ +declare abstract class FetchEvent extends ExtendableEvent { + /** + * The **`request`** read-only property of the the event handler. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/request) + */ + readonly request: Request; + /** + * The **`respondWith()`** method of allows you to provide a promise for a Response yourself. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/respondWith) + */ + respondWith(promise: Response | Promise): void; + passThroughOnException(): void; +} +type HeadersInit = Headers | Iterable> | Record; +/** + * The **`Headers`** interface of the Fetch API allows you to perform various actions on HTTP request and response headers. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers) + */ +declare class Headers { + constructor(init?: HeadersInit); + /** + * The **`get()`** method of the Headers interface returns a byte string of all the values of a header within a `Headers` object with a given name. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/get) + */ + get(name: string): string | null; + getAll(name: string): string[]; + /** + * The **`getSetCookie()`** method of the Headers interface returns an array containing the values of all Set-Cookie headers associated with a response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/getSetCookie) + */ + getSetCookie(): string[]; + /** + * The **`has()`** method of the Headers interface returns a boolean stating whether a `Headers` object contains a certain header. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/has) + */ + has(name: string): boolean; + /** + * The **`set()`** method of the Headers interface sets a new value for an existing header inside a `Headers` object, or adds the header if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/set) + */ + set(name: string, value: string): void; + /** + * The **`append()`** method of the Headers interface appends a new value onto an existing header inside a `Headers` object, or adds the header if it does not already exist. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/append) + */ + append(name: string, value: string): void; + /** + * The **`delete()`** method of the Headers interface deletes a header from the current `Headers` object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/delete) + */ + delete(name: string): void; + forEach(callback: (this: This, value: string, key: string, parent: Headers) => void, thisArg?: This): void; + /* Returns an iterator allowing to go through all key/value pairs contained in this object. */ + entries(): IterableIterator<[ + key: string, + value: string + ]>; + /* Returns an iterator allowing to go through all keys of the key/value pairs contained in this object. */ + keys(): IterableIterator; + /* Returns an iterator allowing to go through all values of the key/value pairs contained in this object. */ + values(): IterableIterator; + [Symbol.iterator](): IterableIterator<[ + key: string, + value: string + ]>; +} +type BodyInit = ReadableStream | string | ArrayBuffer | ArrayBufferView | Blob | URLSearchParams | FormData; +declare abstract class Body { + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/body) */ + get body(): ReadableStream | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bodyUsed) */ + get bodyUsed(): boolean; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/arrayBuffer) */ + arrayBuffer(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bytes) */ + bytes(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/text) */ + text(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/json) */ + json(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/formData) */ + formData(): Promise; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/blob) */ + blob(): Promise; +} +/** + * The **`Response`** interface of the Fetch API represents the response to a request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) + */ +declare var Response: { + prototype: Response; + new (body?: BodyInit | null, init?: ResponseInit): Response; + error(): Response; + redirect(url: string, status?: number): Response; + json(any: any, maybeInit?: (ResponseInit | Response)): Response; +}; +/** + * The **`Response`** interface of the Fetch API represents the response to a request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) + */ +interface Response extends Body { + /** + * The **`clone()`** method of the Response interface creates a clone of a response object, identical in every way, but stored in a different variable. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/clone) + */ + clone(): Response; + /** + * The **`status`** read-only property of the Response interface contains the HTTP status codes of the response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/status) + */ + status: number; + /** + * The **`statusText`** read-only property of the Response interface contains the status message corresponding to the HTTP status code in Response.status. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/statusText) + */ + statusText: string; + /** + * The **`headers`** read-only property of the with the response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/headers) + */ + headers: Headers; + /** + * The **`ok`** read-only property of the Response interface contains a Boolean stating whether the response was successful (status in the range 200-299) or not. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/ok) + */ + ok: boolean; + /** + * The **`redirected`** read-only property of the Response interface indicates whether or not the response is the result of a request you made which was redirected. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/redirected) + */ + redirected: boolean; + /** + * The **`url`** read-only property of the Response interface contains the URL of the response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/url) + */ + url: string; + webSocket: WebSocket | null; + cf: any | undefined; + /** + * The **`type`** read-only property of the Response interface contains the type of the response. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/type) + */ + type: "default" | "error"; +} +interface ResponseInit { + status?: number; + statusText?: string; + headers?: HeadersInit; + cf?: any; + webSocket?: (WebSocket | null); + encodeBody?: "automatic" | "manual"; +} +type RequestInfo> = Request | string; +/** + * The **`Request`** interface of the Fetch API represents a resource request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) + */ +declare var Request: { + prototype: Request; + new >(input: RequestInfo | URL, init?: RequestInit): Request; +}; +/** + * The **`Request`** interface of the Fetch API represents a resource request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) + */ +interface Request> extends Body { + /** + * The **`clone()`** method of the Request interface creates a copy of the current `Request` object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/clone) + */ + clone(): Request; + /** + * The **`method`** read-only property of the `POST`, etc.) A String indicating the method of the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/method) + */ + method: string; + /** + * The **`url`** read-only property of the Request interface contains the URL of the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/url) + */ + url: string; + /** + * The **`headers`** read-only property of the with the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/headers) + */ + headers: Headers; + /** + * The **`redirect`** read-only property of the Request interface contains the mode for how redirects are handled. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/redirect) + */ + redirect: string; + fetcher: Fetcher | null; + /** + * The read-only **`signal`** property of the Request interface returns the AbortSignal associated with the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/signal) + */ + signal: AbortSignal; + cf: Cf | undefined; + /** + * The **`integrity`** read-only property of the Request interface contains the subresource integrity value of the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/integrity) + */ + integrity: string; + /** + * The **`keepalive`** read-only property of the Request interface contains the request's `keepalive` setting (`true` or `false`), which indicates whether the browser will keep the associated request alive if the page that initiated it is unloaded before the request is complete. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/keepalive) + */ + keepalive: boolean; + /** + * The **`cache`** read-only property of the Request interface contains the cache mode of the request. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/cache) + */ + cache?: "no-store" | "no-cache"; +} +interface RequestInit { + /* A string to set request's method. */ + method?: string; + /* A Headers object, an object literal, or an array of two-item arrays to set request's headers. */ + headers?: HeadersInit; + /* A BodyInit object or null to set request's body. */ + body?: BodyInit | null; + /* A string indicating whether request follows redirects, results in an error upon encountering a redirect, or returns the redirect (in an opaque fashion). Sets request's redirect. */ + redirect?: string; + fetcher?: (Fetcher | null); + cf?: Cf; + /* A string indicating how the request will interact with the browser's cache to set request's cache. */ + cache?: "no-store" | "no-cache"; + /* A cryptographic hash of the resource to be fetched by request. Sets request's integrity. */ + integrity?: string; + /* An AbortSignal to set request's signal. */ + signal?: (AbortSignal | null); + encodeResponseBody?: "automatic" | "manual"; +} +type Service Rpc.WorkerEntrypointBranded) | Rpc.WorkerEntrypointBranded | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? Fetcher> : T extends Rpc.WorkerEntrypointBranded ? Fetcher : T extends Exclude ? never : Fetcher; +type Fetcher = (T extends Rpc.EntrypointBranded ? Rpc.Provider : unknown) & { + fetch(input: RequestInfo | URL, init?: RequestInit): Promise; + connect(address: SocketAddress | string, options?: SocketOptions): Socket; +}; +interface KVNamespaceListKey { + name: Key; + expiration?: number; + metadata?: Metadata; +} +type KVNamespaceListResult = { + list_complete: false; + keys: KVNamespaceListKey[]; + cursor: string; + cacheStatus: string | null; +} | { + list_complete: true; + keys: KVNamespaceListKey[]; + cacheStatus: string | null; +}; +interface KVNamespace { + get(key: Key, options?: Partial>): Promise; + get(key: Key, type: "text"): Promise; + get(key: Key, type: "json"): Promise; + get(key: Key, type: "arrayBuffer"): Promise; + get(key: Key, type: "stream"): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"text">): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"json">): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"arrayBuffer">): Promise; + get(key: Key, options?: KVNamespaceGetOptions<"stream">): Promise; + get(key: Array, type: "text"): Promise>; + get(key: Array, type: "json"): Promise>; + get(key: Array, options?: Partial>): Promise>; + get(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>; + get(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>; + list(options?: KVNamespaceListOptions): Promise>; + put(key: Key, value: string | ArrayBuffer | ArrayBufferView | ReadableStream, options?: KVNamespacePutOptions): Promise; + getWithMetadata(key: Key, options?: Partial>): Promise>; + getWithMetadata(key: Key, type: "text"): Promise>; + getWithMetadata(key: Key, type: "json"): Promise>; + getWithMetadata(key: Key, type: "arrayBuffer"): Promise>; + getWithMetadata(key: Key, type: "stream"): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"text">): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"json">): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"arrayBuffer">): Promise>; + getWithMetadata(key: Key, options: KVNamespaceGetOptions<"stream">): Promise>; + getWithMetadata(key: Array, type: "text"): Promise>>; + getWithMetadata(key: Array, type: "json"): Promise>>; + getWithMetadata(key: Array, options?: Partial>): Promise>>; + getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>>; + getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>>; + delete(key: Key): Promise; +} +interface KVNamespaceListOptions { + limit?: number; + prefix?: (string | null); + cursor?: (string | null); +} +interface KVNamespaceGetOptions { + type: Type; + cacheTtl?: number; +} +interface KVNamespacePutOptions { + expiration?: number; + expirationTtl?: number; + metadata?: (any | null); +} +interface KVNamespaceGetWithMetadataResult { + value: Value | null; + metadata: Metadata | null; + cacheStatus: string | null; +} +type QueueContentType = "text" | "bytes" | "json" | "v8"; +interface Queue { + send(message: Body, options?: QueueSendOptions): Promise; + sendBatch(messages: Iterable>, options?: QueueSendBatchOptions): Promise; +} +interface QueueSendOptions { + contentType?: QueueContentType; + delaySeconds?: number; +} +interface QueueSendBatchOptions { + delaySeconds?: number; +} +interface MessageSendRequest { + body: Body; + contentType?: QueueContentType; + delaySeconds?: number; +} +interface QueueRetryOptions { + delaySeconds?: number; +} +interface Message { + readonly id: string; + readonly timestamp: Date; + readonly body: Body; + readonly attempts: number; + retry(options?: QueueRetryOptions): void; + ack(): void; +} +interface QueueEvent extends ExtendableEvent { + readonly messages: readonly Message[]; + readonly queue: string; + retryAll(options?: QueueRetryOptions): void; + ackAll(): void; +} +interface MessageBatch { + readonly messages: readonly Message[]; + readonly queue: string; + retryAll(options?: QueueRetryOptions): void; + ackAll(): void; +} +interface R2Error extends Error { + readonly name: string; + readonly code: number; + readonly message: string; + readonly action: string; + readonly stack: any; +} +interface R2ListOptions { + limit?: number; + prefix?: string; + cursor?: string; + delimiter?: string; + startAfter?: string; + include?: ("httpMetadata" | "customMetadata")[]; +} +declare abstract class R2Bucket { + head(key: string): Promise; + get(key: string, options: R2GetOptions & { + onlyIf: R2Conditional | Headers; + }): Promise; + get(key: string, options?: R2GetOptions): Promise; + put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions & { + onlyIf: R2Conditional | Headers; + }): Promise; + put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions): Promise; + createMultipartUpload(key: string, options?: R2MultipartOptions): Promise; + resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload; + delete(keys: string | string[]): Promise; + list(options?: R2ListOptions): Promise; +} +interface R2MultipartUpload { + readonly key: string; + readonly uploadId: string; + uploadPart(partNumber: number, value: ReadableStream | (ArrayBuffer | ArrayBufferView) | string | Blob, options?: R2UploadPartOptions): Promise; + abort(): Promise; + complete(uploadedParts: R2UploadedPart[]): Promise; +} +interface R2UploadedPart { + partNumber: number; + etag: string; +} +declare abstract class R2Object { + readonly key: string; + readonly version: string; + readonly size: number; + readonly etag: string; + readonly httpEtag: string; + readonly checksums: R2Checksums; + readonly uploaded: Date; + readonly httpMetadata?: R2HTTPMetadata; + readonly customMetadata?: Record; + readonly range?: R2Range; + readonly storageClass: string; + readonly ssecKeyMd5?: string; + writeHttpMetadata(headers: Headers): void; +} +interface R2ObjectBody extends R2Object { + get body(): ReadableStream; + get bodyUsed(): boolean; + arrayBuffer(): Promise; + bytes(): Promise; + text(): Promise; + json(): Promise; + blob(): Promise; +} +type R2Range = { + offset: number; + length?: number; +} | { + offset?: number; + length: number; +} | { + suffix: number; +}; +interface R2Conditional { + etagMatches?: string; + etagDoesNotMatch?: string; + uploadedBefore?: Date; + uploadedAfter?: Date; + secondsGranularity?: boolean; +} +interface R2GetOptions { + onlyIf?: (R2Conditional | Headers); + range?: (R2Range | Headers); + ssecKey?: (ArrayBuffer | string); +} +interface R2PutOptions { + onlyIf?: (R2Conditional | Headers); + httpMetadata?: (R2HTTPMetadata | Headers); + customMetadata?: Record; + md5?: ((ArrayBuffer | ArrayBufferView) | string); + sha1?: ((ArrayBuffer | ArrayBufferView) | string); + sha256?: ((ArrayBuffer | ArrayBufferView) | string); + sha384?: ((ArrayBuffer | ArrayBufferView) | string); + sha512?: ((ArrayBuffer | ArrayBufferView) | string); + storageClass?: string; + ssecKey?: (ArrayBuffer | string); +} +interface R2MultipartOptions { + httpMetadata?: (R2HTTPMetadata | Headers); + customMetadata?: Record; + storageClass?: string; + ssecKey?: (ArrayBuffer | string); +} +interface R2Checksums { + readonly md5?: ArrayBuffer; + readonly sha1?: ArrayBuffer; + readonly sha256?: ArrayBuffer; + readonly sha384?: ArrayBuffer; + readonly sha512?: ArrayBuffer; + toJSON(): R2StringChecksums; +} +interface R2StringChecksums { + md5?: string; + sha1?: string; + sha256?: string; + sha384?: string; + sha512?: string; +} +interface R2HTTPMetadata { + contentType?: string; + contentLanguage?: string; + contentDisposition?: string; + contentEncoding?: string; + cacheControl?: string; + cacheExpiry?: Date; +} +type R2Objects = { + objects: R2Object[]; + delimitedPrefixes: string[]; +} & ({ + truncated: true; + cursor: string; +} | { + truncated: false; +}); +interface R2UploadPartOptions { + ssecKey?: (ArrayBuffer | string); +} +declare abstract class ScheduledEvent extends ExtendableEvent { + readonly scheduledTime: number; + readonly cron: string; + noRetry(): void; +} +interface ScheduledController { + readonly scheduledTime: number; + readonly cron: string; + noRetry(): void; +} +interface QueuingStrategy { + highWaterMark?: (number | bigint); + size?: (chunk: T) => number | bigint; +} +interface UnderlyingSink { + type?: string; + start?: (controller: WritableStreamDefaultController) => void | Promise; + write?: (chunk: W, controller: WritableStreamDefaultController) => void | Promise; + abort?: (reason: any) => void | Promise; + close?: () => void | Promise; +} +interface UnderlyingByteSource { + type: "bytes"; + autoAllocateChunkSize?: number; + start?: (controller: ReadableByteStreamController) => void | Promise; + pull?: (controller: ReadableByteStreamController) => void | Promise; + cancel?: (reason: any) => void | Promise; +} +interface UnderlyingSource { + type?: "" | undefined; + start?: (controller: ReadableStreamDefaultController) => void | Promise; + pull?: (controller: ReadableStreamDefaultController) => void | Promise; + cancel?: (reason: any) => void | Promise; + expectedLength?: (number | bigint); +} +interface Transformer { + readableType?: string; + writableType?: string; + start?: (controller: TransformStreamDefaultController) => void | Promise; + transform?: (chunk: I, controller: TransformStreamDefaultController) => void | Promise; + flush?: (controller: TransformStreamDefaultController) => void | Promise; + cancel?: (reason: any) => void | Promise; + expectedLength?: number; +} +interface StreamPipeOptions { + /** + * Pipes this readable stream to a given writable stream destination. The way in which the piping process behaves under various error conditions can be customized with a number of passed options. It returns a promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. + * + * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. + * + * Errors and closures of the source and destination streams propagate as follows: + * + * An error in this source readable stream will abort destination, unless preventAbort is truthy. The returned promise will be rejected with the source's error, or with any error that occurs during aborting the destination. + * + * An error in destination will cancel this source readable stream, unless preventCancel is truthy. The returned promise will be rejected with the destination's error, or with any error that occurs during canceling the source. + * + * When this source readable stream closes, destination will be closed, unless preventClose is truthy. The returned promise will be fulfilled once this process completes, unless an error is encountered while closing the destination, in which case it will be rejected with that error. + * + * If destination starts out closed or closing, this source readable stream will be canceled, unless preventCancel is true. The returned promise will be rejected with an error indicating piping to a closed stream failed, or with any error that occurs during canceling the source. + * + * The signal option can be set to an AbortSignal to allow aborting an ongoing pipe operation via the corresponding AbortController. In this case, this source readable stream will be canceled, and destination aborted, unless the respective options preventCancel or preventAbort are set. + */ + preventClose?: boolean; + preventAbort?: boolean; + preventCancel?: boolean; + signal?: AbortSignal; +} +type ReadableStreamReadResult = { + done: false; + value: R; +} | { + done: true; + value?: undefined; +}; +/** + * The `ReadableStream` interface of the Streams API represents a readable stream of byte data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) + */ +interface ReadableStream { + /** + * The **`locked`** read-only property of the ReadableStream interface returns whether or not the readable stream is locked to a reader. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/locked) + */ + get locked(): boolean; + /** + * The **`cancel()`** method of the ReadableStream interface returns a Promise that resolves when the stream is canceled. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/cancel) + */ + cancel(reason?: any): Promise; + /** + * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) + */ + getReader(): ReadableStreamDefaultReader; + /** + * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) + */ + getReader(options: ReadableStreamGetReaderOptions): ReadableStreamBYOBReader; + /** + * The **`pipeThrough()`** method of the ReadableStream interface provides a chainable way of piping the current stream through a transform stream or any other writable/readable pair. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeThrough) + */ + pipeThrough(transform: ReadableWritablePair, options?: StreamPipeOptions): ReadableStream; + /** + * The **`pipeTo()`** method of the ReadableStream interface pipes the current `ReadableStream` to a given WritableStream and returns a Promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeTo) + */ + pipeTo(destination: WritableStream, options?: StreamPipeOptions): Promise; + /** + * The **`tee()`** method of the two-element array containing the two resulting branches as new ReadableStream instances. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/tee) + */ + tee(): [ + ReadableStream, + ReadableStream + ]; + values(options?: ReadableStreamValuesOptions): AsyncIterableIterator; + [Symbol.asyncIterator](options?: ReadableStreamValuesOptions): AsyncIterableIterator; +} +/** + * The `ReadableStream` interface of the Streams API represents a readable stream of byte data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) + */ +declare const ReadableStream: { + prototype: ReadableStream; + new (underlyingSource: UnderlyingByteSource, strategy?: QueuingStrategy): ReadableStream; + new (underlyingSource?: UnderlyingSource, strategy?: QueuingStrategy): ReadableStream; +}; +/** + * The **`ReadableStreamDefaultReader`** interface of the Streams API represents a default reader that can be used to read stream data supplied from a network (such as a fetch request). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader) + */ +declare class ReadableStreamDefaultReader { + constructor(stream: ReadableStream); + get closed(): Promise; + cancel(reason?: any): Promise; + /** + * The **`read()`** method of the ReadableStreamDefaultReader interface returns a Promise providing access to the next chunk in the stream's internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/read) + */ + read(): Promise>; + /** + * The **`releaseLock()`** method of the ReadableStreamDefaultReader interface releases the reader's lock on the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/releaseLock) + */ + releaseLock(): void; +} +/** + * The `ReadableStreamBYOBReader` interface of the Streams API defines a reader for a ReadableStream that supports zero-copy reading from an underlying byte source. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader) + */ +declare class ReadableStreamBYOBReader { + constructor(stream: ReadableStream); + get closed(): Promise; + cancel(reason?: any): Promise; + /** + * The **`read()`** method of the ReadableStreamBYOBReader interface is used to read data into a view on a user-supplied buffer from an associated readable byte stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/read) + */ + read(view: T): Promise>; + /** + * The **`releaseLock()`** method of the ReadableStreamBYOBReader interface releases the reader's lock on the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/releaseLock) + */ + releaseLock(): void; + readAtLeast(minElements: number, view: T): Promise>; +} +interface ReadableStreamBYOBReaderReadableStreamBYOBReaderReadOptions { + min?: number; +} +interface ReadableStreamGetReaderOptions { + /** + * Creates a ReadableStreamBYOBReader and locks the stream to the new reader. + * + * This call behaves the same way as the no-argument variant, except that it only works on readable byte streams, i.e. streams which were constructed specifically with the ability to handle "bring your own buffer" reading. The returned BYOB reader provides the ability to directly read individual chunks from the stream via its read() method, into developer-supplied buffers, allowing more precise control over allocation. + */ + mode: "byob"; +} +/** + * The **`ReadableStreamBYOBRequest`** interface of the Streams API represents a 'pull request' for data from an underlying source that will made as a zero-copy transfer to a consumer (bypassing the stream's internal queues). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest) + */ +declare abstract class ReadableStreamBYOBRequest { + /** + * The **`view`** getter property of the ReadableStreamBYOBRequest interface returns the current view. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/view) + */ + get view(): Uint8Array | null; + /** + * The **`respond()`** method of the ReadableStreamBYOBRequest interface is used to signal to the associated readable byte stream that the specified number of bytes were written into the ReadableStreamBYOBRequest.view. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respond) + */ + respond(bytesWritten: number): void; + /** + * The **`respondWithNewView()`** method of the ReadableStreamBYOBRequest interface specifies a new view that the consumer of the associated readable byte stream should write to instead of ReadableStreamBYOBRequest.view. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respondWithNewView) + */ + respondWithNewView(view: ArrayBuffer | ArrayBufferView): void; + get atLeast(): number | null; +} +/** + * The **`ReadableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a ReadableStream's state and internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController) + */ +declare abstract class ReadableStreamDefaultController { + /** + * The **`desiredSize`** read-only property of the required to fill the stream's internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`close()`** method of the ReadableStreamDefaultController interface closes the associated stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/close) + */ + close(): void; + /** + * The **`enqueue()`** method of the ```js-nolint enqueue(chunk) ``` - `chunk` - : The chunk to enqueue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/enqueue) + */ + enqueue(chunk?: R): void; + /** + * The **`error()`** method of the with the associated stream to error. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/error) + */ + error(reason: any): void; +} +/** + * The **`ReadableByteStreamController`** interface of the Streams API represents a controller for a readable byte stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController) + */ +declare abstract class ReadableByteStreamController { + /** + * The **`byobRequest`** read-only property of the ReadableByteStreamController interface returns the current BYOB request, or `null` if there are no pending requests. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/byobRequest) + */ + get byobRequest(): ReadableStreamBYOBRequest | null; + /** + * The **`desiredSize`** read-only property of the ReadableByteStreamController interface returns the number of bytes required to fill the stream's internal queue to its 'desired size'. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`close()`** method of the ReadableByteStreamController interface closes the associated stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/close) + */ + close(): void; + /** + * The **`enqueue()`** method of the ReadableByteStreamController interface enqueues a given chunk on the associated readable byte stream (the chunk is copied into the stream's internal queues). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/enqueue) + */ + enqueue(chunk: ArrayBuffer | ArrayBufferView): void; + /** + * The **`error()`** method of the ReadableByteStreamController interface causes any future interactions with the associated stream to error with the specified reason. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/error) + */ + error(reason: any): void; +} +/** + * The **`WritableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a WritableStream's state. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController) + */ +declare abstract class WritableStreamDefaultController { + /** + * The read-only **`signal`** property of the WritableStreamDefaultController interface returns the AbortSignal associated with the controller. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/signal) + */ + get signal(): AbortSignal; + /** + * The **`error()`** method of the with the associated stream to error. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/error) + */ + error(reason?: any): void; +} +/** + * The **`TransformStreamDefaultController`** interface of the Streams API provides methods to manipulate the associated ReadableStream and WritableStream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController) + */ +declare abstract class TransformStreamDefaultController { + /** + * The **`desiredSize`** read-only property of the TransformStreamDefaultController interface returns the desired size to fill the queue of the associated ReadableStream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`enqueue()`** method of the TransformStreamDefaultController interface enqueues the given chunk in the readable side of the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/enqueue) + */ + enqueue(chunk?: O): void; + /** + * The **`error()`** method of the TransformStreamDefaultController interface errors both sides of the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/error) + */ + error(reason: any): void; + /** + * The **`terminate()`** method of the TransformStreamDefaultController interface closes the readable side and errors the writable side of the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/terminate) + */ + terminate(): void; +} +interface ReadableWritablePair { + /** + * Provides a convenient, chainable way of piping this readable stream through a transform stream (or any other { writable, readable } pair). It simply pipes the stream into the writable side of the supplied pair, and returns the readable side for further use. + * + * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. + */ + writable: WritableStream; + readable: ReadableStream; +} +/** + * The **`WritableStream`** interface of the Streams API provides a standard abstraction for writing streaming data to a destination, known as a sink. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream) + */ +declare class WritableStream { + constructor(underlyingSink?: UnderlyingSink, queuingStrategy?: QueuingStrategy); + /** + * The **`locked`** read-only property of the WritableStream interface returns a boolean indicating whether the `WritableStream` is locked to a writer. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/locked) + */ + get locked(): boolean; + /** + * The **`abort()`** method of the WritableStream interface aborts the stream, signaling that the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/abort) + */ + abort(reason?: any): Promise; + /** + * The **`close()`** method of the WritableStream interface closes the associated stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/close) + */ + close(): Promise; + /** + * The **`getWriter()`** method of the WritableStream interface returns a new instance of WritableStreamDefaultWriter and locks the stream to that instance. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/getWriter) + */ + getWriter(): WritableStreamDefaultWriter; +} +/** + * The **`WritableStreamDefaultWriter`** interface of the Streams API is the object returned by WritableStream.getWriter() and once created locks the writer to the `WritableStream` ensuring that no other streams can write to the underlying sink. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter) + */ +declare class WritableStreamDefaultWriter { + constructor(stream: WritableStream); + /** + * The **`closed`** read-only property of the the stream errors or the writer's lock is released. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/closed) + */ + get closed(): Promise; + /** + * The **`ready`** read-only property of the that resolves when the desired size of the stream's internal queue transitions from non-positive to positive, signaling that it is no longer applying backpressure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/ready) + */ + get ready(): Promise; + /** + * The **`desiredSize`** read-only property of the to fill the stream's internal queue. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/desiredSize) + */ + get desiredSize(): number | null; + /** + * The **`abort()`** method of the the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/abort) + */ + abort(reason?: any): Promise; + /** + * The **`close()`** method of the stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/close) + */ + close(): Promise; + /** + * The **`write()`** method of the operation. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/write) + */ + write(chunk?: W): Promise; + /** + * The **`releaseLock()`** method of the corresponding stream. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/releaseLock) + */ + releaseLock(): void; +} +/** + * The **`TransformStream`** interface of the Streams API represents a concrete implementation of the pipe chain _transform stream_ concept. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream) + */ +declare class TransformStream { + constructor(transformer?: Transformer, writableStrategy?: QueuingStrategy, readableStrategy?: QueuingStrategy); + /** + * The **`readable`** read-only property of the TransformStream interface returns the ReadableStream instance controlled by this `TransformStream`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/readable) + */ + get readable(): ReadableStream; + /** + * The **`writable`** read-only property of the TransformStream interface returns the WritableStream instance controlled by this `TransformStream`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/writable) + */ + get writable(): WritableStream; +} +declare class FixedLengthStream extends IdentityTransformStream { + constructor(expectedLength: number | bigint, queuingStrategy?: IdentityTransformStreamQueuingStrategy); +} +declare class IdentityTransformStream extends TransformStream { + constructor(queuingStrategy?: IdentityTransformStreamQueuingStrategy); +} +interface IdentityTransformStreamQueuingStrategy { + highWaterMark?: (number | bigint); +} +interface ReadableStreamValuesOptions { + preventCancel?: boolean; +} +/** + * The **`CompressionStream`** interface of the Compression Streams API is an API for compressing a stream of data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CompressionStream) + */ +declare class CompressionStream extends TransformStream { + constructor(format: "gzip" | "deflate" | "deflate-raw"); +} +/** + * The **`DecompressionStream`** interface of the Compression Streams API is an API for decompressing a stream of data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DecompressionStream) + */ +declare class DecompressionStream extends TransformStream { + constructor(format: "gzip" | "deflate" | "deflate-raw"); +} +/** + * The **`TextEncoderStream`** interface of the Encoding API converts a stream of strings into bytes in the UTF-8 encoding. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoderStream) + */ +declare class TextEncoderStream extends TransformStream { + constructor(); + get encoding(): string; +} +/** + * The **`TextDecoderStream`** interface of the Encoding API converts a stream of text in a binary encoding, such as UTF-8 etc., to a stream of strings. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoderStream) + */ +declare class TextDecoderStream extends TransformStream { + constructor(label?: string, options?: TextDecoderStreamTextDecoderStreamInit); + get encoding(): string; + get fatal(): boolean; + get ignoreBOM(): boolean; +} +interface TextDecoderStreamTextDecoderStreamInit { + fatal?: boolean; + ignoreBOM?: boolean; +} +/** + * The **`ByteLengthQueuingStrategy`** interface of the Streams API provides a built-in byte length queuing strategy that can be used when constructing streams. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy) + */ +declare class ByteLengthQueuingStrategy implements QueuingStrategy { + constructor(init: QueuingStrategyInit); + /** + * The read-only **`ByteLengthQueuingStrategy.highWaterMark`** property returns the total number of bytes that can be contained in the internal queue before backpressure is applied. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/highWaterMark) + */ + get highWaterMark(): number; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/size) */ + get size(): (chunk?: any) => number; +} +/** + * The **`CountQueuingStrategy`** interface of the Streams API provides a built-in chunk counting queuing strategy that can be used when constructing streams. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy) + */ +declare class CountQueuingStrategy implements QueuingStrategy { + constructor(init: QueuingStrategyInit); + /** + * The read-only **`CountQueuingStrategy.highWaterMark`** property returns the total number of chunks that can be contained in the internal queue before backpressure is applied. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/highWaterMark) + */ + get highWaterMark(): number; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/size) */ + get size(): (chunk?: any) => number; +} +interface QueuingStrategyInit { + /** + * Creates a new ByteLengthQueuingStrategy with the provided high water mark. + * + * Note that the provided high water mark will not be validated ahead of time. Instead, if it is negative, NaN, or not a number, the resulting ByteLengthQueuingStrategy will cause the corresponding stream constructor to throw. + */ + highWaterMark: number; +} +interface ScriptVersion { + id?: string; + tag?: string; + message?: string; +} +declare abstract class TailEvent extends ExtendableEvent { + readonly events: TraceItem[]; + readonly traces: TraceItem[]; +} +interface TraceItem { + readonly event: (TraceItemFetchEventInfo | TraceItemJsRpcEventInfo | TraceItemScheduledEventInfo | TraceItemAlarmEventInfo | TraceItemQueueEventInfo | TraceItemEmailEventInfo | TraceItemTailEventInfo | TraceItemCustomEventInfo | TraceItemHibernatableWebSocketEventInfo) | null; + readonly eventTimestamp: number | null; + readonly logs: TraceLog[]; + readonly exceptions: TraceException[]; + readonly diagnosticsChannelEvents: TraceDiagnosticChannelEvent[]; + readonly scriptName: string | null; + readonly entrypoint?: string; + readonly scriptVersion?: ScriptVersion; + readonly dispatchNamespace?: string; + readonly scriptTags?: string[]; + readonly durableObjectId?: string; + readonly outcome: string; + readonly executionModel: string; + readonly truncated: boolean; + readonly cpuTime: number; + readonly wallTime: number; +} +interface TraceItemAlarmEventInfo { + readonly scheduledTime: Date; +} +interface TraceItemCustomEventInfo { +} +interface TraceItemScheduledEventInfo { + readonly scheduledTime: number; + readonly cron: string; +} +interface TraceItemQueueEventInfo { + readonly queue: string; + readonly batchSize: number; +} +interface TraceItemEmailEventInfo { + readonly mailFrom: string; + readonly rcptTo: string; + readonly rawSize: number; +} +interface TraceItemTailEventInfo { + readonly consumedEvents: TraceItemTailEventInfoTailItem[]; +} +interface TraceItemTailEventInfoTailItem { + readonly scriptName: string | null; +} +interface TraceItemFetchEventInfo { + readonly response?: TraceItemFetchEventInfoResponse; + readonly request: TraceItemFetchEventInfoRequest; +} +interface TraceItemFetchEventInfoRequest { + readonly cf?: any; + readonly headers: Record; + readonly method: string; + readonly url: string; + getUnredacted(): TraceItemFetchEventInfoRequest; +} +interface TraceItemFetchEventInfoResponse { + readonly status: number; +} +interface TraceItemJsRpcEventInfo { + readonly rpcMethod: string; +} +interface TraceItemHibernatableWebSocketEventInfo { + readonly getWebSocketEvent: TraceItemHibernatableWebSocketEventInfoMessage | TraceItemHibernatableWebSocketEventInfoClose | TraceItemHibernatableWebSocketEventInfoError; +} +interface TraceItemHibernatableWebSocketEventInfoMessage { + readonly webSocketEventType: string; +} +interface TraceItemHibernatableWebSocketEventInfoClose { + readonly webSocketEventType: string; + readonly code: number; + readonly wasClean: boolean; +} +interface TraceItemHibernatableWebSocketEventInfoError { + readonly webSocketEventType: string; +} +interface TraceLog { + readonly timestamp: number; + readonly level: string; + readonly message: any; +} +interface TraceException { + readonly timestamp: number; + readonly message: string; + readonly name: string; + readonly stack?: string; +} +interface TraceDiagnosticChannelEvent { + readonly timestamp: number; + readonly channel: string; + readonly message: any; +} +interface TraceMetrics { + readonly cpuTime: number; + readonly wallTime: number; +} +interface UnsafeTraceMetrics { + fromTrace(item: TraceItem): TraceMetrics; +} +/** + * The **`URL`** interface is used to parse, construct, normalize, and encode URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL) + */ +declare class URL { + constructor(url: string | URL, base?: string | URL); + /** + * The **`origin`** read-only property of the URL interface returns a string containing the Unicode serialization of the origin of the represented URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/origin) + */ + get origin(): string; + /** + * The **`href`** property of the URL interface is a string containing the whole URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) + */ + get href(): string; + /** + * The **`href`** property of the URL interface is a string containing the whole URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) + */ + set href(value: string); + /** + * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final `':'`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) + */ + get protocol(): string; + /** + * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final `':'`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) + */ + set protocol(value: string); + /** + * The **`username`** property of the URL interface is a string containing the username component of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) + */ + get username(): string; + /** + * The **`username`** property of the URL interface is a string containing the username component of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) + */ + set username(value: string); + /** + * The **`password`** property of the URL interface is a string containing the password component of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) + */ + get password(): string; + /** + * The **`password`** property of the URL interface is a string containing the password component of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) + */ + set password(value: string); + /** + * The **`host`** property of the URL interface is a string containing the host, which is the URL.hostname, and then, if the port of the URL is nonempty, a `':'`, followed by the URL.port of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) + */ + get host(): string; + /** + * The **`host`** property of the URL interface is a string containing the host, which is the URL.hostname, and then, if the port of the URL is nonempty, a `':'`, followed by the URL.port of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) + */ + set host(value: string); + /** + * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) + */ + get hostname(): string; + /** + * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) + */ + set hostname(value: string); + /** + * The **`port`** property of the URL interface is a string containing the port number of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) + */ + get port(): string; + /** + * The **`port`** property of the URL interface is a string containing the port number of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) + */ + set port(value: string); + /** + * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) + */ + get pathname(): string; + /** + * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) + */ + set pathname(value: string); + /** + * The **`search`** property of the URL interface is a search string, also called a _query string_, that is a string containing a `'?'` followed by the parameters of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) + */ + get search(): string; + /** + * The **`search`** property of the URL interface is a search string, also called a _query string_, that is a string containing a `'?'` followed by the parameters of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) + */ + set search(value: string); + /** + * The **`hash`** property of the URL interface is a string containing a `'#'` followed by the fragment identifier of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) + */ + get hash(): string; + /** + * The **`hash`** property of the URL interface is a string containing a `'#'` followed by the fragment identifier of the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) + */ + set hash(value: string); + /** + * The **`searchParams`** read-only property of the access to the [MISSING: httpmethod('GET')] decoded query arguments contained in the URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/searchParams) + */ + get searchParams(): URLSearchParams; + /** + * The **`toJSON()`** method of the URL interface returns a string containing a serialized version of the URL, although in practice it seems to have the same effect as ```js-nolint toJSON() ``` None. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/toJSON) + */ + toJSON(): string; + /*function toString() { [native code] }*/ + toString(): string; + /** + * The **`URL.canParse()`** static method of the URL interface returns a boolean indicating whether or not an absolute URL, or a relative URL combined with a base URL, are parsable and valid. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/canParse_static) + */ + static canParse(url: string, base?: string): boolean; + /** + * The **`URL.parse()`** static method of the URL interface returns a newly created URL object representing the URL defined by the parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/parse_static) + */ + static parse(url: string, base?: string): URL | null; + /** + * The **`createObjectURL()`** static method of the URL interface creates a string containing a URL representing the object given in the parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/createObjectURL_static) + */ + static createObjectURL(object: File | Blob): string; + /** + * The **`revokeObjectURL()`** static method of the URL interface releases an existing object URL which was previously created by calling Call this method when you've finished using an object URL to let the browser know not to keep the reference to the file any longer. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/revokeObjectURL_static) + */ + static revokeObjectURL(object_url: string): void; +} +/** + * The **`URLSearchParams`** interface defines utility methods to work with the query string of a URL. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams) + */ +declare class URLSearchParams { + constructor(init?: (Iterable> | Record | string)); + /** + * The **`size`** read-only property of the URLSearchParams interface indicates the total number of search parameter entries. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/size) + */ + get size(): number; + /** + * The **`append()`** method of the URLSearchParams interface appends a specified key/value pair as a new search parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/append) + */ + append(name: string, value: string): void; + /** + * The **`delete()`** method of the URLSearchParams interface deletes specified parameters and their associated value(s) from the list of all search parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/delete) + */ + delete(name: string, value?: string): void; + /** + * The **`get()`** method of the URLSearchParams interface returns the first value associated to the given search parameter. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/get) + */ + get(name: string): string | null; + /** + * The **`getAll()`** method of the URLSearchParams interface returns all the values associated with a given search parameter as an array. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/getAll) + */ + getAll(name: string): string[]; + /** + * The **`has()`** method of the URLSearchParams interface returns a boolean value that indicates whether the specified parameter is in the search parameters. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/has) + */ + has(name: string, value?: string): boolean; + /** + * The **`set()`** method of the URLSearchParams interface sets the value associated with a given search parameter to the given value. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/set) + */ + set(name: string, value: string): void; + /** + * The **`URLSearchParams.sort()`** method sorts all key/value pairs contained in this object in place and returns `undefined`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/sort) + */ + sort(): void; + /* Returns an array of key, value pairs for every entry in the search params. */ + entries(): IterableIterator<[ + key: string, + value: string + ]>; + /* Returns a list of keys in the search params. */ + keys(): IterableIterator; + /* Returns a list of values in the search params. */ + values(): IterableIterator; + forEach(callback: (this: This, value: string, key: string, parent: URLSearchParams) => void, thisArg?: This): void; + /*function toString() { [native code] }*/ + toString(): string; + [Symbol.iterator](): IterableIterator<[ + key: string, + value: string + ]>; +} +declare class URLPattern { + constructor(input?: (string | URLPatternInit), baseURL?: (string | URLPatternOptions), patternOptions?: URLPatternOptions); + get protocol(): string; + get username(): string; + get password(): string; + get hostname(): string; + get port(): string; + get pathname(): string; + get search(): string; + get hash(): string; + get hasRegExpGroups(): boolean; + test(input?: (string | URLPatternInit), baseURL?: string): boolean; + exec(input?: (string | URLPatternInit), baseURL?: string): URLPatternResult | null; +} +interface URLPatternInit { + protocol?: string; + username?: string; + password?: string; + hostname?: string; + port?: string; + pathname?: string; + search?: string; + hash?: string; + baseURL?: string; +} +interface URLPatternComponentResult { + input: string; + groups: Record; +} +interface URLPatternResult { + inputs: (string | URLPatternInit)[]; + protocol: URLPatternComponentResult; + username: URLPatternComponentResult; + password: URLPatternComponentResult; + hostname: URLPatternComponentResult; + port: URLPatternComponentResult; + pathname: URLPatternComponentResult; + search: URLPatternComponentResult; + hash: URLPatternComponentResult; +} +interface URLPatternOptions { + ignoreCase?: boolean; +} +/** + * A `CloseEvent` is sent to clients using WebSockets when the connection is closed. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent) + */ +declare class CloseEvent extends Event { + constructor(type: string, initializer?: CloseEventInit); + /** + * The **`code`** read-only property of the CloseEvent interface returns a WebSocket connection close code indicating the reason the connection was closed. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/code) + */ + readonly code: number; + /** + * The **`reason`** read-only property of the CloseEvent interface returns the WebSocket connection close reason the server gave for closing the connection; that is, a concise human-readable prose explanation for the closure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/reason) + */ + readonly reason: string; + /** + * The **`wasClean`** read-only property of the CloseEvent interface returns `true` if the connection closed cleanly. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/wasClean) + */ + readonly wasClean: boolean; +} +interface CloseEventInit { + code?: number; + reason?: string; + wasClean?: boolean; +} +type WebSocketEventMap = { + close: CloseEvent; + message: MessageEvent; + open: Event; + error: ErrorEvent; +}; +/** + * The `WebSocket` object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) + */ +declare var WebSocket: { + prototype: WebSocket; + new (url: string, protocols?: (string[] | string)): WebSocket; + readonly READY_STATE_CONNECTING: number; + readonly CONNECTING: number; + readonly READY_STATE_OPEN: number; + readonly OPEN: number; + readonly READY_STATE_CLOSING: number; + readonly CLOSING: number; + readonly READY_STATE_CLOSED: number; + readonly CLOSED: number; +}; +/** + * The `WebSocket` object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) + */ +interface WebSocket extends EventTarget { + accept(): void; + /** + * The **`WebSocket.send()`** method enqueues the specified data to be transmitted to the server over the WebSocket connection, increasing the value of `bufferedAmount` by the number of bytes needed to contain the data. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/send) + */ + send(message: (ArrayBuffer | ArrayBufferView) | string): void; + /** + * The **`WebSocket.close()`** method closes the already `CLOSED`, this method does nothing. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/close) + */ + close(code?: number, reason?: string): void; + serializeAttachment(attachment: any): void; + deserializeAttachment(): any | null; + /** + * The **`WebSocket.readyState`** read-only property returns the current state of the WebSocket connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/readyState) + */ + readyState: number; + /** + * The **`WebSocket.url`** read-only property returns the absolute URL of the WebSocket as resolved by the constructor. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/url) + */ + url: string | null; + /** + * The **`WebSocket.protocol`** read-only property returns the name of the sub-protocol the server selected; this will be one of the strings specified in the `protocols` parameter when creating the WebSocket object, or the empty string if no connection is established. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/protocol) + */ + protocol: string | null; + /** + * The **`WebSocket.extensions`** read-only property returns the extensions selected by the server. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/extensions) + */ + extensions: string | null; +} +declare const WebSocketPair: { + new (): { + 0: WebSocket; + 1: WebSocket; + }; +}; +interface SqlStorage { + exec>(query: string, ...bindings: any[]): SqlStorageCursor; + get databaseSize(): number; + Cursor: typeof SqlStorageCursor; + Statement: typeof SqlStorageStatement; +} +declare abstract class SqlStorageStatement { +} +type SqlStorageValue = ArrayBuffer | string | number | null; +declare abstract class SqlStorageCursor> { + next(): { + done?: false; + value: T; + } | { + done: true; + value?: never; + }; + toArray(): T[]; + one(): T; + raw(): IterableIterator; + columnNames: string[]; + get rowsRead(): number; + get rowsWritten(): number; + [Symbol.iterator](): IterableIterator; +} +interface Socket { + get readable(): ReadableStream; + get writable(): WritableStream; + get closed(): Promise; + get opened(): Promise; + get upgraded(): boolean; + get secureTransport(): "on" | "off" | "starttls"; + close(): Promise; + startTls(options?: TlsOptions): Socket; +} +interface SocketOptions { + secureTransport?: string; + allowHalfOpen: boolean; + highWaterMark?: (number | bigint); +} +interface SocketAddress { + hostname: string; + port: number; +} +interface TlsOptions { + expectedServerHostname?: string; +} +interface SocketInfo { + remoteAddress?: string; + localAddress?: string; +} +/** + * The **`EventSource`** interface is web content's interface to server-sent events. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource) + */ +declare class EventSource extends EventTarget { + constructor(url: string, init?: EventSourceEventSourceInit); + /** + * The **`close()`** method of the EventSource interface closes the connection, if one is made, and sets the ```js-nolint close() ``` None. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/close) + */ + close(): void; + /** + * The **`url`** read-only property of the URL of the source. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/url) + */ + get url(): string; + /** + * The **`withCredentials`** read-only property of the the `EventSource` object was instantiated with CORS credentials set. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/withCredentials) + */ + get withCredentials(): boolean; + /** + * The **`readyState`** read-only property of the connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/readyState) + */ + get readyState(): number; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ + get onopen(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ + set onopen(value: any | null); + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ + get onmessage(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ + set onmessage(value: any | null); + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ + get onerror(): any | null; + /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ + set onerror(value: any | null); + static readonly CONNECTING: number; + static readonly OPEN: number; + static readonly CLOSED: number; + static from(stream: ReadableStream): EventSource; +} +interface EventSourceEventSourceInit { + withCredentials?: boolean; + fetcher?: Fetcher; +} +interface Container { + get running(): boolean; + start(options?: ContainerStartupOptions): void; + monitor(): Promise; + destroy(error?: any): Promise; + signal(signo: number): void; + getTcpPort(port: number): Fetcher; + setInactivityTimeout(durationMs: number | bigint): Promise; +} +interface ContainerStartupOptions { + entrypoint?: string[]; + enableInternet: boolean; + env?: Record; + hardTimeout?: (number | bigint); +} +/** + * The **`MessagePort`** interface of the Channel Messaging API represents one of the two ports of a MessageChannel, allowing messages to be sent from one port and listening out for them arriving at the other. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort) + */ +declare abstract class MessagePort extends EventTarget { + /** + * The **`postMessage()`** method of the transfers ownership of objects to other browsing contexts. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/postMessage) + */ + postMessage(data?: any, options?: (any[] | MessagePortPostMessageOptions)): void; + /** + * The **`close()`** method of the MessagePort interface disconnects the port, so it is no longer active. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/close) + */ + close(): void; + /** + * The **`start()`** method of the MessagePort interface starts the sending of messages queued on the port. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/start) + */ + start(): void; + get onmessage(): any | null; + set onmessage(value: any | null); +} +/** + * The **`MessageChannel`** interface of the Channel Messaging API allows us to create a new message channel and send data through it via its two MessagePort properties. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel) + */ +declare class MessageChannel { + constructor(); + /** + * The **`port1`** read-only property of the the port attached to the context that originated the channel. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port1) + */ + readonly port1: MessagePort; + /** + * The **`port2`** read-only property of the the port attached to the context at the other end of the channel, which the message is initially sent to. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port2) + */ + readonly port2: MessagePort; +} +interface MessagePortPostMessageOptions { + transfer?: any[]; +} +type LoopbackForExport Rpc.EntrypointBranded) | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? LoopbackServiceStub> : T extends new (...args: any[]) => Rpc.DurableObjectBranded ? LoopbackDurableObjectClass> : T extends ExportedHandler ? LoopbackServiceStub : undefined; +type LoopbackServiceStub = Fetcher & (T extends CloudflareWorkersModule.WorkerEntrypoint ? (opts: { + props?: Props; +}) => Fetcher : (opts: { + props?: any; +}) => Fetcher); +type LoopbackDurableObjectClass = DurableObjectClass & (T extends CloudflareWorkersModule.DurableObject ? (opts: { + props?: Props; +}) => DurableObjectClass : (opts: { + props?: any; +}) => DurableObjectClass); +interface SyncKvStorage { + get(key: string): T | undefined; + list(options?: SyncKvListOptions): Iterable<[ + string, + T + ]>; + put(key: string, value: T): void; + delete(key: string): boolean; +} +interface SyncKvListOptions { + start?: string; + startAfter?: string; + end?: string; + prefix?: string; + reverse?: boolean; + limit?: number; +} +interface WorkerStub { + getEntrypoint(name?: string, options?: WorkerStubEntrypointOptions): Fetcher; +} +interface WorkerStubEntrypointOptions { + props?: any; +} +interface WorkerLoader { + get(name: string | null, getCode: () => WorkerLoaderWorkerCode | Promise): WorkerStub; +} +interface WorkerLoaderModule { + js?: string; + cjs?: string; + text?: string; + data?: ArrayBuffer; + json?: any; + py?: string; + wasm?: ArrayBuffer; +} +interface WorkerLoaderWorkerCode { + compatibilityDate: string; + compatibilityFlags?: string[]; + allowExperimental?: boolean; + mainModule: string; + modules: Record; + env?: any; + globalOutbound?: (Fetcher | null); + tails?: Fetcher[]; + streamingTails?: Fetcher[]; +} +/** +* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, +* as well as timing of subrequests and other operations. +* +* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) +*/ +declare abstract class Performance { + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancetimeorigin) */ + get timeOrigin(): number; + /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancenow) */ + now(): number; +} +type AiImageClassificationInput = { + image: number[]; +}; +type AiImageClassificationOutput = { + score?: number; + label?: string; +}[]; +declare abstract class BaseAiImageClassification { + inputs: AiImageClassificationInput; + postProcessedOutputs: AiImageClassificationOutput; +} +type AiImageToTextInput = { + image: number[]; + prompt?: string; + max_tokens?: number; + temperature?: number; + top_p?: number; + top_k?: number; + seed?: number; + repetition_penalty?: number; + frequency_penalty?: number; + presence_penalty?: number; + raw?: boolean; + messages?: RoleScopedChatInput[]; +}; +type AiImageToTextOutput = { + description: string; +}; +declare abstract class BaseAiImageToText { + inputs: AiImageToTextInput; + postProcessedOutputs: AiImageToTextOutput; +} +type AiImageTextToTextInput = { + image: string; + prompt?: string; + max_tokens?: number; + temperature?: number; + ignore_eos?: boolean; + top_p?: number; + top_k?: number; + seed?: number; + repetition_penalty?: number; + frequency_penalty?: number; + presence_penalty?: number; + raw?: boolean; + messages?: RoleScopedChatInput[]; +}; +type AiImageTextToTextOutput = { + description: string; +}; +declare abstract class BaseAiImageTextToText { + inputs: AiImageTextToTextInput; + postProcessedOutputs: AiImageTextToTextOutput; +} +type AiMultimodalEmbeddingsInput = { + image: string; + text: string[]; +}; +type AiIMultimodalEmbeddingsOutput = { + data: number[][]; + shape: number[]; +}; +declare abstract class BaseAiMultimodalEmbeddings { + inputs: AiImageTextToTextInput; + postProcessedOutputs: AiImageTextToTextOutput; +} +type AiObjectDetectionInput = { + image: number[]; +}; +type AiObjectDetectionOutput = { + score?: number; + label?: string; +}[]; +declare abstract class BaseAiObjectDetection { + inputs: AiObjectDetectionInput; + postProcessedOutputs: AiObjectDetectionOutput; +} +type AiSentenceSimilarityInput = { + source: string; + sentences: string[]; +}; +type AiSentenceSimilarityOutput = number[]; +declare abstract class BaseAiSentenceSimilarity { + inputs: AiSentenceSimilarityInput; + postProcessedOutputs: AiSentenceSimilarityOutput; +} +type AiAutomaticSpeechRecognitionInput = { + audio: number[]; +}; +type AiAutomaticSpeechRecognitionOutput = { + text?: string; + words?: { + word: string; + start: number; + end: number; + }[]; + vtt?: string; +}; +declare abstract class BaseAiAutomaticSpeechRecognition { + inputs: AiAutomaticSpeechRecognitionInput; + postProcessedOutputs: AiAutomaticSpeechRecognitionOutput; +} +type AiSummarizationInput = { + input_text: string; + max_length?: number; +}; +type AiSummarizationOutput = { + summary: string; +}; +declare abstract class BaseAiSummarization { + inputs: AiSummarizationInput; + postProcessedOutputs: AiSummarizationOutput; +} +type AiTextClassificationInput = { + text: string; +}; +type AiTextClassificationOutput = { + score?: number; + label?: string; +}[]; +declare abstract class BaseAiTextClassification { + inputs: AiTextClassificationInput; + postProcessedOutputs: AiTextClassificationOutput; +} +type AiTextEmbeddingsInput = { + text: string | string[]; +}; +type AiTextEmbeddingsOutput = { + shape: number[]; + data: number[][]; +}; +declare abstract class BaseAiTextEmbeddings { + inputs: AiTextEmbeddingsInput; + postProcessedOutputs: AiTextEmbeddingsOutput; +} +type RoleScopedChatInput = { + role: "user" | "assistant" | "system" | "tool" | (string & NonNullable); + content: string; + name?: string; +}; +type AiTextGenerationToolLegacyInput = { + name: string; + description: string; + parameters?: { + type: "object" | (string & NonNullable); + properties: { + [key: string]: { + type: string; + description?: string; + }; + }; + required: string[]; + }; +}; +type AiTextGenerationToolInput = { + type: "function" | (string & NonNullable); + function: { + name: string; + description: string; + parameters?: { + type: "object" | (string & NonNullable); + properties: { + [key: string]: { + type: string; + description?: string; + }; + }; + required: string[]; + }; + }; +}; +type AiTextGenerationFunctionsInput = { + name: string; + code: string; +}; +type AiTextGenerationResponseFormat = { + type: string; + json_schema?: any; +}; +type AiTextGenerationInput = { + prompt?: string; + raw?: boolean; + stream?: boolean; + max_tokens?: number; + temperature?: number; + top_p?: number; + top_k?: number; + seed?: number; + repetition_penalty?: number; + frequency_penalty?: number; + presence_penalty?: number; + messages?: RoleScopedChatInput[]; + response_format?: AiTextGenerationResponseFormat; + tools?: AiTextGenerationToolInput[] | AiTextGenerationToolLegacyInput[] | (object & NonNullable); + functions?: AiTextGenerationFunctionsInput[]; +}; +type AiTextGenerationToolLegacyOutput = { + name: string; + arguments: unknown; +}; +type AiTextGenerationToolOutput = { + id: string; + type: "function"; + function: { + name: string; + arguments: string; + }; +}; +type UsageTags = { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; +}; +type AiTextGenerationOutput = { + response?: string; + tool_calls?: AiTextGenerationToolLegacyOutput[] & AiTextGenerationToolOutput[]; + usage?: UsageTags; +}; +declare abstract class BaseAiTextGeneration { + inputs: AiTextGenerationInput; + postProcessedOutputs: AiTextGenerationOutput; +} +type AiTextToSpeechInput = { + prompt: string; + lang?: string; +}; +type AiTextToSpeechOutput = Uint8Array | { + audio: string; +}; +declare abstract class BaseAiTextToSpeech { + inputs: AiTextToSpeechInput; + postProcessedOutputs: AiTextToSpeechOutput; +} +type AiTextToImageInput = { + prompt: string; + negative_prompt?: string; + height?: number; + width?: number; + image?: number[]; + image_b64?: string; + mask?: number[]; + num_steps?: number; + strength?: number; + guidance?: number; + seed?: number; +}; +type AiTextToImageOutput = ReadableStream; +declare abstract class BaseAiTextToImage { + inputs: AiTextToImageInput; + postProcessedOutputs: AiTextToImageOutput; +} +type AiTranslationInput = { + text: string; + target_lang: string; + source_lang?: string; +}; +type AiTranslationOutput = { + translated_text?: string; +}; +declare abstract class BaseAiTranslation { + inputs: AiTranslationInput; + postProcessedOutputs: AiTranslationOutput; +} +/** + * Workers AI support for OpenAI's Responses API + * Reference: https://github.com/openai/openai-node/blob/master/src/resources/responses/responses.ts + * + * It's a stripped down version from its source. + * It currently supports basic function calling, json mode and accepts images as input. + * + * It does not include types for WebSearch, CodeInterpreter, FileInputs, MCP, CustomTools. + * We plan to add those incrementally as model + platform capabilities evolve. + */ +type ResponsesInput = { + background?: boolean | null; + conversation?: string | ResponseConversationParam | null; + include?: Array | null; + input?: string | ResponseInput; + instructions?: string | null; + max_output_tokens?: number | null; + parallel_tool_calls?: boolean | null; + previous_response_id?: string | null; + prompt_cache_key?: string; + reasoning?: Reasoning | null; + safety_identifier?: string; + service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; + stream?: boolean | null; + stream_options?: StreamOptions | null; + temperature?: number | null; + text?: ResponseTextConfig; + tool_choice?: ToolChoiceOptions | ToolChoiceFunction; + tools?: Array; + top_p?: number | null; + truncation?: "auto" | "disabled" | null; +}; +type ResponsesOutput = { + id?: string; + created_at?: number; + output_text?: string; + error?: ResponseError | null; + incomplete_details?: ResponseIncompleteDetails | null; + instructions?: string | Array | null; + object?: "response"; + output?: Array; + parallel_tool_calls?: boolean; + temperature?: number | null; + tool_choice?: ToolChoiceOptions | ToolChoiceFunction; + tools?: Array; + top_p?: number | null; + max_output_tokens?: number | null; + previous_response_id?: string | null; + prompt?: ResponsePrompt | null; + reasoning?: Reasoning | null; + safety_identifier?: string; + service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; + status?: ResponseStatus; + text?: ResponseTextConfig; + truncation?: "auto" | "disabled" | null; + usage?: ResponseUsage; +}; +type EasyInputMessage = { + content: string | ResponseInputMessageContentList; + role: "user" | "assistant" | "system" | "developer"; + type?: "message"; +}; +type ResponsesFunctionTool = { + name: string; + parameters: { + [key: string]: unknown; + } | null; + strict: boolean | null; + type: "function"; + description?: string | null; +}; +type ResponseIncompleteDetails = { + reason?: "max_output_tokens" | "content_filter"; +}; +type ResponsePrompt = { + id: string; + variables?: { + [key: string]: string | ResponseInputText | ResponseInputImage; + } | null; + version?: string | null; +}; +type Reasoning = { + effort?: ReasoningEffort | null; + generate_summary?: "auto" | "concise" | "detailed" | null; + summary?: "auto" | "concise" | "detailed" | null; +}; +type ResponseContent = ResponseInputText | ResponseInputImage | ResponseOutputText | ResponseOutputRefusal | ResponseContentReasoningText; +type ResponseContentReasoningText = { + text: string; + type: "reasoning_text"; +}; +type ResponseConversationParam = { + id: string; +}; +type ResponseCreatedEvent = { + response: Response; + sequence_number: number; + type: "response.created"; +}; +type ResponseCustomToolCallOutput = { + call_id: string; + output: string | Array; + type: "custom_tool_call_output"; + id?: string; +}; +type ResponseError = { + code: "server_error" | "rate_limit_exceeded" | "invalid_prompt" | "vector_store_timeout" | "invalid_image" | "invalid_image_format" | "invalid_base64_image" | "invalid_image_url" | "image_too_large" | "image_too_small" | "image_parse_error" | "image_content_policy_violation" | "invalid_image_mode" | "image_file_too_large" | "unsupported_image_media_type" | "empty_image_file" | "failed_to_download_image" | "image_file_not_found"; + message: string; +}; +type ResponseErrorEvent = { + code: string | null; + message: string; + param: string | null; + sequence_number: number; + type: "error"; +}; +type ResponseFailedEvent = { + response: Response; + sequence_number: number; + type: "response.failed"; +}; +type ResponseFormatText = { + type: "text"; +}; +type ResponseFormatJSONObject = { + type: "json_object"; +}; +type ResponseFormatTextConfig = ResponseFormatText | ResponseFormatTextJSONSchemaConfig | ResponseFormatJSONObject; +type ResponseFormatTextJSONSchemaConfig = { + name: string; + schema: { + [key: string]: unknown; + }; + type: "json_schema"; + description?: string; + strict?: boolean | null; +}; +type ResponseFunctionCallArgumentsDeltaEvent = { + delta: string; + item_id: string; + output_index: number; + sequence_number: number; + type: "response.function_call_arguments.delta"; +}; +type ResponseFunctionCallArgumentsDoneEvent = { + arguments: string; + item_id: string; + name: string; + output_index: number; + sequence_number: number; + type: "response.function_call_arguments.done"; +}; +type ResponseFunctionCallOutputItem = ResponseInputTextContent | ResponseInputImageContent; +type ResponseFunctionCallOutputItemList = Array; +type ResponseFunctionToolCall = { + arguments: string; + call_id: string; + name: string; + type: "function_call"; + id?: string; + status?: "in_progress" | "completed" | "incomplete"; +}; +interface ResponseFunctionToolCallItem extends ResponseFunctionToolCall { + id: string; +} +type ResponseFunctionToolCallOutputItem = { + id: string; + call_id: string; + output: string | Array; + type: "function_call_output"; + status?: "in_progress" | "completed" | "incomplete"; +}; +type ResponseIncludable = "message.input_image.image_url" | "message.output_text.logprobs"; +type ResponseIncompleteEvent = { + response: Response; + sequence_number: number; + type: "response.incomplete"; +}; +type ResponseInput = Array; +type ResponseInputContent = ResponseInputText | ResponseInputImage; +type ResponseInputImage = { + detail: "low" | "high" | "auto"; + type: "input_image"; + /** + * Base64 encoded image + */ + image_url?: string | null; +}; +type ResponseInputImageContent = { + type: "input_image"; + detail?: "low" | "high" | "auto" | null; + /** + * Base64 encoded image + */ + image_url?: string | null; +}; +type ResponseInputItem = EasyInputMessage | ResponseInputItemMessage | ResponseOutputMessage | ResponseFunctionToolCall | ResponseInputItemFunctionCallOutput | ResponseReasoningItem; +type ResponseInputItemFunctionCallOutput = { + call_id: string; + output: string | ResponseFunctionCallOutputItemList; + type: "function_call_output"; + id?: string | null; + status?: "in_progress" | "completed" | "incomplete" | null; +}; +type ResponseInputItemMessage = { + content: ResponseInputMessageContentList; + role: "user" | "system" | "developer"; + status?: "in_progress" | "completed" | "incomplete"; + type?: "message"; +}; +type ResponseInputMessageContentList = Array; +type ResponseInputMessageItem = { + id: string; + content: ResponseInputMessageContentList; + role: "user" | "system" | "developer"; + status?: "in_progress" | "completed" | "incomplete"; + type?: "message"; +}; +type ResponseInputText = { + text: string; + type: "input_text"; +}; +type ResponseInputTextContent = { + text: string; + type: "input_text"; +}; +type ResponseItem = ResponseInputMessageItem | ResponseOutputMessage | ResponseFunctionToolCallItem | ResponseFunctionToolCallOutputItem; +type ResponseOutputItem = ResponseOutputMessage | ResponseFunctionToolCall | ResponseReasoningItem; +type ResponseOutputItemAddedEvent = { + item: ResponseOutputItem; + output_index: number; + sequence_number: number; + type: "response.output_item.added"; +}; +type ResponseOutputItemDoneEvent = { + item: ResponseOutputItem; + output_index: number; + sequence_number: number; + type: "response.output_item.done"; +}; +type ResponseOutputMessage = { + id: string; + content: Array; + role: "assistant"; + status: "in_progress" | "completed" | "incomplete"; + type: "message"; +}; +type ResponseOutputRefusal = { + refusal: string; + type: "refusal"; +}; +type ResponseOutputText = { + text: string; + type: "output_text"; + logprobs?: Array; +}; +type ResponseReasoningItem = { + id: string; + summary: Array; + type: "reasoning"; + content?: Array; + encrypted_content?: string | null; + status?: "in_progress" | "completed" | "incomplete"; +}; +type ResponseReasoningSummaryItem = { + text: string; + type: "summary_text"; +}; +type ResponseReasoningContentItem = { + text: string; + type: "reasoning_text"; +}; +type ResponseReasoningTextDeltaEvent = { + content_index: number; + delta: string; + item_id: string; + output_index: number; + sequence_number: number; + type: "response.reasoning_text.delta"; +}; +type ResponseReasoningTextDoneEvent = { + content_index: number; + item_id: string; + output_index: number; + sequence_number: number; + text: string; + type: "response.reasoning_text.done"; +}; +type ResponseRefusalDeltaEvent = { + content_index: number; + delta: string; + item_id: string; + output_index: number; + sequence_number: number; + type: "response.refusal.delta"; +}; +type ResponseRefusalDoneEvent = { + content_index: number; + item_id: string; + output_index: number; + refusal: string; + sequence_number: number; + type: "response.refusal.done"; +}; +type ResponseStatus = "completed" | "failed" | "in_progress" | "cancelled" | "queued" | "incomplete"; +type ResponseStreamEvent = ResponseCompletedEvent | ResponseCreatedEvent | ResponseErrorEvent | ResponseFunctionCallArgumentsDeltaEvent | ResponseFunctionCallArgumentsDoneEvent | ResponseFailedEvent | ResponseIncompleteEvent | ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent | ResponseReasoningTextDeltaEvent | ResponseReasoningTextDoneEvent | ResponseRefusalDeltaEvent | ResponseRefusalDoneEvent | ResponseTextDeltaEvent | ResponseTextDoneEvent; +type ResponseCompletedEvent = { + response: Response; + sequence_number: number; + type: "response.completed"; +}; +type ResponseTextConfig = { + format?: ResponseFormatTextConfig; + verbosity?: "low" | "medium" | "high" | null; +}; +type ResponseTextDeltaEvent = { + content_index: number; + delta: string; + item_id: string; + logprobs: Array; + output_index: number; + sequence_number: number; + type: "response.output_text.delta"; +}; +type ResponseTextDoneEvent = { + content_index: number; + item_id: string; + logprobs: Array; + output_index: number; + sequence_number: number; + text: string; + type: "response.output_text.done"; +}; +type Logprob = { + token: string; + logprob: number; + top_logprobs?: Array; +}; +type TopLogprob = { + token?: string; + logprob?: number; +}; +type ResponseUsage = { + input_tokens: number; + output_tokens: number; + total_tokens: number; +}; +type Tool = ResponsesFunctionTool; +type ToolChoiceFunction = { + name: string; + type: "function"; +}; +type ToolChoiceOptions = "none"; +type ReasoningEffort = "minimal" | "low" | "medium" | "high" | null; +type StreamOptions = { + include_obfuscation?: boolean; +}; +type Ai_Cf_Baai_Bge_Base_En_V1_5_Input = { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; + }[]; +}; +type Ai_Cf_Baai_Bge_Base_En_V1_5_Output = { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} | Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse; +interface Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Base_En_V1_5 { + inputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Output; +} +type Ai_Cf_Openai_Whisper_Input = string | { + /** + * An array of integers that represent the audio data constrained to 8-bit unsigned integer values + */ + audio: number[]; +}; +interface Ai_Cf_Openai_Whisper_Output { + /** + * The transcription + */ + text: string; + word_count?: number; + words?: { + word?: string; + /** + * The second this word begins in the recording + */ + start?: number; + /** + * The ending second when the word completes + */ + end?: number; + }[]; + vtt?: string; +} +declare abstract class Base_Ai_Cf_Openai_Whisper { + inputs: Ai_Cf_Openai_Whisper_Input; + postProcessedOutputs: Ai_Cf_Openai_Whisper_Output; +} +type Ai_Cf_Meta_M2M100_1_2B_Input = { + /** + * The text to be translated + */ + text: string; + /** + * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified + */ + source_lang?: string; + /** + * The language code to translate the text into (e.g., 'es' for Spanish) + */ + target_lang: string; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + /** + * The text to be translated + */ + text: string; + /** + * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified + */ + source_lang?: string; + /** + * The language code to translate the text into (e.g., 'es' for Spanish) + */ + target_lang: string; + }[]; +}; +type Ai_Cf_Meta_M2M100_1_2B_Output = { + /** + * The translated text in the target language + */ + translated_text?: string; +} | Ai_Cf_Meta_M2M100_1_2B_AsyncResponse; +interface Ai_Cf_Meta_M2M100_1_2B_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Meta_M2M100_1_2B { + inputs: Ai_Cf_Meta_M2M100_1_2B_Input; + postProcessedOutputs: Ai_Cf_Meta_M2M100_1_2B_Output; +} +type Ai_Cf_Baai_Bge_Small_En_V1_5_Input = { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; + }[]; +}; +type Ai_Cf_Baai_Bge_Small_En_V1_5_Output = { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} | Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse; +interface Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Small_En_V1_5 { + inputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Output; +} +type Ai_Cf_Baai_Bge_Large_En_V1_5_Input = { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; +} | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: { + text: string | string[]; + /** + * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. + */ + pooling?: "mean" | "cls"; + }[]; +}; +type Ai_Cf_Baai_Bge_Large_En_V1_5_Output = { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} | Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse; +interface Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Large_En_V1_5 { + inputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Output; +} +type Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input = string | { + /** + * The input text prompt for the model to generate a response. + */ + prompt?: string; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; + image: number[] | (string & NonNullable); + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; +}; +interface Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output { + description?: string; +} +declare abstract class Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M { + inputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input; + postProcessedOutputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output; +} +type Ai_Cf_Openai_Whisper_Tiny_En_Input = string | { + /** + * An array of integers that represent the audio data constrained to 8-bit unsigned integer values + */ + audio: number[]; +}; +interface Ai_Cf_Openai_Whisper_Tiny_En_Output { + /** + * The transcription + */ + text: string; + word_count?: number; + words?: { + word?: string; + /** + * The second this word begins in the recording + */ + start?: number; + /** + * The ending second when the word completes + */ + end?: number; + }[]; + vtt?: string; +} +declare abstract class Base_Ai_Cf_Openai_Whisper_Tiny_En { + inputs: Ai_Cf_Openai_Whisper_Tiny_En_Input; + postProcessedOutputs: Ai_Cf_Openai_Whisper_Tiny_En_Output; +} +interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input { + /** + * Base64 encoded value of the audio data. + */ + audio: string; + /** + * Supported tasks are 'translate' or 'transcribe'. + */ + task?: string; + /** + * The language of the audio being transcribed or translated. + */ + language?: string; + /** + * Preprocess the audio with a voice activity detection model. + */ + vad_filter?: boolean; + /** + * A text prompt to help provide context to the model on the contents of the audio. + */ + initial_prompt?: string; + /** + * The prefix it appended the the beginning of the output of the transcription and can guide the transcription result. + */ + prefix?: string; +} +interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output { + transcription_info?: { + /** + * The language of the audio being transcribed or translated. + */ + language?: string; + /** + * The confidence level or probability of the detected language being accurate, represented as a decimal between 0 and 1. + */ + language_probability?: number; + /** + * The total duration of the original audio file, in seconds. + */ + duration?: number; + /** + * The duration of the audio after applying Voice Activity Detection (VAD) to remove silent or irrelevant sections, in seconds. + */ + duration_after_vad?: number; + }; + /** + * The complete transcription of the audio. + */ + text: string; + /** + * The total number of words in the transcription. + */ + word_count?: number; + segments?: { + /** + * The starting time of the segment within the audio, in seconds. + */ + start?: number; + /** + * The ending time of the segment within the audio, in seconds. + */ + end?: number; + /** + * The transcription of the segment. + */ + text?: string; + /** + * The temperature used in the decoding process, controlling randomness in predictions. Lower values result in more deterministic outputs. + */ + temperature?: number; + /** + * The average log probability of the predictions for the words in this segment, indicating overall confidence. + */ + avg_logprob?: number; + /** + * The compression ratio of the input to the output, measuring how much the text was compressed during the transcription process. + */ + compression_ratio?: number; + /** + * The probability that the segment contains no speech, represented as a decimal between 0 and 1. + */ + no_speech_prob?: number; + words?: { + /** + * The individual word transcribed from the audio. + */ + word?: string; + /** + * The starting time of the word within the audio, in seconds. + */ + start?: number; + /** + * The ending time of the word within the audio, in seconds. + */ + end?: number; + }[]; + }[]; + /** + * The transcription in WebVTT format, which includes timing and text information for use in subtitles. + */ + vtt?: string; +} +declare abstract class Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo { + inputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input; + postProcessedOutputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output; +} +type Ai_Cf_Baai_Bge_M3_Input = Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts | Ai_Cf_Baai_Bge_M3_Input_Embedding | { + /** + * Batch of the embeddings requests to run using async-queue + */ + requests: (Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 | Ai_Cf_Baai_Bge_M3_Input_Embedding_1)[]; +}; +interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts { + /** + * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts + */ + query?: string; + /** + * List of provided contexts. Note that the index in this array is important, as the response will refer to it. + */ + contexts: { + /** + * One of the provided context content + */ + text?: string; + }[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +interface Ai_Cf_Baai_Bge_M3_Input_Embedding { + text: string | string[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 { + /** + * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts + */ + query?: string; + /** + * List of provided contexts. Note that the index in this array is important, as the response will refer to it. + */ + contexts: { + /** + * One of the provided context content + */ + text?: string; + }[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +interface Ai_Cf_Baai_Bge_M3_Input_Embedding_1 { + text: string | string[]; + /** + * When provided with too long context should the model error out or truncate the context to fit? + */ + truncate_inputs?: boolean; +} +type Ai_Cf_Baai_Bge_M3_Output = Ai_Cf_Baai_Bge_M3_Ouput_Query | Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts | Ai_Cf_Baai_Bge_M3_Ouput_Embedding | Ai_Cf_Baai_Bge_M3_AsyncResponse; +interface Ai_Cf_Baai_Bge_M3_Ouput_Query { + response?: { + /** + * Index of the context in the request + */ + id?: number; + /** + * Score of the context under the index. + */ + score?: number; + }[]; +} +interface Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts { + response?: number[][]; + shape?: number[]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} +interface Ai_Cf_Baai_Bge_M3_Ouput_Embedding { + shape?: number[]; + /** + * Embeddings of the requested text values + */ + data?: number[][]; + /** + * The pooling method used in the embedding process. + */ + pooling?: "mean" | "cls"; +} +interface Ai_Cf_Baai_Bge_M3_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Baai_Bge_M3 { + inputs: Ai_Cf_Baai_Bge_M3_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_M3_Output; +} +interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input { + /** + * A text description of the image you want to generate. + */ + prompt: string; + /** + * The number of diffusion steps; higher values can improve quality but take longer. + */ + steps?: number; +} +interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output { + /** + * The generated image in Base64 format. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell { + inputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input; + postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output; +} +type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input = Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt | Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages; +interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + image?: number[] | (string & NonNullable); + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; +} +interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. Must be supplied for tool calls for Mistral-3. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + image?: number[] | (string & NonNullable); + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * If true, the response will be streamed back incrementally. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output = { + /** + * The generated text response from the model + */ + response?: string; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct { + inputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output; +} +type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input = Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch; +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch { + requests?: { + /** + * User-supplied reference. This field will be present in the response as well it can be used to reference the request and response. It's NOT validated to be unique. + */ + external_reference?: string; + /** + * Prompt for the text generation model + */ + prompt?: string; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; + response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2; + }[]; +} +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +} | string | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse; +interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast { + inputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output; +} +interface Ai_Cf_Meta_Llama_Guard_3_8B_Input { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender must alternate between 'user' and 'assistant'. + */ + role: "user" | "assistant"; + /** + * The content of the message as a string. + */ + content: string; + }[]; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Dictate the output format of the generated response. + */ + response_format?: { + /** + * Set to json_object to process and output generated text as JSON. + */ + type?: string; + }; +} +interface Ai_Cf_Meta_Llama_Guard_3_8B_Output { + response?: string | { + /** + * Whether the conversation is safe or not. + */ + safe?: boolean; + /** + * A list of what hazard categories predicted for the conversation, if the conversation is deemed unsafe. + */ + categories?: string[]; + }; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; +} +declare abstract class Base_Ai_Cf_Meta_Llama_Guard_3_8B { + inputs: Ai_Cf_Meta_Llama_Guard_3_8B_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_Guard_3_8B_Output; +} +interface Ai_Cf_Baai_Bge_Reranker_Base_Input { + /** + * A query you wish to perform against the provided contexts. + */ + /** + * Number of returned results starting with the best score. + */ + top_k?: number; + /** + * List of provided contexts. Note that the index in this array is important, as the response will refer to it. + */ + contexts: { + /** + * One of the provided context content + */ + text?: string; + }[]; +} +interface Ai_Cf_Baai_Bge_Reranker_Base_Output { + response?: { + /** + * Index of the context in the request + */ + id?: number; + /** + * Score of the context under the index. + */ + score?: number; + }[]; +} +declare abstract class Base_Ai_Cf_Baai_Bge_Reranker_Base { + inputs: Ai_Cf_Baai_Bge_Reranker_Base_Input; + postProcessedOutputs: Ai_Cf_Baai_Bge_Reranker_Base_Output; +} +type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input = Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt | Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages; +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct { + inputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output; +} +type Ai_Cf_Qwen_Qwq_32B_Input = Ai_Cf_Qwen_Qwq_32B_Prompt | Ai_Cf_Qwen_Qwq_32B_Messages; +interface Ai_Cf_Qwen_Qwq_32B_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwq_32B_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. Must be supplied for tool calls for Mistral-3. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Qwen_Qwq_32B_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Qwen_Qwq_32B { + inputs: Ai_Cf_Qwen_Qwq_32B_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwq_32B_Output; +} +type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input = Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt | Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages; +interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. Must be supplied for tool calls for Mistral-3. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct { + inputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input; + postProcessedOutputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output; +} +type Ai_Cf_Google_Gemma_3_12B_It_Input = Ai_Cf_Google_Gemma_3_12B_It_Prompt | Ai_Cf_Google_Gemma_3_12B_It_Messages; +interface Ai_Cf_Google_Gemma_3_12B_It_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Google_Gemma_3_12B_It_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[]; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Google_Gemma_3_12B_It_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + /** + * The name of the tool to be called + */ + name?: string; + }[]; +}; +declare abstract class Base_Ai_Cf_Google_Gemma_3_12B_It { + inputs: Ai_Cf_Google_Gemma_3_12B_It_Input; + postProcessedOutputs: Ai_Cf_Google_Gemma_3_12B_It_Output; +} +type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input = Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch; +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch { + requests: (Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner)[]; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * JSON schema that should be fulfilled for the response. + */ + guided_json?: object; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role?: string; + /** + * The tool call id. If you don't know what to put here you can fall back to 000000001 + */ + tool_call_id?: string; + content?: string | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }[] | { + /** + * Type of the content provided + */ + type?: string; + text?: string; + image_url?: { + /** + * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted + */ + url?: string; + }; + }; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; + /** + * JSON schema that should be fufilled for the response. + */ + guided_json?: object; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output = { + /** + * The generated text response from the model + */ + response: string; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * An array of tool calls requests made during the response generation + */ + tool_calls?: { + /** + * The tool call id. + */ + id?: string; + /** + * Specifies the type of tool (e.g., 'function'). + */ + type?: string; + /** + * Details of the function tool. + */ + function?: { + /** + * The name of the tool to be called + */ + name?: string; + /** + * The arguments passed to be passed to the tool call request + */ + arguments?: object; + }; + }[]; +}; +declare abstract class Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct { + inputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input; + postProcessedOutputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output; +} +type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch; +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch { + requests: (Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1)[]; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1 { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response | string | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse; +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "chat.completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index?: number; + /** + * The message generated by the model + */ + message?: { + /** + * Role of the message author + */ + role: string; + /** + * The content of the message + */ + content: string; + /** + * Internal reasoning content (if available) + */ + reasoning_content?: string; + /** + * Tool calls made by the assistant + */ + tool_calls?: { + /** + * Unique identifier for the tool call + */ + id: string; + /** + * Type of tool call + */ + type: "function"; + function: { + /** + * Name of the function to call + */ + name: string; + /** + * JSON string of arguments for the function + */ + arguments: string; + }; + }[]; + }; + /** + * Reason why the model stopped generating + */ + finish_reason?: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "text_completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index: number; + /** + * The generated text completion + */ + text: string; + /** + * Reason why the model stopped generating + */ + finish_reason: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; +} +interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8 { + inputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output; +} +interface Ai_Cf_Deepgram_Nova_3_Input { + audio: { + body: object; + contentType: string; + }; + /** + * Sets how the model will interpret strings submitted to the custom_topic param. When strict, the model will only return topics submitted using the custom_topic param. When extended, the model will return its own detected topics in addition to those submitted using the custom_topic param. + */ + custom_topic_mode?: "extended" | "strict"; + /** + * Custom topics you want the model to detect within your input audio or text if present Submit up to 100 + */ + custom_topic?: string; + /** + * Sets how the model will interpret intents submitted to the custom_intent param. When strict, the model will only return intents submitted using the custom_intent param. When extended, the model will return its own detected intents in addition those submitted using the custom_intents param + */ + custom_intent_mode?: "extended" | "strict"; + /** + * Custom intents you want the model to detect within your input audio if present + */ + custom_intent?: string; + /** + * Identifies and extracts key entities from content in submitted audio + */ + detect_entities?: boolean; + /** + * Identifies the dominant language spoken in submitted audio + */ + detect_language?: boolean; + /** + * Recognize speaker changes. Each word in the transcript will be assigned a speaker number starting at 0 + */ + diarize?: boolean; + /** + * Identify and extract key entities from content in submitted audio + */ + dictation?: boolean; + /** + * Specify the expected encoding of your submitted audio + */ + encoding?: "linear16" | "flac" | "mulaw" | "amr-nb" | "amr-wb" | "opus" | "speex" | "g729"; + /** + * Arbitrary key-value pairs that are attached to the API response for usage in downstream processing + */ + extra?: string; + /** + * Filler Words can help transcribe interruptions in your audio, like 'uh' and 'um' + */ + filler_words?: boolean; + /** + * Key term prompting can boost or suppress specialized terminology and brands. + */ + keyterm?: string; + /** + * Keywords can boost or suppress specialized terminology and brands. + */ + keywords?: string; + /** + * The BCP-47 language tag that hints at the primary spoken language. Depending on the Model and API endpoint you choose only certain languages are available. + */ + language?: string; + /** + * Spoken measurements will be converted to their corresponding abbreviations. + */ + measurements?: boolean; + /** + * Opts out requests from the Deepgram Model Improvement Program. Refer to our Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip. + */ + mip_opt_out?: boolean; + /** + * Mode of operation for the model representing broad area of topic that will be talked about in the supplied audio + */ + mode?: "general" | "medical" | "finance"; + /** + * Transcribe each audio channel independently. + */ + multichannel?: boolean; + /** + * Numerals converts numbers from written format to numerical format. + */ + numerals?: boolean; + /** + * Splits audio into paragraphs to improve transcript readability. + */ + paragraphs?: boolean; + /** + * Profanity Filter looks for recognized profanity and converts it to the nearest recognized non-profane word or removes it from the transcript completely. + */ + profanity_filter?: boolean; + /** + * Add punctuation and capitalization to the transcript. + */ + punctuate?: boolean; + /** + * Redaction removes sensitive information from your transcripts. + */ + redact?: string; + /** + * Search for terms or phrases in submitted audio and replaces them. + */ + replace?: string; + /** + * Search for terms or phrases in submitted audio. + */ + search?: string; + /** + * Recognizes the sentiment throughout a transcript or text. + */ + sentiment?: boolean; + /** + * Apply formatting to transcript output. When set to true, additional formatting will be applied to transcripts to improve readability. + */ + smart_format?: boolean; + /** + * Detect topics throughout a transcript or text. + */ + topics?: boolean; + /** + * Segments speech into meaningful semantic units. + */ + utterances?: boolean; + /** + * Seconds to wait before detecting a pause between words in submitted audio. + */ + utt_split?: number; + /** + * The number of channels in the submitted audio + */ + channels?: number; + /** + * Specifies whether the streaming endpoint should provide ongoing transcription updates as more audio is received. When set to true, the endpoint sends continuous updates, meaning transcription results may evolve over time. Note: Supported only for webosockets. + */ + interim_results?: boolean; + /** + * Indicates how long model will wait to detect whether a speaker has finished speaking or pauses for a significant period of time. When set to a value, the streaming endpoint immediately finalizes the transcription for the processed time range and returns the transcript with a speech_final parameter set to true. Can also be set to false to disable endpointing + */ + endpointing?: string; + /** + * Indicates that speech has started. You'll begin receiving Speech Started messages upon speech starting. Note: Supported only for webosockets. + */ + vad_events?: boolean; + /** + * Indicates how long model will wait to send an UtteranceEnd message after a word has been transcribed. Use with interim_results. Note: Supported only for webosockets. + */ + utterance_end_ms?: boolean; +} +interface Ai_Cf_Deepgram_Nova_3_Output { + results?: { + channels?: { + alternatives?: { + confidence?: number; + transcript?: string; + words?: { + confidence?: number; + end?: number; + start?: number; + word?: string; + }[]; + }[]; + }[]; + summary?: { + result?: string; + short?: string; + }; + sentiments?: { + segments?: { + text?: string; + start_word?: number; + end_word?: number; + sentiment?: string; + sentiment_score?: number; + }[]; + average?: { + sentiment?: string; + sentiment_score?: number; + }; + }; + }; +} +declare abstract class Base_Ai_Cf_Deepgram_Nova_3 { + inputs: Ai_Cf_Deepgram_Nova_3_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Nova_3_Output; +} +interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input { + queries?: string | string[]; + /** + * Optional instruction for the task + */ + instruction?: string; + documents?: string | string[]; + text?: string | string[]; +} +interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output { + data?: number[][]; + shape?: number[]; +} +declare abstract class Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B { + inputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input; + postProcessedOutputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output; +} +type Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input = { + /** + * readable stream with audio data and content-type specified for that data + */ + audio: { + body: object; + contentType: string; + }; + /** + * type of data PCM data that's sent to the inference server as raw array + */ + dtype?: "uint8" | "float32" | "float64"; +} | { + /** + * base64 encoded audio data + */ + audio: string; + /** + * type of data PCM data that's sent to the inference server as raw array + */ + dtype?: "uint8" | "float32" | "float64"; +}; +interface Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output { + /** + * if true, end-of-turn was detected + */ + is_complete?: boolean; + /** + * probability of the end-of-turn detection + */ + probability?: number; +} +declare abstract class Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2 { + inputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input; + postProcessedOutputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output; +} +declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_120B { + inputs: ResponsesInput; + postProcessedOutputs: ResponsesOutput; +} +declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_20B { + inputs: ResponsesInput; + postProcessedOutputs: ResponsesOutput; +} +interface Ai_Cf_Leonardo_Phoenix_1_0_Input { + /** + * A text description of the image you want to generate. + */ + prompt: string; + /** + * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt + */ + guidance?: number; + /** + * Random seed for reproducibility of the image generation + */ + seed?: number; + /** + * The height of the generated image in pixels + */ + height?: number; + /** + * The width of the generated image in pixels + */ + width?: number; + /** + * The number of diffusion steps; higher values can improve quality but take longer + */ + num_steps?: number; + /** + * Specify what to exclude from the generated images + */ + negative_prompt?: string; +} +/** + * The generated image in JPEG format + */ +type Ai_Cf_Leonardo_Phoenix_1_0_Output = string; +declare abstract class Base_Ai_Cf_Leonardo_Phoenix_1_0 { + inputs: Ai_Cf_Leonardo_Phoenix_1_0_Input; + postProcessedOutputs: Ai_Cf_Leonardo_Phoenix_1_0_Output; +} +interface Ai_Cf_Leonardo_Lucid_Origin_Input { + /** + * A text description of the image you want to generate. + */ + prompt: string; + /** + * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt + */ + guidance?: number; + /** + * Random seed for reproducibility of the image generation + */ + seed?: number; + /** + * The height of the generated image in pixels + */ + height?: number; + /** + * The width of the generated image in pixels + */ + width?: number; + /** + * The number of diffusion steps; higher values can improve quality but take longer + */ + num_steps?: number; + /** + * The number of diffusion steps; higher values can improve quality but take longer + */ + steps?: number; +} +interface Ai_Cf_Leonardo_Lucid_Origin_Output { + /** + * The generated image in Base64 format. + */ + image?: string; +} +declare abstract class Base_Ai_Cf_Leonardo_Lucid_Origin { + inputs: Ai_Cf_Leonardo_Lucid_Origin_Input; + postProcessedOutputs: Ai_Cf_Leonardo_Lucid_Origin_Output; +} +interface Ai_Cf_Deepgram_Aura_1_Input { + /** + * Speaker used to produce the audio. + */ + speaker?: "angus" | "asteria" | "arcas" | "orion" | "orpheus" | "athena" | "luna" | "zeus" | "perseus" | "helios" | "hera" | "stella"; + /** + * Encoding of the output audio. + */ + encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; + /** + * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. + */ + container?: "none" | "wav" | "ogg"; + /** + * The text content to be converted to speech + */ + text: string; + /** + * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable + */ + sample_rate?: number; + /** + * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. + */ + bit_rate?: number; +} +/** + * The generated audio in MP3 format + */ +type Ai_Cf_Deepgram_Aura_1_Output = string; +declare abstract class Base_Ai_Cf_Deepgram_Aura_1 { + inputs: Ai_Cf_Deepgram_Aura_1_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Aura_1_Output; +} +interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input { + /** + * Input text to translate. Can be a single string or a list of strings. + */ + text: string | string[]; + /** + * Target langauge to translate to + */ + target_language: "asm_Beng" | "awa_Deva" | "ben_Beng" | "bho_Deva" | "brx_Deva" | "doi_Deva" | "eng_Latn" | "gom_Deva" | "gon_Deva" | "guj_Gujr" | "hin_Deva" | "hne_Deva" | "kan_Knda" | "kas_Arab" | "kas_Deva" | "kha_Latn" | "lus_Latn" | "mag_Deva" | "mai_Deva" | "mal_Mlym" | "mar_Deva" | "mni_Beng" | "mni_Mtei" | "npi_Deva" | "ory_Orya" | "pan_Guru" | "san_Deva" | "sat_Olck" | "snd_Arab" | "snd_Deva" | "tam_Taml" | "tel_Telu" | "urd_Arab" | "unr_Deva"; +} +interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output { + /** + * Translated texts + */ + translations: string[]; +} +declare abstract class Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B { + inputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input; + postProcessedOutputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output; +} +type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch; +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch { + requests: (Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1)[]; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 { + /** + * The input text prompt for the model to generate a response. + */ + prompt: string; + /** + * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. + */ + lora?: string; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1 { + /** + * An array of message objects representing the conversation history. + */ + messages: { + /** + * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). + */ + role: string; + /** + * The content of the message as a string. + */ + content: string; + }[]; + functions?: { + name: string; + code: string; + }[]; + /** + * A list of tools available for the assistant to use. + */ + tools?: ({ + /** + * The name of the tool. More descriptive the better. + */ + name: string; + /** + * A brief description of what the tool does. + */ + description: string; + /** + * Schema defining the parameters accepted by the tool. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + } | { + /** + * Specifies the type of tool (e.g., 'function'). + */ + type: string; + /** + * Details of the function tool. + */ + function: { + /** + * The name of the function. + */ + name: string; + /** + * A brief description of what the function does. + */ + description: string; + /** + * Schema defining the parameters accepted by the function. + */ + parameters: { + /** + * The type of the parameters object (usually 'object'). + */ + type: string; + /** + * List of required parameter names. + */ + required?: string[]; + /** + * Definitions of each parameter. + */ + properties: { + [k: string]: { + /** + * The data type of the parameter. + */ + type: string; + /** + * A description of the expected parameter. + */ + description: string; + }; + }; + }; + }; + })[]; + response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3; + /** + * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. + */ + raw?: boolean; + /** + * If true, the response will be streamed back incrementally using SSE, Server Sent Events. + */ + stream?: boolean; + /** + * The maximum number of tokens to generate in the response. + */ + max_tokens?: number; + /** + * Controls the randomness of the output; higher values produce more random results. + */ + temperature?: number; + /** + * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. + */ + top_p?: number; + /** + * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. + */ + top_k?: number; + /** + * Random seed for reproducibility of the generation. + */ + seed?: number; + /** + * Penalty for repeated tokens; higher values discourage repetition. + */ + repetition_penalty?: number; + /** + * Decreases the likelihood of the model repeating the same lines verbatim. + */ + frequency_penalty?: number; + /** + * Increases the likelihood of the model introducing new topics. + */ + presence_penalty?: number; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3 { + type?: "json_object" | "json_schema"; + json_schema?: unknown; +} +type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response | string | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse; +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "chat.completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index?: number; + /** + * The message generated by the model + */ + message?: { + /** + * Role of the message author + */ + role: string; + /** + * The content of the message + */ + content: string; + /** + * Internal reasoning content (if available) + */ + reasoning_content?: string; + /** + * Tool calls made by the assistant + */ + tool_calls?: { + /** + * Unique identifier for the tool call + */ + id: string; + /** + * Type of tool call + */ + type: "function"; + function: { + /** + * Name of the function to call + */ + name: string; + /** + * JSON string of arguments for the function + */ + arguments: string; + }; + }[]; + }; + /** + * Reason why the model stopped generating + */ + finish_reason?: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response { + /** + * Unique identifier for the completion + */ + id?: string; + /** + * Object type identifier + */ + object?: "text_completion"; + /** + * Unix timestamp of when the completion was created + */ + created?: number; + /** + * Model used for the completion + */ + model?: string; + /** + * List of completion choices + */ + choices?: { + /** + * Index of the choice in the list + */ + index: number; + /** + * The generated text completion + */ + text: string; + /** + * Reason why the model stopped generating + */ + finish_reason: string; + /** + * Stop reason (may be null) + */ + stop_reason?: string | null; + /** + * Log probabilities (if requested) + */ + logprobs?: {} | null; + /** + * Log probabilities for the prompt (if requested) + */ + prompt_logprobs?: {} | null; + }[]; + /** + * Usage statistics for the inference request + */ + usage?: { + /** + * Total number of tokens in input + */ + prompt_tokens?: number; + /** + * Total number of tokens in output + */ + completion_tokens?: number; + /** + * Total number of input and output tokens + */ + total_tokens?: number; + }; +} +interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse { + /** + * The async request id that can be used to obtain the results. + */ + request_id?: string; +} +declare abstract class Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It { + inputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input; + postProcessedOutputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output; +} +interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Input { + /** + * Input text to embed. Can be a single string or a list of strings. + */ + text: string | string[]; +} +interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Output { + /** + * Embedding vectors, where each vector is a list of floats. + */ + data: number[][]; + /** + * Shape of the embedding data as [number_of_embeddings, embedding_dimension]. + * + * @minItems 2 + * @maxItems 2 + */ + shape: [ + number, + number + ]; +} +declare abstract class Base_Ai_Cf_Pfnet_Plamo_Embedding_1B { + inputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Input; + postProcessedOutputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Output; +} +interface Ai_Cf_Deepgram_Flux_Input { + /** + * Encoding of the audio stream. Currently only supports raw signed little-endian 16-bit PCM. + */ + encoding: "linear16"; + /** + * Sample rate of the audio stream in Hz. + */ + sample_rate: string; + /** + * End-of-turn confidence required to fire an eager end-of-turn event. When set, enables EagerEndOfTurn and TurnResumed events. Valid Values 0.3 - 0.9. + */ + eager_eot_threshold?: string; + /** + * End-of-turn confidence required to finish a turn. Valid Values 0.5 - 0.9. + */ + eot_threshold?: string; + /** + * A turn will be finished when this much time has passed after speech, regardless of EOT confidence. + */ + eot_timeout_ms?: string; + /** + * Keyterm prompting can improve recognition of specialized terminology. Pass multiple keyterm query parameters to boost multiple keyterms. + */ + keyterm?: string; + /** + * Opts out requests from the Deepgram Model Improvement Program. Refer to Deepgram Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip + */ + mip_opt_out?: "true" | "false"; + /** + * Label your requests for the purpose of identification during usage reporting + */ + tag?: string; +} +/** + * Output will be returned as websocket messages. + */ +interface Ai_Cf_Deepgram_Flux_Output { + /** + * The unique identifier of the request (uuid) + */ + request_id?: string; + /** + * Starts at 0 and increments for each message the server sends to the client. + */ + sequence_id?: number; + /** + * The type of event being reported. + */ + event?: "Update" | "StartOfTurn" | "EagerEndOfTurn" | "TurnResumed" | "EndOfTurn"; + /** + * The index of the current turn + */ + turn_index?: number; + /** + * Start time in seconds of the audio range that was transcribed + */ + audio_window_start?: number; + /** + * End time in seconds of the audio range that was transcribed + */ + audio_window_end?: number; + /** + * Text that was said over the course of the current turn + */ + transcript?: string; + /** + * The words in the transcript + */ + words?: { + /** + * The individual punctuated, properly-cased word from the transcript + */ + word: string; + /** + * Confidence that this word was transcribed correctly + */ + confidence: number; + }[]; + /** + * Confidence that no more speech is coming in this turn + */ + end_of_turn_confidence?: number; +} +declare abstract class Base_Ai_Cf_Deepgram_Flux { + inputs: Ai_Cf_Deepgram_Flux_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Flux_Output; +} +interface Ai_Cf_Deepgram_Aura_2_En_Input { + /** + * Speaker used to produce the audio. + */ + speaker?: "amalthea" | "andromeda" | "apollo" | "arcas" | "aries" | "asteria" | "athena" | "atlas" | "aurora" | "callista" | "cora" | "cordelia" | "delia" | "draco" | "electra" | "harmonia" | "helena" | "hera" | "hermes" | "hyperion" | "iris" | "janus" | "juno" | "jupiter" | "luna" | "mars" | "minerva" | "neptune" | "odysseus" | "ophelia" | "orion" | "orpheus" | "pandora" | "phoebe" | "pluto" | "saturn" | "thalia" | "theia" | "vesta" | "zeus"; + /** + * Encoding of the output audio. + */ + encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; + /** + * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. + */ + container?: "none" | "wav" | "ogg"; + /** + * The text content to be converted to speech + */ + text: string; + /** + * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable + */ + sample_rate?: number; + /** + * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. + */ + bit_rate?: number; +} +/** + * The generated audio in MP3 format + */ +type Ai_Cf_Deepgram_Aura_2_En_Output = string; +declare abstract class Base_Ai_Cf_Deepgram_Aura_2_En { + inputs: Ai_Cf_Deepgram_Aura_2_En_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_En_Output; +} +interface Ai_Cf_Deepgram_Aura_2_Es_Input { + /** + * Speaker used to produce the audio. + */ + speaker?: "sirio" | "nestor" | "carina" | "celeste" | "alvaro" | "diana" | "aquila" | "selena" | "estrella" | "javier"; + /** + * Encoding of the output audio. + */ + encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; + /** + * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. + */ + container?: "none" | "wav" | "ogg"; + /** + * The text content to be converted to speech + */ + text: string; + /** + * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable + */ + sample_rate?: number; + /** + * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. + */ + bit_rate?: number; +} +/** + * The generated audio in MP3 format + */ +type Ai_Cf_Deepgram_Aura_2_Es_Output = string; +declare abstract class Base_Ai_Cf_Deepgram_Aura_2_Es { + inputs: Ai_Cf_Deepgram_Aura_2_Es_Input; + postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_Es_Output; +} +interface AiModels { + "@cf/huggingface/distilbert-sst-2-int8": BaseAiTextClassification; + "@cf/stabilityai/stable-diffusion-xl-base-1.0": BaseAiTextToImage; + "@cf/runwayml/stable-diffusion-v1-5-inpainting": BaseAiTextToImage; + "@cf/runwayml/stable-diffusion-v1-5-img2img": BaseAiTextToImage; + "@cf/lykon/dreamshaper-8-lcm": BaseAiTextToImage; + "@cf/bytedance/stable-diffusion-xl-lightning": BaseAiTextToImage; + "@cf/myshell-ai/melotts": BaseAiTextToSpeech; + "@cf/google/embeddinggemma-300m": BaseAiTextEmbeddings; + "@cf/microsoft/resnet-50": BaseAiImageClassification; + "@cf/meta/llama-2-7b-chat-int8": BaseAiTextGeneration; + "@cf/mistral/mistral-7b-instruct-v0.1": BaseAiTextGeneration; + "@cf/meta/llama-2-7b-chat-fp16": BaseAiTextGeneration; + "@hf/thebloke/llama-2-13b-chat-awq": BaseAiTextGeneration; + "@hf/thebloke/mistral-7b-instruct-v0.1-awq": BaseAiTextGeneration; + "@hf/thebloke/zephyr-7b-beta-awq": BaseAiTextGeneration; + "@hf/thebloke/openhermes-2.5-mistral-7b-awq": BaseAiTextGeneration; + "@hf/thebloke/neural-chat-7b-v3-1-awq": BaseAiTextGeneration; + "@hf/thebloke/llamaguard-7b-awq": BaseAiTextGeneration; + "@hf/thebloke/deepseek-coder-6.7b-base-awq": BaseAiTextGeneration; + "@hf/thebloke/deepseek-coder-6.7b-instruct-awq": BaseAiTextGeneration; + "@cf/deepseek-ai/deepseek-math-7b-instruct": BaseAiTextGeneration; + "@cf/defog/sqlcoder-7b-2": BaseAiTextGeneration; + "@cf/openchat/openchat-3.5-0106": BaseAiTextGeneration; + "@cf/tiiuae/falcon-7b-instruct": BaseAiTextGeneration; + "@cf/thebloke/discolm-german-7b-v1-awq": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-0.5b-chat": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-7b-chat-awq": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-14b-chat-awq": BaseAiTextGeneration; + "@cf/tinyllama/tinyllama-1.1b-chat-v1.0": BaseAiTextGeneration; + "@cf/microsoft/phi-2": BaseAiTextGeneration; + "@cf/qwen/qwen1.5-1.8b-chat": BaseAiTextGeneration; + "@cf/mistral/mistral-7b-instruct-v0.2-lora": BaseAiTextGeneration; + "@hf/nousresearch/hermes-2-pro-mistral-7b": BaseAiTextGeneration; + "@hf/nexusflow/starling-lm-7b-beta": BaseAiTextGeneration; + "@hf/google/gemma-7b-it": BaseAiTextGeneration; + "@cf/meta-llama/llama-2-7b-chat-hf-lora": BaseAiTextGeneration; + "@cf/google/gemma-2b-it-lora": BaseAiTextGeneration; + "@cf/google/gemma-7b-it-lora": BaseAiTextGeneration; + "@hf/mistral/mistral-7b-instruct-v0.2": BaseAiTextGeneration; + "@cf/meta/llama-3-8b-instruct": BaseAiTextGeneration; + "@cf/fblgit/una-cybertron-7b-v2-bf16": BaseAiTextGeneration; + "@cf/meta/llama-3-8b-instruct-awq": BaseAiTextGeneration; + "@cf/meta/llama-3.1-8b-instruct-fp8": BaseAiTextGeneration; + "@cf/meta/llama-3.1-8b-instruct-awq": BaseAiTextGeneration; + "@cf/meta/llama-3.2-3b-instruct": BaseAiTextGeneration; + "@cf/meta/llama-3.2-1b-instruct": BaseAiTextGeneration; + "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": BaseAiTextGeneration; + "@cf/ibm-granite/granite-4.0-h-micro": BaseAiTextGeneration; + "@cf/facebook/bart-large-cnn": BaseAiSummarization; + "@cf/llava-hf/llava-1.5-7b-hf": BaseAiImageToText; + "@cf/baai/bge-base-en-v1.5": Base_Ai_Cf_Baai_Bge_Base_En_V1_5; + "@cf/openai/whisper": Base_Ai_Cf_Openai_Whisper; + "@cf/meta/m2m100-1.2b": Base_Ai_Cf_Meta_M2M100_1_2B; + "@cf/baai/bge-small-en-v1.5": Base_Ai_Cf_Baai_Bge_Small_En_V1_5; + "@cf/baai/bge-large-en-v1.5": Base_Ai_Cf_Baai_Bge_Large_En_V1_5; + "@cf/unum/uform-gen2-qwen-500m": Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M; + "@cf/openai/whisper-tiny-en": Base_Ai_Cf_Openai_Whisper_Tiny_En; + "@cf/openai/whisper-large-v3-turbo": Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo; + "@cf/baai/bge-m3": Base_Ai_Cf_Baai_Bge_M3; + "@cf/black-forest-labs/flux-1-schnell": Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell; + "@cf/meta/llama-3.2-11b-vision-instruct": Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct; + "@cf/meta/llama-3.3-70b-instruct-fp8-fast": Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast; + "@cf/meta/llama-guard-3-8b": Base_Ai_Cf_Meta_Llama_Guard_3_8B; + "@cf/baai/bge-reranker-base": Base_Ai_Cf_Baai_Bge_Reranker_Base; + "@cf/qwen/qwen2.5-coder-32b-instruct": Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct; + "@cf/qwen/qwq-32b": Base_Ai_Cf_Qwen_Qwq_32B; + "@cf/mistralai/mistral-small-3.1-24b-instruct": Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct; + "@cf/google/gemma-3-12b-it": Base_Ai_Cf_Google_Gemma_3_12B_It; + "@cf/meta/llama-4-scout-17b-16e-instruct": Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct; + "@cf/qwen/qwen3-30b-a3b-fp8": Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8; + "@cf/deepgram/nova-3": Base_Ai_Cf_Deepgram_Nova_3; + "@cf/qwen/qwen3-embedding-0.6b": Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B; + "@cf/pipecat-ai/smart-turn-v2": Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2; + "@cf/openai/gpt-oss-120b": Base_Ai_Cf_Openai_Gpt_Oss_120B; + "@cf/openai/gpt-oss-20b": Base_Ai_Cf_Openai_Gpt_Oss_20B; + "@cf/leonardo/phoenix-1.0": Base_Ai_Cf_Leonardo_Phoenix_1_0; + "@cf/leonardo/lucid-origin": Base_Ai_Cf_Leonardo_Lucid_Origin; + "@cf/deepgram/aura-1": Base_Ai_Cf_Deepgram_Aura_1; + "@cf/ai4bharat/indictrans2-en-indic-1B": Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B; + "@cf/aisingapore/gemma-sea-lion-v4-27b-it": Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It; + "@cf/pfnet/plamo-embedding-1b": Base_Ai_Cf_Pfnet_Plamo_Embedding_1B; + "@cf/deepgram/flux": Base_Ai_Cf_Deepgram_Flux; + "@cf/deepgram/aura-2-en": Base_Ai_Cf_Deepgram_Aura_2_En; + "@cf/deepgram/aura-2-es": Base_Ai_Cf_Deepgram_Aura_2_Es; +} +type AiOptions = { + /** + * Send requests as an asynchronous batch job, only works for supported models + * https://developers.cloudflare.com/workers-ai/features/batch-api + */ + queueRequest?: boolean; + /** + * Establish websocket connections, only works for supported models + */ + websocket?: boolean; + /** + * Tag your requests to group and view them in Cloudflare dashboard. + * + * Rules: + * Tags must only contain letters, numbers, and the symbols: : - . / @ + * Each tag can have maximum 50 characters. + * Maximum 5 tags are allowed each request. + * Duplicate tags will removed. + */ + tags?: string[]; + gateway?: GatewayOptions; + returnRawResponse?: boolean; + prefix?: string; + extraHeaders?: object; +}; +type AiModelsSearchParams = { + author?: string; + hide_experimental?: boolean; + page?: number; + per_page?: number; + search?: string; + source?: number; + task?: string; +}; +type AiModelsSearchObject = { + id: string; + source: number; + name: string; + description: string; + task: { + id: string; + name: string; + description: string; + }; + tags: string[]; + properties: { + property_id: string; + value: string; + }[]; +}; +interface InferenceUpstreamError extends Error { +} +interface AiInternalError extends Error { +} +type AiModelListType = Record; +declare abstract class Ai { + aiGatewayLogId: string | null; + gateway(gatewayId: string): AiGateway; + autorag(autoragId: string): AutoRAG; + run(model: Name, inputs: InputOptions, options?: Options): Promise; + models(params?: AiModelsSearchParams): Promise; + toMarkdown(): ToMarkdownService; + toMarkdown(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; + toMarkdown(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; +} +type GatewayRetries = { + maxAttempts?: 1 | 2 | 3 | 4 | 5; + retryDelayMs?: number; + backoff?: 'constant' | 'linear' | 'exponential'; +}; +type GatewayOptions = { + id: string; + cacheKey?: string; + cacheTtl?: number; + skipCache?: boolean; + metadata?: Record; + collectLog?: boolean; + eventId?: string; + requestTimeoutMs?: number; + retries?: GatewayRetries; +}; +type UniversalGatewayOptions = Exclude & { + /** + ** @deprecated + */ + id?: string; +}; +type AiGatewayPatchLog = { + score?: number | null; + feedback?: -1 | 1 | null; + metadata?: Record | null; +}; +type AiGatewayLog = { + id: string; + provider: string; + model: string; + model_type?: string; + path: string; + duration: number; + request_type?: string; + request_content_type?: string; + status_code: number; + response_content_type?: string; + success: boolean; + cached: boolean; + tokens_in?: number; + tokens_out?: number; + metadata?: Record; + step?: number; + cost?: number; + custom_cost?: boolean; + request_size: number; + request_head?: string; + request_head_complete: boolean; + response_size: number; + response_head?: string; + response_head_complete: boolean; + created_at: Date; +}; +type AIGatewayProviders = 'workers-ai' | 'anthropic' | 'aws-bedrock' | 'azure-openai' | 'google-vertex-ai' | 'huggingface' | 'openai' | 'perplexity-ai' | 'replicate' | 'groq' | 'cohere' | 'google-ai-studio' | 'mistral' | 'grok' | 'openrouter' | 'deepseek' | 'cerebras' | 'cartesia' | 'elevenlabs' | 'adobe-firefly'; +type AIGatewayHeaders = { + 'cf-aig-metadata': Record | string; + 'cf-aig-custom-cost': { + per_token_in?: number; + per_token_out?: number; + } | { + total_cost?: number; + } | string; + 'cf-aig-cache-ttl': number | string; + 'cf-aig-skip-cache': boolean | string; + 'cf-aig-cache-key': string; + 'cf-aig-event-id': string; + 'cf-aig-request-timeout': number | string; + 'cf-aig-max-attempts': number | string; + 'cf-aig-retry-delay': number | string; + 'cf-aig-backoff': string; + 'cf-aig-collect-log': boolean | string; + Authorization: string; + 'Content-Type': string; + [key: string]: string | number | boolean | object; +}; +type AIGatewayUniversalRequest = { + provider: AIGatewayProviders | string; // eslint-disable-line + endpoint: string; + headers: Partial; + query: unknown; +}; +interface AiGatewayInternalError extends Error { +} +interface AiGatewayLogNotFound extends Error { +} +declare abstract class AiGateway { + patchLog(logId: string, data: AiGatewayPatchLog): Promise; + getLog(logId: string): Promise; + run(data: AIGatewayUniversalRequest | AIGatewayUniversalRequest[], options?: { + gateway?: UniversalGatewayOptions; + extraHeaders?: object; + }): Promise; + getUrl(provider?: AIGatewayProviders | string): Promise; // eslint-disable-line +} +interface AutoRAGInternalError extends Error { +} +interface AutoRAGNotFoundError extends Error { +} +interface AutoRAGUnauthorizedError extends Error { +} +interface AutoRAGNameNotSetError extends Error { +} +type ComparisonFilter = { + key: string; + type: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte'; + value: string | number | boolean; +}; +type CompoundFilter = { + type: 'and' | 'or'; + filters: ComparisonFilter[]; +}; +type AutoRagSearchRequest = { + query: string; + filters?: CompoundFilter | ComparisonFilter; + max_num_results?: number; + ranking_options?: { + ranker?: string; + score_threshold?: number; + }; + reranking?: { + enabled?: boolean; + model?: string; + }; + rewrite_query?: boolean; +}; +type AutoRagAiSearchRequest = AutoRagSearchRequest & { + stream?: boolean; + system_prompt?: string; +}; +type AutoRagAiSearchRequestStreaming = Omit & { + stream: true; +}; +type AutoRagSearchResponse = { + object: 'vector_store.search_results.page'; + search_query: string; + data: { + file_id: string; + filename: string; + score: number; + attributes: Record; + content: { + type: 'text'; + text: string; + }[]; + }[]; + has_more: boolean; + next_page: string | null; +}; +type AutoRagListResponse = { + id: string; + enable: boolean; + type: string; + source: string; + vectorize_name: string; + paused: boolean; + status: string; +}[]; +type AutoRagAiSearchResponse = AutoRagSearchResponse & { + response: string; +}; +declare abstract class AutoRAG { + list(): Promise; + search(params: AutoRagSearchRequest): Promise; + aiSearch(params: AutoRagAiSearchRequestStreaming): Promise; + aiSearch(params: AutoRagAiSearchRequest): Promise; + aiSearch(params: AutoRagAiSearchRequest): Promise; +} +interface BasicImageTransformations { + /** + * Maximum width in image pixels. The value must be an integer. + */ + width?: number; + /** + * Maximum height in image pixels. The value must be an integer. + */ + height?: number; + /** + * Resizing mode as a string. It affects interpretation of width and height + * options: + * - scale-down: Similar to contain, but the image is never enlarged. If + * the image is larger than given width or height, it will be resized. + * Otherwise its original size will be kept. + * - contain: Resizes to maximum size that fits within the given width and + * height. If only a single dimension is given (e.g. only width), the + * image will be shrunk or enlarged to exactly match that dimension. + * Aspect ratio is always preserved. + * - cover: Resizes (shrinks or enlarges) to fill the entire area of width + * and height. If the image has an aspect ratio different from the ratio + * of width and height, it will be cropped to fit. + * - crop: The image will be shrunk and cropped to fit within the area + * specified by width and height. The image will not be enlarged. For images + * smaller than the given dimensions it's the same as scale-down. For + * images larger than the given dimensions, it's the same as cover. + * See also trim. + * - pad: Resizes to the maximum size that fits within the given width and + * height, and then fills the remaining area with a background color + * (white by default). Use of this mode is not recommended, as the same + * effect can be more efficiently achieved with the contain mode and the + * CSS object-fit: contain property. + * - squeeze: Stretches and deforms to the width and height given, even if it + * breaks aspect ratio + */ + fit?: "scale-down" | "contain" | "cover" | "crop" | "pad" | "squeeze"; + /** + * Image segmentation using artificial intelligence models. Sets pixels not + * within selected segment area to transparent e.g "foreground" sets every + * background pixel as transparent. + */ + segment?: "foreground"; + /** + * When cropping with fit: "cover", this defines the side or point that should + * be left uncropped. The value is either a string + * "left", "right", "top", "bottom", "auto", or "center" (the default), + * or an object {x, y} containing focal point coordinates in the original + * image expressed as fractions ranging from 0.0 (top or left) to 1.0 + * (bottom or right), 0.5 being the center. {fit: "cover", gravity: "top"} will + * crop bottom or left and right sides as necessary, but won’t crop anything + * from the top. {fit: "cover", gravity: {x:0.5, y:0.2}} will crop each side to + * preserve as much as possible around a point at 20% of the height of the + * source image. + */ + gravity?: 'face' | 'left' | 'right' | 'top' | 'bottom' | 'center' | 'auto' | 'entropy' | BasicImageTransformationsGravityCoordinates; + /** + * Background color to add underneath the image. Applies only to images with + * transparency (such as PNG). Accepts any CSS color (#RRGGBB, rgba(…), + * hsl(…), etc.) + */ + background?: string; + /** + * Number of degrees (90, 180, 270) to rotate the image by. width and height + * options refer to axes after rotation. + */ + rotate?: 0 | 90 | 180 | 270 | 360; +} +interface BasicImageTransformationsGravityCoordinates { + x?: number; + y?: number; + mode?: 'remainder' | 'box-center'; +} +/** + * In addition to the properties you can set in the RequestInit dict + * that you pass as an argument to the Request constructor, you can + * set certain properties of a `cf` object to control how Cloudflare + * features are applied to that new Request. + * + * Note: Currently, these properties cannot be tested in the + * playground. + */ +interface RequestInitCfProperties extends Record { + cacheEverything?: boolean; + /** + * A request's cache key is what determines if two requests are + * "the same" for caching purposes. If a request has the same cache key + * as some previous request, then we can serve the same cached response for + * both. (e.g. 'some-key') + * + * Only available for Enterprise customers. + */ + cacheKey?: string; + /** + * This allows you to append additional Cache-Tag response headers + * to the origin response without modifications to the origin server. + * This will allow for greater control over the Purge by Cache Tag feature + * utilizing changes only in the Workers process. + * + * Only available for Enterprise customers. + */ + cacheTags?: string[]; + /** + * Force response to be cached for a given number of seconds. (e.g. 300) + */ + cacheTtl?: number; + /** + * Force response to be cached for a given number of seconds based on the Origin status code. + * (e.g. { '200-299': 86400, '404': 1, '500-599': 0 }) + */ + cacheTtlByStatus?: Record; + scrapeShield?: boolean; + apps?: boolean; + image?: RequestInitCfPropertiesImage; + minify?: RequestInitCfPropertiesImageMinify; + mirage?: boolean; + polish?: "lossy" | "lossless" | "off"; + r2?: RequestInitCfPropertiesR2; + /** + * Redirects the request to an alternate origin server. You can use this, + * for example, to implement load balancing across several origins. + * (e.g.us-east.example.com) + * + * Note - For security reasons, the hostname set in resolveOverride must + * be proxied on the same Cloudflare zone of the incoming request. + * Otherwise, the setting is ignored. CNAME hosts are allowed, so to + * resolve to a host under a different domain or a DNS only domain first + * declare a CNAME record within your own zone’s DNS mapping to the + * external hostname, set proxy on Cloudflare, then set resolveOverride + * to point to that CNAME record. + */ + resolveOverride?: string; +} +interface RequestInitCfPropertiesImageDraw extends BasicImageTransformations { + /** + * Absolute URL of the image file to use for the drawing. It can be any of + * the supported file formats. For drawing of watermarks or non-rectangular + * overlays we recommend using PNG or WebP images. + */ + url: string; + /** + * Floating-point number between 0 (transparent) and 1 (opaque). + * For example, opacity: 0.5 makes overlay semitransparent. + */ + opacity?: number; + /** + * - If set to true, the overlay image will be tiled to cover the entire + * area. This is useful for stock-photo-like watermarks. + * - If set to "x", the overlay image will be tiled horizontally only + * (form a line). + * - If set to "y", the overlay image will be tiled vertically only + * (form a line). + */ + repeat?: true | "x" | "y"; + /** + * Position of the overlay image relative to a given edge. Each property is + * an offset in pixels. 0 aligns exactly to the edge. For example, left: 10 + * positions left side of the overlay 10 pixels from the left edge of the + * image it's drawn over. bottom: 0 aligns bottom of the overlay with bottom + * of the background image. + * + * Setting both left & right, or both top & bottom is an error. + * + * If no position is specified, the image will be centered. + */ + top?: number; + left?: number; + bottom?: number; + right?: number; +} +interface RequestInitCfPropertiesImage extends BasicImageTransformations { + /** + * Device Pixel Ratio. Default 1. Multiplier for width/height that makes it + * easier to specify higher-DPI sizes in . + */ + dpr?: number; + /** + * Allows you to trim your image. Takes dpr into account and is performed before + * resizing or rotation. + * + * It can be used as: + * - left, top, right, bottom - it will specify the number of pixels to cut + * off each side + * - width, height - the width/height you'd like to end up with - can be used + * in combination with the properties above + * - border - this will automatically trim the surroundings of an image based on + * it's color. It consists of three properties: + * - color: rgb or hex representation of the color you wish to trim (todo: verify the rgba bit) + * - tolerance: difference from color to treat as color + * - keep: the number of pixels of border to keep + */ + trim?: "border" | { + top?: number; + bottom?: number; + left?: number; + right?: number; + width?: number; + height?: number; + border?: boolean | { + color?: string; + tolerance?: number; + keep?: number; + }; + }; + /** + * Quality setting from 1-100 (useful values are in 60-90 range). Lower values + * make images look worse, but load faster. The default is 85. It applies only + * to JPEG and WebP images. It doesn’t have any effect on PNG. + */ + quality?: number | "low" | "medium-low" | "medium-high" | "high"; + /** + * Output format to generate. It can be: + * - avif: generate images in AVIF format. + * - webp: generate images in Google WebP format. Set quality to 100 to get + * the WebP-lossless format. + * - json: instead of generating an image, outputs information about the + * image, in JSON format. The JSON object will contain image size + * (before and after resizing), source image’s MIME type, file size, etc. + * - jpeg: generate images in JPEG format. + * - png: generate images in PNG format. + */ + format?: "avif" | "webp" | "json" | "jpeg" | "png" | "baseline-jpeg" | "png-force" | "svg"; + /** + * Whether to preserve animation frames from input files. Default is true. + * Setting it to false reduces animations to still images. This setting is + * recommended when enlarging images or processing arbitrary user content, + * because large GIF animations can weigh tens or even hundreds of megabytes. + * It is also useful to set anim:false when using format:"json" to get the + * response quicker without the number of frames. + */ + anim?: boolean; + /** + * What EXIF data should be preserved in the output image. Note that EXIF + * rotation and embedded color profiles are always applied ("baked in" into + * the image), and aren't affected by this option. Note that if the Polish + * feature is enabled, all metadata may have been removed already and this + * option may have no effect. + * - keep: Preserve most of EXIF metadata, including GPS location if there's + * any. + * - copyright: Only keep the copyright tag, and discard everything else. + * This is the default behavior for JPEG files. + * - none: Discard all invisible EXIF metadata. Currently WebP and PNG + * output formats always discard metadata. + */ + metadata?: "keep" | "copyright" | "none"; + /** + * Strength of sharpening filter to apply to the image. Floating-point + * number between 0 (no sharpening, default) and 10 (maximum). 1.0 is a + * recommended value for downscaled images. + */ + sharpen?: number; + /** + * Radius of a blur filter (approximate gaussian). Maximum supported radius + * is 250. + */ + blur?: number; + /** + * Overlays are drawn in the order they appear in the array (last array + * entry is the topmost layer). + */ + draw?: RequestInitCfPropertiesImageDraw[]; + /** + * Fetching image from authenticated origin. Setting this property will + * pass authentication headers (Authorization, Cookie, etc.) through to + * the origin. + */ + "origin-auth"?: "share-publicly"; + /** + * Adds a border around the image. The border is added after resizing. Border + * width takes dpr into account, and can be specified either using a single + * width property, or individually for each side. + */ + border?: { + color: string; + width: number; + } | { + color: string; + top: number; + right: number; + bottom: number; + left: number; + }; + /** + * Increase brightness by a factor. A value of 1.0 equals no change, a value + * of 0.5 equals half brightness, and a value of 2.0 equals twice as bright. + * 0 is ignored. + */ + brightness?: number; + /** + * Increase contrast by a factor. A value of 1.0 equals no change, a value of + * 0.5 equals low contrast, and a value of 2.0 equals high contrast. 0 is + * ignored. + */ + contrast?: number; + /** + * Increase exposure by a factor. A value of 1.0 equals no change, a value of + * 0.5 darkens the image, and a value of 2.0 lightens the image. 0 is ignored. + */ + gamma?: number; + /** + * Increase contrast by a factor. A value of 1.0 equals no change, a value of + * 0.5 equals low contrast, and a value of 2.0 equals high contrast. 0 is + * ignored. + */ + saturation?: number; + /** + * Flips the images horizontally, vertically, or both. Flipping is applied before + * rotation, so if you apply flip=h,rotate=90 then the image will be flipped + * horizontally, then rotated by 90 degrees. + */ + flip?: 'h' | 'v' | 'hv'; + /** + * Slightly reduces latency on a cache miss by selecting a + * quickest-to-compress file format, at a cost of increased file size and + * lower image quality. It will usually override the format option and choose + * JPEG over WebP or AVIF. We do not recommend using this option, except in + * unusual circumstances like resizing uncacheable dynamically-generated + * images. + */ + compression?: "fast"; +} +interface RequestInitCfPropertiesImageMinify { + javascript?: boolean; + css?: boolean; + html?: boolean; +} +interface RequestInitCfPropertiesR2 { + /** + * Colo id of bucket that an object is stored in + */ + bucketColoId?: number; +} +/** + * Request metadata provided by Cloudflare's edge. + */ +type IncomingRequestCfProperties = IncomingRequestCfPropertiesBase & IncomingRequestCfPropertiesBotManagementEnterprise & IncomingRequestCfPropertiesCloudflareForSaaSEnterprise & IncomingRequestCfPropertiesGeographicInformation & IncomingRequestCfPropertiesCloudflareAccessOrApiShield; +interface IncomingRequestCfPropertiesBase extends Record { + /** + * [ASN](https://www.iana.org/assignments/as-numbers/as-numbers.xhtml) of the incoming request. + * + * @example 395747 + */ + asn?: number; + /** + * The organization which owns the ASN of the incoming request. + * + * @example "Google Cloud" + */ + asOrganization?: string; + /** + * The original value of the `Accept-Encoding` header if Cloudflare modified it. + * + * @example "gzip, deflate, br" + */ + clientAcceptEncoding?: string; + /** + * The number of milliseconds it took for the request to reach your worker. + * + * @example 22 + */ + clientTcpRtt?: number; + /** + * The three-letter [IATA](https://en.wikipedia.org/wiki/IATA_airport_code) + * airport code of the data center that the request hit. + * + * @example "DFW" + */ + colo: string; + /** + * Represents the upstream's response to a + * [TCP `keepalive` message](https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html) + * from cloudflare. + * + * For workers with no upstream, this will always be `1`. + * + * @example 3 + */ + edgeRequestKeepAliveStatus: IncomingRequestCfPropertiesEdgeRequestKeepAliveStatus; + /** + * The HTTP Protocol the request used. + * + * @example "HTTP/2" + */ + httpProtocol: string; + /** + * The browser-requested prioritization information in the request object. + * + * If no information was set, defaults to the empty string `""` + * + * @example "weight=192;exclusive=0;group=3;group-weight=127" + * @default "" + */ + requestPriority: string; + /** + * The TLS version of the connection to Cloudflare. + * In requests served over plaintext (without TLS), this property is the empty string `""`. + * + * @example "TLSv1.3" + */ + tlsVersion: string; + /** + * The cipher for the connection to Cloudflare. + * In requests served over plaintext (without TLS), this property is the empty string `""`. + * + * @example "AEAD-AES128-GCM-SHA256" + */ + tlsCipher: string; + /** + * Metadata containing the [`HELLO`](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2) and [`FINISHED`](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9) messages from this request's TLS handshake. + * + * If the incoming request was served over plaintext (without TLS) this field is undefined. + */ + tlsExportedAuthenticator?: IncomingRequestCfPropertiesExportedAuthenticatorMetadata; +} +interface IncomingRequestCfPropertiesBotManagementBase { + /** + * Cloudflare’s [level of certainty](https://developers.cloudflare.com/bots/concepts/bot-score/) that a request comes from a bot, + * represented as an integer percentage between `1` (almost certainly a bot) and `99` (almost certainly human). + * + * @example 54 + */ + score: number; + /** + * A boolean value that is true if the request comes from a good bot, like Google or Bing. + * Most customers choose to allow this traffic. For more details, see [Traffic from known bots](https://developers.cloudflare.com/firewall/known-issues-and-faq/#how-does-firewall-rules-handle-traffic-from-known-bots). + */ + verifiedBot: boolean; + /** + * A boolean value that is true if the request originates from a + * Cloudflare-verified proxy service. + */ + corporateProxy: boolean; + /** + * A boolean value that's true if the request matches [file extensions](https://developers.cloudflare.com/bots/reference/static-resources/) for many types of static resources. + */ + staticResource: boolean; + /** + * List of IDs that correlate to the Bot Management heuristic detections made on a request (you can have multiple heuristic detections on the same request). + */ + detectionIds: number[]; +} +interface IncomingRequestCfPropertiesBotManagement { + /** + * Results of Cloudflare's Bot Management analysis + */ + botManagement: IncomingRequestCfPropertiesBotManagementBase; + /** + * Duplicate of `botManagement.score`. + * + * @deprecated + */ + clientTrustScore: number; +} +interface IncomingRequestCfPropertiesBotManagementEnterprise extends IncomingRequestCfPropertiesBotManagement { + /** + * Results of Cloudflare's Bot Management analysis + */ + botManagement: IncomingRequestCfPropertiesBotManagementBase & { + /** + * A [JA3 Fingerprint](https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/) to help profile specific SSL/TLS clients + * across different destination IPs, Ports, and X509 certificates. + */ + ja3Hash: string; + }; +} +interface IncomingRequestCfPropertiesCloudflareForSaaSEnterprise { + /** + * Custom metadata set per-host in [Cloudflare for SaaS](https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/). + * + * This field is only present if you have Cloudflare for SaaS enabled on your account + * and you have followed the [required steps to enable it]((https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/domain-support/custom-metadata/)). + */ + hostMetadata?: HostMetadata; +} +interface IncomingRequestCfPropertiesCloudflareAccessOrApiShield { + /** + * Information about the client certificate presented to Cloudflare. + * + * This is populated when the incoming request is served over TLS using + * either Cloudflare Access or API Shield (mTLS) + * and the presented SSL certificate has a valid + * [Certificate Serial Number](https://ldapwiki.com/wiki/Certificate%20Serial%20Number) + * (i.e., not `null` or `""`). + * + * Otherwise, a set of placeholder values are used. + * + * The property `certPresented` will be set to `"1"` when + * the object is populated (i.e. the above conditions were met). + */ + tlsClientAuth: IncomingRequestCfPropertiesTLSClientAuth | IncomingRequestCfPropertiesTLSClientAuthPlaceholder; +} +/** + * Metadata about the request's TLS handshake + */ +interface IncomingRequestCfPropertiesExportedAuthenticatorMetadata { + /** + * The client's [`HELLO` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2), encoded in hexadecimal + * + * @example "44372ba35fa1270921d318f34c12f155dc87b682cf36a790cfaa3ba8737a1b5d" + */ + clientHandshake: string; + /** + * The server's [`HELLO` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2), encoded in hexadecimal + * + * @example "44372ba35fa1270921d318f34c12f155dc87b682cf36a790cfaa3ba8737a1b5d" + */ + serverHandshake: string; + /** + * The client's [`FINISHED` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9), encoded in hexadecimal + * + * @example "084ee802fe1348f688220e2a6040a05b2199a761f33cf753abb1b006792d3f8b" + */ + clientFinished: string; + /** + * The server's [`FINISHED` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9), encoded in hexadecimal + * + * @example "084ee802fe1348f688220e2a6040a05b2199a761f33cf753abb1b006792d3f8b" + */ + serverFinished: string; +} +/** + * Geographic data about the request's origin. + */ +interface IncomingRequestCfPropertiesGeographicInformation { + /** + * The [ISO 3166-1 Alpha 2](https://www.iso.org/iso-3166-country-codes.html) country code the request originated from. + * + * If your worker is [configured to accept TOR connections](https://support.cloudflare.com/hc/en-us/articles/203306930-Understanding-Cloudflare-Tor-support-and-Onion-Routing), this may also be `"T1"`, indicating a request that originated over TOR. + * + * If Cloudflare is unable to determine where the request originated this property is omitted. + * + * The country code `"T1"` is used for requests originating on TOR. + * + * @example "GB" + */ + country?: Iso3166Alpha2Code | "T1"; + /** + * If present, this property indicates that the request originated in the EU + * + * @example "1" + */ + isEUCountry?: "1"; + /** + * A two-letter code indicating the continent the request originated from. + * + * @example "AN" + */ + continent?: ContinentCode; + /** + * The city the request originated from + * + * @example "Austin" + */ + city?: string; + /** + * Postal code of the incoming request + * + * @example "78701" + */ + postalCode?: string; + /** + * Latitude of the incoming request + * + * @example "30.27130" + */ + latitude?: string; + /** + * Longitude of the incoming request + * + * @example "-97.74260" + */ + longitude?: string; + /** + * Timezone of the incoming request + * + * @example "America/Chicago" + */ + timezone?: string; + /** + * If known, the ISO 3166-2 name for the first level region associated with + * the IP address of the incoming request + * + * @example "Texas" + */ + region?: string; + /** + * If known, the ISO 3166-2 code for the first-level region associated with + * the IP address of the incoming request + * + * @example "TX" + */ + regionCode?: string; + /** + * Metro code (DMA) of the incoming request + * + * @example "635" + */ + metroCode?: string; +} +/** Data about the incoming request's TLS certificate */ +interface IncomingRequestCfPropertiesTLSClientAuth { + /** Always `"1"`, indicating that the certificate was presented */ + certPresented: "1"; + /** + * Result of certificate verification. + * + * @example "FAILED:self signed certificate" + */ + certVerified: Exclude; + /** The presented certificate's revokation status. + * + * - A value of `"1"` indicates the certificate has been revoked + * - A value of `"0"` indicates the certificate has not been revoked + */ + certRevoked: "1" | "0"; + /** + * The certificate issuer's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) + * + * @example "CN=cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" + */ + certIssuerDN: string; + /** + * The certificate subject's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) + * + * @example "CN=*.cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" + */ + certSubjectDN: string; + /** + * The certificate issuer's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) ([RFC 2253](https://www.rfc-editor.org/rfc/rfc2253.html) formatted) + * + * @example "CN=cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" + */ + certIssuerDNRFC2253: string; + /** + * The certificate subject's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) ([RFC 2253](https://www.rfc-editor.org/rfc/rfc2253.html) formatted) + * + * @example "CN=*.cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" + */ + certSubjectDNRFC2253: string; + /** The certificate issuer's distinguished name (legacy policies) */ + certIssuerDNLegacy: string; + /** The certificate subject's distinguished name (legacy policies) */ + certSubjectDNLegacy: string; + /** + * The certificate's serial number + * + * @example "00936EACBE07F201DF" + */ + certSerial: string; + /** + * The certificate issuer's serial number + * + * @example "2489002934BDFEA34" + */ + certIssuerSerial: string; + /** + * The certificate's Subject Key Identifier + * + * @example "BB:AF:7E:02:3D:FA:A6:F1:3C:84:8E:AD:EE:38:98:EC:D9:32:32:D4" + */ + certSKI: string; + /** + * The certificate issuer's Subject Key Identifier + * + * @example "BB:AF:7E:02:3D:FA:A6:F1:3C:84:8E:AD:EE:38:98:EC:D9:32:32:D4" + */ + certIssuerSKI: string; + /** + * The certificate's SHA-1 fingerprint + * + * @example "6b9109f323999e52259cda7373ff0b4d26bd232e" + */ + certFingerprintSHA1: string; + /** + * The certificate's SHA-256 fingerprint + * + * @example "acf77cf37b4156a2708e34c4eb755f9b5dbbe5ebb55adfec8f11493438d19e6ad3f157f81fa3b98278453d5652b0c1fd1d71e5695ae4d709803a4d3f39de9dea" + */ + certFingerprintSHA256: string; + /** + * The effective starting date of the certificate + * + * @example "Dec 22 19:39:00 2018 GMT" + */ + certNotBefore: string; + /** + * The effective expiration date of the certificate + * + * @example "Dec 22 19:39:00 2018 GMT" + */ + certNotAfter: string; +} +/** Placeholder values for TLS Client Authorization */ +interface IncomingRequestCfPropertiesTLSClientAuthPlaceholder { + certPresented: "0"; + certVerified: "NONE"; + certRevoked: "0"; + certIssuerDN: ""; + certSubjectDN: ""; + certIssuerDNRFC2253: ""; + certSubjectDNRFC2253: ""; + certIssuerDNLegacy: ""; + certSubjectDNLegacy: ""; + certSerial: ""; + certIssuerSerial: ""; + certSKI: ""; + certIssuerSKI: ""; + certFingerprintSHA1: ""; + certFingerprintSHA256: ""; + certNotBefore: ""; + certNotAfter: ""; +} +/** Possible outcomes of TLS verification */ +declare type CertVerificationStatus = +/** Authentication succeeded */ +"SUCCESS" +/** No certificate was presented */ + | "NONE" +/** Failed because the certificate was self-signed */ + | "FAILED:self signed certificate" +/** Failed because the certificate failed a trust chain check */ + | "FAILED:unable to verify the first certificate" +/** Failed because the certificate not yet valid */ + | "FAILED:certificate is not yet valid" +/** Failed because the certificate is expired */ + | "FAILED:certificate has expired" +/** Failed for another unspecified reason */ + | "FAILED"; +/** + * An upstream endpoint's response to a TCP `keepalive` message from Cloudflare. + */ +declare type IncomingRequestCfPropertiesEdgeRequestKeepAliveStatus = 0 /** Unknown */ | 1 /** no keepalives (not found) */ | 2 /** no connection re-use, opening keepalive connection failed */ | 3 /** no connection re-use, keepalive accepted and saved */ | 4 /** connection re-use, refused by the origin server (`TCP FIN`) */ | 5; /** connection re-use, accepted by the origin server */ +/** ISO 3166-1 Alpha-2 codes */ +declare type Iso3166Alpha2Code = "AD" | "AE" | "AF" | "AG" | "AI" | "AL" | "AM" | "AO" | "AQ" | "AR" | "AS" | "AT" | "AU" | "AW" | "AX" | "AZ" | "BA" | "BB" | "BD" | "BE" | "BF" | "BG" | "BH" | "BI" | "BJ" | "BL" | "BM" | "BN" | "BO" | "BQ" | "BR" | "BS" | "BT" | "BV" | "BW" | "BY" | "BZ" | "CA" | "CC" | "CD" | "CF" | "CG" | "CH" | "CI" | "CK" | "CL" | "CM" | "CN" | "CO" | "CR" | "CU" | "CV" | "CW" | "CX" | "CY" | "CZ" | "DE" | "DJ" | "DK" | "DM" | "DO" | "DZ" | "EC" | "EE" | "EG" | "EH" | "ER" | "ES" | "ET" | "FI" | "FJ" | "FK" | "FM" | "FO" | "FR" | "GA" | "GB" | "GD" | "GE" | "GF" | "GG" | "GH" | "GI" | "GL" | "GM" | "GN" | "GP" | "GQ" | "GR" | "GS" | "GT" | "GU" | "GW" | "GY" | "HK" | "HM" | "HN" | "HR" | "HT" | "HU" | "ID" | "IE" | "IL" | "IM" | "IN" | "IO" | "IQ" | "IR" | "IS" | "IT" | "JE" | "JM" | "JO" | "JP" | "KE" | "KG" | "KH" | "KI" | "KM" | "KN" | "KP" | "KR" | "KW" | "KY" | "KZ" | "LA" | "LB" | "LC" | "LI" | "LK" | "LR" | "LS" | "LT" | "LU" | "LV" | "LY" | "MA" | "MC" | "MD" | "ME" | "MF" | "MG" | "MH" | "MK" | "ML" | "MM" | "MN" | "MO" | "MP" | "MQ" | "MR" | "MS" | "MT" | "MU" | "MV" | "MW" | "MX" | "MY" | "MZ" | "NA" | "NC" | "NE" | "NF" | "NG" | "NI" | "NL" | "NO" | "NP" | "NR" | "NU" | "NZ" | "OM" | "PA" | "PE" | "PF" | "PG" | "PH" | "PK" | "PL" | "PM" | "PN" | "PR" | "PS" | "PT" | "PW" | "PY" | "QA" | "RE" | "RO" | "RS" | "RU" | "RW" | "SA" | "SB" | "SC" | "SD" | "SE" | "SG" | "SH" | "SI" | "SJ" | "SK" | "SL" | "SM" | "SN" | "SO" | "SR" | "SS" | "ST" | "SV" | "SX" | "SY" | "SZ" | "TC" | "TD" | "TF" | "TG" | "TH" | "TJ" | "TK" | "TL" | "TM" | "TN" | "TO" | "TR" | "TT" | "TV" | "TW" | "TZ" | "UA" | "UG" | "UM" | "US" | "UY" | "UZ" | "VA" | "VC" | "VE" | "VG" | "VI" | "VN" | "VU" | "WF" | "WS" | "YE" | "YT" | "ZA" | "ZM" | "ZW"; +/** The 2-letter continent codes Cloudflare uses */ +declare type ContinentCode = "AF" | "AN" | "AS" | "EU" | "NA" | "OC" | "SA"; +type CfProperties = IncomingRequestCfProperties | RequestInitCfProperties; +interface D1Meta { + duration: number; + size_after: number; + rows_read: number; + rows_written: number; + last_row_id: number; + changed_db: boolean; + changes: number; + /** + * The region of the database instance that executed the query. + */ + served_by_region?: string; + /** + * True if-and-only-if the database instance that executed the query was the primary. + */ + served_by_primary?: boolean; + timings?: { + /** + * The duration of the SQL query execution by the database instance. It doesn't include any network time. + */ + sql_duration_ms: number; + }; + /** + * Number of total attempts to execute the query, due to automatic retries. + * Note: All other fields in the response like `timings` only apply to the last attempt. + */ + total_attempts?: number; +} +interface D1Response { + success: true; + meta: D1Meta & Record; + error?: never; +} +type D1Result = D1Response & { + results: T[]; +}; +interface D1ExecResult { + count: number; + duration: number; +} +type D1SessionConstraint = +// Indicates that the first query should go to the primary, and the rest queries +// using the same D1DatabaseSession will go to any replica that is consistent with +// the bookmark maintained by the session (returned by the first query). +'first-primary' +// Indicates that the first query can go anywhere (primary or replica), and the rest queries +// using the same D1DatabaseSession will go to any replica that is consistent with +// the bookmark maintained by the session (returned by the first query). + | 'first-unconstrained'; +type D1SessionBookmark = string; +declare abstract class D1Database { + prepare(query: string): D1PreparedStatement; + batch(statements: D1PreparedStatement[]): Promise[]>; + exec(query: string): Promise; + /** + * Creates a new D1 Session anchored at the given constraint or the bookmark. + * All queries executed using the created session will have sequential consistency, + * meaning that all writes done through the session will be visible in subsequent reads. + * + * @param constraintOrBookmark Either the session constraint or the explicit bookmark to anchor the created session. + */ + withSession(constraintOrBookmark?: D1SessionBookmark | D1SessionConstraint): D1DatabaseSession; + /** + * @deprecated dump() will be removed soon, only applies to deprecated alpha v1 databases. + */ + dump(): Promise; +} +declare abstract class D1DatabaseSession { + prepare(query: string): D1PreparedStatement; + batch(statements: D1PreparedStatement[]): Promise[]>; + /** + * @returns The latest session bookmark across all executed queries on the session. + * If no query has been executed yet, `null` is returned. + */ + getBookmark(): D1SessionBookmark | null; +} +declare abstract class D1PreparedStatement { + bind(...values: unknown[]): D1PreparedStatement; + first(colName: string): Promise; + first>(): Promise; + run>(): Promise>; + all>(): Promise>; + raw(options: { + columnNames: true; + }): Promise<[ + string[], + ...T[] + ]>; + raw(options?: { + columnNames?: false; + }): Promise; +} +// `Disposable` was added to TypeScript's standard lib types in version 5.2. +// To support older TypeScript versions, define an empty `Disposable` interface. +// Users won't be able to use `using`/`Symbol.dispose` without upgrading to 5.2, +// but this will ensure type checking on older versions still passes. +// TypeScript's interface merging will ensure our empty interface is effectively +// ignored when `Disposable` is included in the standard lib. +interface Disposable { +} +/** + * An email message that can be sent from a Worker. + */ +interface EmailMessage { + /** + * Envelope From attribute of the email message. + */ + readonly from: string; + /** + * Envelope To attribute of the email message. + */ + readonly to: string; +} +/** + * An email message that is sent to a consumer Worker and can be rejected/forwarded. + */ +interface ForwardableEmailMessage extends EmailMessage { + /** + * Stream of the email message content. + */ + readonly raw: ReadableStream; + /** + * An [Headers object](https://developer.mozilla.org/en-US/docs/Web/API/Headers). + */ + readonly headers: Headers; + /** + * Size of the email message content. + */ + readonly rawSize: number; + /** + * Reject this email message by returning a permanent SMTP error back to the connecting client including the given reason. + * @param reason The reject reason. + * @returns void + */ + setReject(reason: string): void; + /** + * Forward this email message to a verified destination address of the account. + * @param rcptTo Verified destination address. + * @param headers A [Headers object](https://developer.mozilla.org/en-US/docs/Web/API/Headers). + * @returns A promise that resolves when the email message is forwarded. + */ + forward(rcptTo: string, headers?: Headers): Promise; + /** + * Reply to the sender of this email message with a new EmailMessage object. + * @param message The reply message. + * @returns A promise that resolves when the email message is replied. + */ + reply(message: EmailMessage): Promise; +} +/** + * A binding that allows a Worker to send email messages. + */ +interface SendEmail { + send(message: EmailMessage): Promise; +} +declare abstract class EmailEvent extends ExtendableEvent { + readonly message: ForwardableEmailMessage; +} +declare type EmailExportedHandler = (message: ForwardableEmailMessage, env: Env, ctx: ExecutionContext) => void | Promise; +declare module "cloudflare:email" { + let _EmailMessage: { + prototype: EmailMessage; + new (from: string, to: string, raw: ReadableStream | string): EmailMessage; + }; + export { _EmailMessage as EmailMessage }; +} +/** + * Hello World binding to serve as an explanatory example. DO NOT USE + */ +interface HelloWorldBinding { + /** + * Retrieve the current stored value + */ + get(): Promise<{ + value: string; + ms?: number; + }>; + /** + * Set a new stored value + */ + set(value: string): Promise; +} +interface Hyperdrive { + /** + * Connect directly to Hyperdrive as if it's your database, returning a TCP socket. + * + * Calling this method returns an idential socket to if you call + * `connect("host:port")` using the `host` and `port` fields from this object. + * Pick whichever approach works better with your preferred DB client library. + * + * Note that this socket is not yet authenticated -- it's expected that your + * code (or preferably, the client library of your choice) will authenticate + * using the information in this class's readonly fields. + */ + connect(): Socket; + /** + * A valid DB connection string that can be passed straight into the typical + * client library/driver/ORM. This will typically be the easiest way to use + * Hyperdrive. + */ + readonly connectionString: string; + /* + * A randomly generated hostname that is only valid within the context of the + * currently running Worker which, when passed into `connect()` function from + * the "cloudflare:sockets" module, will connect to the Hyperdrive instance + * for your database. + */ + readonly host: string; + /* + * The port that must be paired the the host field when connecting. + */ + readonly port: number; + /* + * The username to use when authenticating to your database via Hyperdrive. + * Unlike the host and password, this will be the same every time + */ + readonly user: string; + /* + * The randomly generated password to use when authenticating to your + * database via Hyperdrive. Like the host field, this password is only valid + * within the context of the currently running Worker instance from which + * it's read. + */ + readonly password: string; + /* + * The name of the database to connect to. + */ + readonly database: string; +} +// Copyright (c) 2024 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +type ImageInfoResponse = { + format: 'image/svg+xml'; +} | { + format: string; + fileSize: number; + width: number; + height: number; +}; +type ImageTransform = { + width?: number; + height?: number; + background?: string; + blur?: number; + border?: { + color?: string; + width?: number; + } | { + top?: number; + bottom?: number; + left?: number; + right?: number; + }; + brightness?: number; + contrast?: number; + fit?: 'scale-down' | 'contain' | 'pad' | 'squeeze' | 'cover' | 'crop'; + flip?: 'h' | 'v' | 'hv'; + gamma?: number; + segment?: 'foreground'; + gravity?: 'face' | 'left' | 'right' | 'top' | 'bottom' | 'center' | 'auto' | 'entropy' | { + x?: number; + y?: number; + mode: 'remainder' | 'box-center'; + }; + rotate?: 0 | 90 | 180 | 270; + saturation?: number; + sharpen?: number; + trim?: 'border' | { + top?: number; + bottom?: number; + left?: number; + right?: number; + width?: number; + height?: number; + border?: boolean | { + color?: string; + tolerance?: number; + keep?: number; + }; + }; +}; +type ImageDrawOptions = { + opacity?: number; + repeat?: boolean | string; + top?: number; + left?: number; + bottom?: number; + right?: number; +}; +type ImageInputOptions = { + encoding?: 'base64'; +}; +type ImageOutputOptions = { + format: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | 'image/avif' | 'rgb' | 'rgba'; + quality?: number; + background?: string; + anim?: boolean; +}; +interface ImagesBinding { + /** + * Get image metadata (type, width and height) + * @throws {@link ImagesError} with code 9412 if input is not an image + * @param stream The image bytes + */ + info(stream: ReadableStream, options?: ImageInputOptions): Promise; + /** + * Begin applying a series of transformations to an image + * @param stream The image bytes + * @returns A transform handle + */ + input(stream: ReadableStream, options?: ImageInputOptions): ImageTransformer; +} +interface ImageTransformer { + /** + * Apply transform next, returning a transform handle. + * You can then apply more transformations, draw, or retrieve the output. + * @param transform + */ + transform(transform: ImageTransform): ImageTransformer; + /** + * Draw an image on this transformer, returning a transform handle. + * You can then apply more transformations, draw, or retrieve the output. + * @param image The image (or transformer that will give the image) to draw + * @param options The options configuring how to draw the image + */ + draw(image: ReadableStream | ImageTransformer, options?: ImageDrawOptions): ImageTransformer; + /** + * Retrieve the image that results from applying the transforms to the + * provided input + * @param options Options that apply to the output e.g. output format + */ + output(options: ImageOutputOptions): Promise; +} +type ImageTransformationOutputOptions = { + encoding?: 'base64'; +}; +interface ImageTransformationResult { + /** + * The image as a response, ready to store in cache or return to users + */ + response(): Response; + /** + * The content type of the returned image + */ + contentType(): string; + /** + * The bytes of the response + */ + image(options?: ImageTransformationOutputOptions): ReadableStream; +} +interface ImagesError extends Error { + readonly code: number; + readonly message: string; + readonly stack?: string; +} +/** + * Media binding for transforming media streams. + * Provides the entry point for media transformation operations. + */ +interface MediaBinding { + /** + * Creates a media transformer from an input stream. + * @param media - The input media bytes + * @returns A MediaTransformer instance for applying transformations + */ + input(media: ReadableStream): MediaTransformer; +} +/** + * Media transformer for applying transformation operations to media content. + * Handles sizing, fitting, and other input transformation parameters. + */ +interface MediaTransformer { + /** + * Applies transformation options to the media content. + * @param transform - Configuration for how the media should be transformed + * @returns A generator for producing the transformed media output + */ + transform(transform: MediaTransformationInputOptions): MediaTransformationGenerator; +} +/** + * Generator for producing media transformation results. + * Configures the output format and parameters for the transformed media. + */ +interface MediaTransformationGenerator { + /** + * Generates the final media output with specified options. + * @param output - Configuration for the output format and parameters + * @returns The final transformation result containing the transformed media + */ + output(output: MediaTransformationOutputOptions): MediaTransformationResult; +} +/** + * Result of a media transformation operation. + * Provides multiple ways to access the transformed media content. + */ +interface MediaTransformationResult { + /** + * Returns the transformed media as a readable stream of bytes. + * @returns A stream containing the transformed media data + */ + media(): ReadableStream; + /** + * Returns the transformed media as an HTTP response object. + * @returns The transformed media as a Response, ready to store in cache or return to users + */ + response(): Response; + /** + * Returns the MIME type of the transformed media. + * @returns The content type string (e.g., 'image/jpeg', 'video/mp4') + */ + contentType(): string; +} +/** + * Configuration options for transforming media input. + * Controls how the media should be resized and fitted. + */ +type MediaTransformationInputOptions = { + /** How the media should be resized to fit the specified dimensions */ + fit?: 'contain' | 'cover' | 'scale-down'; + /** Target width in pixels */ + width?: number; + /** Target height in pixels */ + height?: number; +}; +/** + * Configuration options for Media Transformations output. + * Controls the format, timing, and type of the generated output. + */ +type MediaTransformationOutputOptions = { + /** + * Output mode determining the type of media to generate + */ + mode?: 'video' | 'spritesheet' | 'frame' | 'audio'; + /** Whether to include audio in the output */ + audio?: boolean; + /** + * Starting timestamp for frame extraction or start time for clips. (e.g. '2s'). + */ + time?: string; + /** + * Duration for video clips, audio extraction, and spritesheet generation (e.g. '5s'). + */ + duration?: string; + /** + * Number of frames in the spritesheet. + */ + imageCount?: number; + /** + * Output format for the generated media. + */ + format?: 'jpg' | 'png' | 'm4a'; +}; +/** + * Error object for media transformation operations. + * Extends the standard Error interface with additional media-specific information. + */ +interface MediaError extends Error { + readonly code: number; + readonly message: string; + readonly stack?: string; +} +declare module 'cloudflare:node' { + interface NodeStyleServer { + listen(...args: unknown[]): this; + address(): { + port?: number | null | undefined; + }; + } + export function httpServerHandler(port: number): ExportedHandler; + export function httpServerHandler(options: { + port: number; + }): ExportedHandler; + export function httpServerHandler(server: NodeStyleServer): ExportedHandler; +} +type Params

= Record; +type EventContext = { + request: Request>; + functionPath: string; + waitUntil: (promise: Promise) => void; + passThroughOnException: () => void; + next: (input?: Request | string, init?: RequestInit) => Promise; + env: Env & { + ASSETS: { + fetch: typeof fetch; + }; + }; + params: Params

; + data: Data; +}; +type PagesFunction = Record> = (context: EventContext) => Response | Promise; +type EventPluginContext = { + request: Request>; + functionPath: string; + waitUntil: (promise: Promise) => void; + passThroughOnException: () => void; + next: (input?: Request | string, init?: RequestInit) => Promise; + env: Env & { + ASSETS: { + fetch: typeof fetch; + }; + }; + params: Params

; + data: Data; + pluginArgs: PluginArgs; +}; +type PagesPluginFunction = Record, PluginArgs = unknown> = (context: EventPluginContext) => Response | Promise; +declare module "assets:*" { + export const onRequest: PagesFunction; +} +// Copyright (c) 2022-2023 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +declare module "cloudflare:pipelines" { + export abstract class PipelineTransformationEntrypoint { + protected env: Env; + protected ctx: ExecutionContext; + constructor(ctx: ExecutionContext, env: Env); + /** + * run recieves an array of PipelineRecord which can be + * transformed and returned to the pipeline + * @param records Incoming records from the pipeline to be transformed + * @param metadata Information about the specific pipeline calling the transformation entrypoint + * @returns A promise containing the transformed PipelineRecord array + */ + public run(records: I[], metadata: PipelineBatchMetadata): Promise; + } + export type PipelineRecord = Record; + export type PipelineBatchMetadata = { + pipelineId: string; + pipelineName: string; + }; + export interface Pipeline { + /** + * The Pipeline interface represents the type of a binding to a Pipeline + * + * @param records The records to send to the pipeline + */ + send(records: T[]): Promise; + } +} +// PubSubMessage represents an incoming PubSub message. +// The message includes metadata about the broker, the client, and the payload +// itself. +// https://developers.cloudflare.com/pub-sub/ +interface PubSubMessage { + // Message ID + readonly mid: number; + // MQTT broker FQDN in the form mqtts://BROKER.NAMESPACE.cloudflarepubsub.com:PORT + readonly broker: string; + // The MQTT topic the message was sent on. + readonly topic: string; + // The client ID of the client that published this message. + readonly clientId: string; + // The unique identifier (JWT ID) used by the client to authenticate, if token + // auth was used. + readonly jti?: string; + // A Unix timestamp (seconds from Jan 1, 1970), set when the Pub/Sub Broker + // received the message from the client. + readonly receivedAt: number; + // An (optional) string with the MIME type of the payload, if set by the + // client. + readonly contentType: string; + // Set to 1 when the payload is a UTF-8 string + // https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901063 + readonly payloadFormatIndicator: number; + // Pub/Sub (MQTT) payloads can be UTF-8 strings, or byte arrays. + // You can use payloadFormatIndicator to inspect this before decoding. + payload: string | Uint8Array; +} +// JsonWebKey extended by kid parameter +interface JsonWebKeyWithKid extends JsonWebKey { + // Key Identifier of the JWK + readonly kid: string; +} +interface RateLimitOptions { + key: string; +} +interface RateLimitOutcome { + success: boolean; +} +interface RateLimit { + /** + * Rate limit a request based on the provided options. + * @see https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/ + * @returns A promise that resolves with the outcome of the rate limit. + */ + limit(options: RateLimitOptions): Promise; +} +// Namespace for RPC utility types. Unfortunately, we can't use a `module` here as these types need +// to referenced by `Fetcher`. This is included in the "importable" version of the types which +// strips all `module` blocks. +declare namespace Rpc { + // Branded types for identifying `WorkerEntrypoint`/`DurableObject`/`Target`s. + // TypeScript uses *structural* typing meaning anything with the same shape as type `T` is a `T`. + // For the classes exported by `cloudflare:workers` we want *nominal* typing (i.e. we only want to + // accept `WorkerEntrypoint` from `cloudflare:workers`, not any other class with the same shape) + export const __RPC_STUB_BRAND: '__RPC_STUB_BRAND'; + export const __RPC_TARGET_BRAND: '__RPC_TARGET_BRAND'; + export const __WORKER_ENTRYPOINT_BRAND: '__WORKER_ENTRYPOINT_BRAND'; + export const __DURABLE_OBJECT_BRAND: '__DURABLE_OBJECT_BRAND'; + export const __WORKFLOW_ENTRYPOINT_BRAND: '__WORKFLOW_ENTRYPOINT_BRAND'; + export interface RpcTargetBranded { + [__RPC_TARGET_BRAND]: never; + } + export interface WorkerEntrypointBranded { + [__WORKER_ENTRYPOINT_BRAND]: never; + } + export interface DurableObjectBranded { + [__DURABLE_OBJECT_BRAND]: never; + } + export interface WorkflowEntrypointBranded { + [__WORKFLOW_ENTRYPOINT_BRAND]: never; + } + export type EntrypointBranded = WorkerEntrypointBranded | DurableObjectBranded | WorkflowEntrypointBranded; + // Types that can be used through `Stub`s + export type Stubable = RpcTargetBranded | ((...args: any[]) => any); + // Types that can be passed over RPC + // The reason for using a generic type here is to build a serializable subset of structured + // cloneable composite types. This allows types defined with the "interface" keyword to pass the + // serializable check as well. Otherwise, only types defined with the "type" keyword would pass. + type Serializable = + // Structured cloneables + BaseType + // Structured cloneable composites + | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyArray ? Serializable : never> | { + [K in keyof T]: K extends number | string ? Serializable : never; + } + // Special types + | Stub + // Serialized as stubs, see `Stubify` + | Stubable; + // Base type for all RPC stubs, including common memory management methods. + // `T` is used as a marker type for unwrapping `Stub`s later. + interface StubBase extends Disposable { + [__RPC_STUB_BRAND]: T; + dup(): this; + } + export type Stub = Provider & StubBase; + // This represents all the types that can be sent as-is over an RPC boundary + type BaseType = void | undefined | null | boolean | number | bigint | string | TypedArray | ArrayBuffer | DataView | Date | Error | RegExp | ReadableStream | WritableStream | Request | Response | Headers; + // Recursively rewrite all `Stubable` types with `Stub`s + // prettier-ignore + type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { + [key: string | number]: any; + } ? { + [K in keyof T]: Stubify; + } : T; + // Recursively rewrite all `Stub`s with the corresponding `T`s. + // Note we use `StubBase` instead of `Stub` here to avoid circular dependencies: + // `Stub` depends on `Provider`, which depends on `Unstubify`, which would depend on `Stub`. + // prettier-ignore + type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { + [key: string | number]: unknown; + } ? { + [K in keyof T]: Unstubify; + } : T; + type UnstubifyAll = { + [I in keyof A]: Unstubify; + }; + // Utility type for adding `Provider`/`Disposable`s to `object` types only. + // Note `unknown & T` is equivalent to `T`. + type MaybeProvider = T extends object ? Provider : unknown; + type MaybeDisposable = T extends object ? Disposable : unknown; + // Type for method return or property on an RPC interface. + // - Stubable types are replaced by stubs. + // - Serializable types are passed by value, with stubable types replaced by stubs + // and a top-level `Disposer`. + // Everything else can't be passed over PRC. + // Technically, we use custom thenables here, but they quack like `Promise`s. + // Intersecting with `(Maybe)Provider` allows pipelining. + // prettier-ignore + type Result = R extends Stubable ? Promise> & Provider : R extends Serializable ? Promise & MaybeDisposable> & MaybeProvider : never; + // Type for method or property on an RPC interface. + // For methods, unwrap `Stub`s in parameters, and rewrite returns to be `Result`s. + // Unwrapping `Stub`s allows calling with `Stubable` arguments. + // For properties, rewrite types to be `Result`s. + // In each case, unwrap `Promise`s. + type MethodOrProperty = V extends (...args: infer P) => infer R ? (...args: UnstubifyAll

) => Result> : Result>; + // Type for the callable part of an `Provider` if `T` is callable. + // This is intersected with methods/properties. + type MaybeCallableProvider = T extends (...args: any[]) => any ? MethodOrProperty : unknown; + // Base type for all other types providing RPC-like interfaces. + // Rewrites all methods/properties to be `MethodOrProperty`s, while preserving callable types. + // `Reserved` names (e.g. stub method names like `dup()`) and symbols can't be accessed over RPC. + export type Provider = MaybeCallableProvider & Pick<{ + [K in keyof T]: MethodOrProperty; + }, Exclude>>; +} +declare namespace Cloudflare { + // Type of `env`. + // + // The specific project can extend `Env` by redeclaring it in project-specific files. Typescript + // will merge all declarations. + // + // You can use `wrangler types` to generate the `Env` type automatically. + interface Env { + } + // Project-specific parameters used to inform types. + // + // This interface is, again, intended to be declared in project-specific files, and then that + // declaration will be merged with this one. + // + // A project should have a declaration like this: + // + // interface GlobalProps { + // // Declares the main module's exports. Used to populate Cloudflare.Exports aka the type + // // of `ctx.exports`. + // mainModule: typeof import("my-main-module"); + // + // // Declares which of the main module's exports are configured with durable storage, and + // // thus should behave as Durable Object namsepace bindings. + // durableNamespaces: "MyDurableObject" | "AnotherDurableObject"; + // } + // + // You can use `wrangler types` to generate `GlobalProps` automatically. + interface GlobalProps { + } + // Evaluates to the type of a property in GlobalProps, defaulting to `Default` if it is not + // present. + type GlobalProp = K extends keyof GlobalProps ? GlobalProps[K] : Default; + // The type of the program's main module exports, if known. Requires `GlobalProps` to declare the + // `mainModule` property. + type MainModule = GlobalProp<"mainModule", {}>; + // The type of ctx.exports, which contains loopback bindings for all top-level exports. + type Exports = { + [K in keyof MainModule]: LoopbackForExport + // If the export is listed in `durableNamespaces`, then it is also a + // DurableObjectNamespace. + & (K extends GlobalProp<"durableNamespaces", never> ? MainModule[K] extends new (...args: any[]) => infer DoInstance ? DoInstance extends Rpc.DurableObjectBranded ? DurableObjectNamespace : DurableObjectNamespace : DurableObjectNamespace : {}); + }; +} +declare namespace CloudflareWorkersModule { + export type RpcStub = Rpc.Stub; + export const RpcStub: { + new (value: T): Rpc.Stub; + }; + export abstract class RpcTarget implements Rpc.RpcTargetBranded { + [Rpc.__RPC_TARGET_BRAND]: never; + } + // `protected` fields don't appear in `keyof`s, so can't be accessed over RPC + export abstract class WorkerEntrypoint implements Rpc.WorkerEntrypointBranded { + [Rpc.__WORKER_ENTRYPOINT_BRAND]: never; + protected ctx: ExecutionContext; + protected env: Env; + constructor(ctx: ExecutionContext, env: Env); + email?(message: ForwardableEmailMessage): void | Promise; + fetch?(request: Request): Response | Promise; + queue?(batch: MessageBatch): void | Promise; + scheduled?(controller: ScheduledController): void | Promise; + tail?(events: TraceItem[]): void | Promise; + tailStream?(event: TailStream.TailEvent): TailStream.TailEventHandlerType | Promise; + test?(controller: TestController): void | Promise; + trace?(traces: TraceItem[]): void | Promise; + } + export abstract class DurableObject implements Rpc.DurableObjectBranded { + [Rpc.__DURABLE_OBJECT_BRAND]: never; + protected ctx: DurableObjectState; + protected env: Env; + constructor(ctx: DurableObjectState, env: Env); + alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; + fetch?(request: Request): Response | Promise; + webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; + webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; + webSocketError?(ws: WebSocket, error: unknown): void | Promise; + } + export type WorkflowDurationLabel = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year'; + export type WorkflowSleepDuration = `${number} ${WorkflowDurationLabel}${'s' | ''}` | number; + export type WorkflowDelayDuration = WorkflowSleepDuration; + export type WorkflowTimeoutDuration = WorkflowSleepDuration; + export type WorkflowRetentionDuration = WorkflowSleepDuration; + export type WorkflowBackoff = 'constant' | 'linear' | 'exponential'; + export type WorkflowStepConfig = { + retries?: { + limit: number; + delay: WorkflowDelayDuration | number; + backoff?: WorkflowBackoff; + }; + timeout?: WorkflowTimeoutDuration | number; + }; + export type WorkflowEvent = { + payload: Readonly; + timestamp: Date; + instanceId: string; + }; + export type WorkflowStepEvent = { + payload: Readonly; + timestamp: Date; + type: string; + }; + export abstract class WorkflowStep { + do>(name: string, callback: () => Promise): Promise; + do>(name: string, config: WorkflowStepConfig, callback: () => Promise): Promise; + sleep: (name: string, duration: WorkflowSleepDuration) => Promise; + sleepUntil: (name: string, timestamp: Date | number) => Promise; + waitForEvent>(name: string, options: { + type: string; + timeout?: WorkflowTimeoutDuration | number; + }): Promise>; + } + export abstract class WorkflowEntrypoint | unknown = unknown> implements Rpc.WorkflowEntrypointBranded { + [Rpc.__WORKFLOW_ENTRYPOINT_BRAND]: never; + protected ctx: ExecutionContext; + protected env: Env; + constructor(ctx: ExecutionContext, env: Env); + run(event: Readonly>, step: WorkflowStep): Promise; + } + export function waitUntil(promise: Promise): void; + export function withEnv(newEnv: unknown, fn: () => unknown): unknown; + export function withExports(newExports: unknown, fn: () => unknown): unknown; + export function withEnvAndExports(newEnv: unknown, newExports: unknown, fn: () => unknown): unknown; + export const env: Cloudflare.Env; + export const exports: Cloudflare.Exports; +} +declare module 'cloudflare:workers' { + export = CloudflareWorkersModule; +} +interface SecretsStoreSecret { + /** + * Get a secret from the Secrets Store, returning a string of the secret value + * if it exists, or throws an error if it does not exist + */ + get(): Promise; +} +declare module "cloudflare:sockets" { + function _connect(address: string | SocketAddress, options?: SocketOptions): Socket; + export { _connect as connect }; +} +type MarkdownDocument = { + name: string; + blob: Blob; +}; +type ConversionResponse = { + name: string; + mimeType: string; + format: 'markdown'; + tokens: number; + data: string; +} | { + name: string; + mimeType: string; + format: 'error'; + error: string; +}; +type ImageConversionOptions = { + descriptionLanguage?: 'en' | 'es' | 'fr' | 'it' | 'pt' | 'de'; +}; +type EmbeddedImageConversionOptions = ImageConversionOptions & { + convert?: boolean; + maxConvertedImages?: number; +}; +type ConversionOptions = { + html?: { + images?: EmbeddedImageConversionOptions & { + convertOGImage?: boolean; + }; + }; + docx?: { + images?: EmbeddedImageConversionOptions; + }; + image?: ImageConversionOptions; + pdf?: { + images?: EmbeddedImageConversionOptions; + metadata?: boolean; + }; +}; +type ConversionRequestOptions = { + gateway?: GatewayOptions; + extraHeaders?: object; + conversionOptions?: ConversionOptions; +}; +type SupportedFileFormat = { + mimeType: string; + extension: string; +}; +declare abstract class ToMarkdownService { + transform(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; + transform(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; + supported(): Promise; +} +declare namespace TailStream { + interface Header { + readonly name: string; + readonly value: string; + } + interface FetchEventInfo { + readonly type: "fetch"; + readonly method: string; + readonly url: string; + readonly cfJson?: object; + readonly headers: Header[]; + } + interface JsRpcEventInfo { + readonly type: "jsrpc"; + } + interface ScheduledEventInfo { + readonly type: "scheduled"; + readonly scheduledTime: Date; + readonly cron: string; + } + interface AlarmEventInfo { + readonly type: "alarm"; + readonly scheduledTime: Date; + } + interface QueueEventInfo { + readonly type: "queue"; + readonly queueName: string; + readonly batchSize: number; + } + interface EmailEventInfo { + readonly type: "email"; + readonly mailFrom: string; + readonly rcptTo: string; + readonly rawSize: number; + } + interface TraceEventInfo { + readonly type: "trace"; + readonly traces: (string | null)[]; + } + interface HibernatableWebSocketEventInfoMessage { + readonly type: "message"; + } + interface HibernatableWebSocketEventInfoError { + readonly type: "error"; + } + interface HibernatableWebSocketEventInfoClose { + readonly type: "close"; + readonly code: number; + readonly wasClean: boolean; + } + interface HibernatableWebSocketEventInfo { + readonly type: "hibernatableWebSocket"; + readonly info: HibernatableWebSocketEventInfoClose | HibernatableWebSocketEventInfoError | HibernatableWebSocketEventInfoMessage; + } + interface CustomEventInfo { + readonly type: "custom"; + } + interface FetchResponseInfo { + readonly type: "fetch"; + readonly statusCode: number; + } + type EventOutcome = "ok" | "canceled" | "exception" | "unknown" | "killSwitch" | "daemonDown" | "exceededCpu" | "exceededMemory" | "loadShed" | "responseStreamDisconnected" | "scriptNotFound"; + interface ScriptVersion { + readonly id: string; + readonly tag?: string; + readonly message?: string; + } + interface Onset { + readonly type: "onset"; + readonly attributes: Attribute[]; + // id for the span being opened by this Onset event. + readonly spanId: string; + readonly dispatchNamespace?: string; + readonly entrypoint?: string; + readonly executionModel: string; + readonly scriptName?: string; + readonly scriptTags?: string[]; + readonly scriptVersion?: ScriptVersion; + readonly info: FetchEventInfo | JsRpcEventInfo | ScheduledEventInfo | AlarmEventInfo | QueueEventInfo | EmailEventInfo | TraceEventInfo | HibernatableWebSocketEventInfo | CustomEventInfo; + } + interface Outcome { + readonly type: "outcome"; + readonly outcome: EventOutcome; + readonly cpuTime: number; + readonly wallTime: number; + } + interface SpanOpen { + readonly type: "spanOpen"; + readonly name: string; + // id for the span being opened by this SpanOpen event. + readonly spanId: string; + readonly info?: FetchEventInfo | JsRpcEventInfo | Attributes; + } + interface SpanClose { + readonly type: "spanClose"; + readonly outcome: EventOutcome; + } + interface DiagnosticChannelEvent { + readonly type: "diagnosticChannel"; + readonly channel: string; + readonly message: any; + } + interface Exception { + readonly type: "exception"; + readonly name: string; + readonly message: string; + readonly stack?: string; + } + interface Log { + readonly type: "log"; + readonly level: "debug" | "error" | "info" | "log" | "warn"; + readonly message: object; + } + // This marks the worker handler return information. + // This is separate from Outcome because the worker invocation can live for a long time after + // returning. For example - Websockets that return an http upgrade response but then continue + // streaming information or SSE http connections. + interface Return { + readonly type: "return"; + readonly info?: FetchResponseInfo; + } + interface Attribute { + readonly name: string; + readonly value: string | string[] | boolean | boolean[] | number | number[] | bigint | bigint[]; + } + interface Attributes { + readonly type: "attributes"; + readonly info: Attribute[]; + } + type EventType = Onset | Outcome | SpanOpen | SpanClose | DiagnosticChannelEvent | Exception | Log | Return | Attributes; + // Context in which this trace event lives. + interface SpanContext { + // Single id for the entire top-level invocation + // This should be a new traceId for the first worker stage invoked in the eyeball request and then + // same-account service-bindings should reuse the same traceId but cross-account service-bindings + // should use a new traceId. + readonly traceId: string; + // spanId in which this event is handled + // for Onset and SpanOpen events this would be the parent span id + // for Outcome and SpanClose these this would be the span id of the opening Onset and SpanOpen events + // For Hibernate and Mark this would be the span under which they were emitted. + // spanId is not set ONLY if: + // 1. This is an Onset event + // 2. We are not inherting any SpanContext. (e.g. this is a cross-account service binding or a new top-level invocation) + readonly spanId?: string; + } + interface TailEvent { + // invocation id of the currently invoked worker stage. + // invocation id will always be unique to every Onset event and will be the same until the Outcome event. + readonly invocationId: string; + // Inherited spanContext for this event. + readonly spanContext: SpanContext; + readonly timestamp: Date; + readonly sequence: number; + readonly event: Event; + } + type TailEventHandler = (event: TailEvent) => void | Promise; + type TailEventHandlerObject = { + outcome?: TailEventHandler; + spanOpen?: TailEventHandler; + spanClose?: TailEventHandler; + diagnosticChannel?: TailEventHandler; + exception?: TailEventHandler; + log?: TailEventHandler; + return?: TailEventHandler; + attributes?: TailEventHandler; + }; + type TailEventHandlerType = TailEventHandler | TailEventHandlerObject; +} +// Copyright (c) 2022-2023 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +/** + * Data types supported for holding vector metadata. + */ +type VectorizeVectorMetadataValue = string | number | boolean | string[]; +/** + * Additional information to associate with a vector. + */ +type VectorizeVectorMetadata = VectorizeVectorMetadataValue | Record; +type VectorFloatArray = Float32Array | Float64Array; +interface VectorizeError { + code?: number; + error: string; +} +/** + * Comparison logic/operation to use for metadata filtering. + * + * This list is expected to grow as support for more operations are released. + */ +type VectorizeVectorMetadataFilterOp = '$eq' | '$ne' | '$lt' | '$lte' | '$gt' | '$gte'; +type VectorizeVectorMetadataFilterCollectionOp = '$in' | '$nin'; +/** + * Filter criteria for vector metadata used to limit the retrieved query result set. + */ +type VectorizeVectorMetadataFilter = { + [field: string]: Exclude | null | { + [Op in VectorizeVectorMetadataFilterOp]?: Exclude | null; + } | { + [Op in VectorizeVectorMetadataFilterCollectionOp]?: Exclude[]; + }; +}; +/** + * Supported distance metrics for an index. + * Distance metrics determine how other "similar" vectors are determined. + */ +type VectorizeDistanceMetric = "euclidean" | "cosine" | "dot-product"; +/** + * Metadata return levels for a Vectorize query. + * + * Default to "none". + * + * @property all Full metadata for the vector return set, including all fields (including those un-indexed) without truncation. This is a more expensive retrieval, as it requires additional fetching & reading of un-indexed data. + * @property indexed Return all metadata fields configured for indexing in the vector return set. This level of retrieval is "free" in that no additional overhead is incurred returning this data. However, note that indexed metadata is subject to truncation (especially for larger strings). + * @property none No indexed metadata will be returned. + */ +type VectorizeMetadataRetrievalLevel = "all" | "indexed" | "none"; +interface VectorizeQueryOptions { + topK?: number; + namespace?: string; + returnValues?: boolean; + returnMetadata?: boolean | VectorizeMetadataRetrievalLevel; + filter?: VectorizeVectorMetadataFilter; +} +/** + * Information about the configuration of an index. + */ +type VectorizeIndexConfig = { + dimensions: number; + metric: VectorizeDistanceMetric; +} | { + preset: string; // keep this generic, as we'll be adding more presets in the future and this is only in a read capacity +}; +/** + * Metadata about an existing index. + * + * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. + * See {@link VectorizeIndexInfo} for its post-beta equivalent. + */ +interface VectorizeIndexDetails { + /** The unique ID of the index */ + readonly id: string; + /** The name of the index. */ + name: string; + /** (optional) A human readable description for the index. */ + description?: string; + /** The index configuration, including the dimension size and distance metric. */ + config: VectorizeIndexConfig; + /** The number of records containing vectors within the index. */ + vectorsCount: number; +} +/** + * Metadata about an existing index. + */ +interface VectorizeIndexInfo { + /** The number of records containing vectors within the index. */ + vectorCount: number; + /** Number of dimensions the index has been configured for. */ + dimensions: number; + /** ISO 8601 datetime of the last processed mutation on in the index. All changes before this mutation will be reflected in the index state. */ + processedUpToDatetime: number; + /** UUIDv4 of the last mutation processed by the index. All changes before this mutation will be reflected in the index state. */ + processedUpToMutation: number; +} +/** + * Represents a single vector value set along with its associated metadata. + */ +interface VectorizeVector { + /** The ID for the vector. This can be user-defined, and must be unique. It should uniquely identify the object, and is best set based on the ID of what the vector represents. */ + id: string; + /** The vector values */ + values: VectorFloatArray | number[]; + /** The namespace this vector belongs to. */ + namespace?: string; + /** Metadata associated with the vector. Includes the values of other fields and potentially additional details. */ + metadata?: Record; +} +/** + * Represents a matched vector for a query along with its score and (if specified) the matching vector information. + */ +type VectorizeMatch = Pick, "values"> & Omit & { + /** The score or rank for similarity, when returned as a result */ + score: number; +}; +/** + * A set of matching {@link VectorizeMatch} for a particular query. + */ +interface VectorizeMatches { + matches: VectorizeMatch[]; + count: number; +} +/** + * Results of an operation that performed a mutation on a set of vectors. + * Here, `ids` is a list of vectors that were successfully processed. + * + * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. + * See {@link VectorizeAsyncMutation} for its post-beta equivalent. + */ +interface VectorizeVectorMutation { + /* List of ids of vectors that were successfully processed. */ + ids: string[]; + /* Total count of the number of processed vectors. */ + count: number; +} +/** + * Result type indicating a mutation on the Vectorize Index. + * Actual mutations are processed async where the `mutationId` is the unique identifier for the operation. + */ +interface VectorizeAsyncMutation { + /** The unique identifier for the async mutation operation containing the changeset. */ + mutationId: string; +} +/** + * A Vectorize Vector Search Index for querying vectors/embeddings. + * + * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. + * See {@link Vectorize} for its new implementation. + */ +declare abstract class VectorizeIndex { + /** + * Get information about the currently bound index. + * @returns A promise that resolves with information about the current index. + */ + public describe(): Promise; + /** + * Use the provided vector to perform a similarity search across the index. + * @param vector Input vector that will be used to drive the similarity search. + * @param options Configuration options to massage the returned data. + * @returns A promise that resolves with matched and scored vectors. + */ + public query(vector: VectorFloatArray | number[], options?: VectorizeQueryOptions): Promise; + /** + * Insert a list of vectors into the index dataset. If a provided id exists, an error will be thrown. + * @param vectors List of vectors that will be inserted. + * @returns A promise that resolves with the ids & count of records that were successfully processed. + */ + public insert(vectors: VectorizeVector[]): Promise; + /** + * Upsert a list of vectors into the index dataset. If a provided id exists, it will be replaced with the new values. + * @param vectors List of vectors that will be upserted. + * @returns A promise that resolves with the ids & count of records that were successfully processed. + */ + public upsert(vectors: VectorizeVector[]): Promise; + /** + * Delete a list of vectors with a matching id. + * @param ids List of vector ids that should be deleted. + * @returns A promise that resolves with the ids & count of records that were successfully processed (and thus deleted). + */ + public deleteByIds(ids: string[]): Promise; + /** + * Get a list of vectors with a matching id. + * @param ids List of vector ids that should be returned. + * @returns A promise that resolves with the raw unscored vectors matching the id set. + */ + public getByIds(ids: string[]): Promise; +} +/** + * A Vectorize Vector Search Index for querying vectors/embeddings. + * + * Mutations in this version are async, returning a mutation id. + */ +declare abstract class Vectorize { + /** + * Get information about the currently bound index. + * @returns A promise that resolves with information about the current index. + */ + public describe(): Promise; + /** + * Use the provided vector to perform a similarity search across the index. + * @param vector Input vector that will be used to drive the similarity search. + * @param options Configuration options to massage the returned data. + * @returns A promise that resolves with matched and scored vectors. + */ + public query(vector: VectorFloatArray | number[], options?: VectorizeQueryOptions): Promise; + /** + * Use the provided vector-id to perform a similarity search across the index. + * @param vectorId Id for a vector in the index against which the index should be queried. + * @param options Configuration options to massage the returned data. + * @returns A promise that resolves with matched and scored vectors. + */ + public queryById(vectorId: string, options?: VectorizeQueryOptions): Promise; + /** + * Insert a list of vectors into the index dataset. If a provided id exists, an error will be thrown. + * @param vectors List of vectors that will be inserted. + * @returns A promise that resolves with a unique identifier of a mutation containing the insert changeset. + */ + public insert(vectors: VectorizeVector[]): Promise; + /** + * Upsert a list of vectors into the index dataset. If a provided id exists, it will be replaced with the new values. + * @param vectors List of vectors that will be upserted. + * @returns A promise that resolves with a unique identifier of a mutation containing the upsert changeset. + */ + public upsert(vectors: VectorizeVector[]): Promise; + /** + * Delete a list of vectors with a matching id. + * @param ids List of vector ids that should be deleted. + * @returns A promise that resolves with a unique identifier of a mutation containing the delete changeset. + */ + public deleteByIds(ids: string[]): Promise; + /** + * Get a list of vectors with a matching id. + * @param ids List of vector ids that should be returned. + * @returns A promise that resolves with the raw unscored vectors matching the id set. + */ + public getByIds(ids: string[]): Promise; +} +/** + * The interface for "version_metadata" binding + * providing metadata about the Worker Version using this binding. + */ +type WorkerVersionMetadata = { + /** The ID of the Worker Version using this binding */ + id: string; + /** The tag of the Worker Version using this binding */ + tag: string; + /** The timestamp of when the Worker Version was uploaded */ + timestamp: string; +}; +interface DynamicDispatchLimits { + /** + * Limit CPU time in milliseconds. + */ + cpuMs?: number; + /** + * Limit number of subrequests. + */ + subRequests?: number; +} +interface DynamicDispatchOptions { + /** + * Limit resources of invoked Worker script. + */ + limits?: DynamicDispatchLimits; + /** + * Arguments for outbound Worker script, if configured. + */ + outbound?: { + [key: string]: any; + }; +} +interface DispatchNamespace { + /** + * @param name Name of the Worker script. + * @param args Arguments to Worker script. + * @param options Options for Dynamic Dispatch invocation. + * @returns A Fetcher object that allows you to send requests to the Worker script. + * @throws If the Worker script does not exist in this dispatch namespace, an error will be thrown. + */ + get(name: string, args?: { + [key: string]: any; + }, options?: DynamicDispatchOptions): Fetcher; +} +declare module 'cloudflare:workflows' { + /** + * NonRetryableError allows for a user to throw a fatal error + * that makes a Workflow instance fail immediately without triggering a retry + */ + export class NonRetryableError extends Error { + public constructor(message: string, name?: string); + } +} +declare abstract class Workflow { + /** + * Get a handle to an existing instance of the Workflow. + * @param id Id for the instance of this Workflow + * @returns A promise that resolves with a handle for the Instance + */ + public get(id: string): Promise; + /** + * Create a new instance and return a handle to it. If a provided id exists, an error will be thrown. + * @param options Options when creating an instance including id and params + * @returns A promise that resolves with a handle for the Instance + */ + public create(options?: WorkflowInstanceCreateOptions): Promise; + /** + * Create a batch of instances and return handle for all of them. If a provided id exists, an error will be thrown. + * `createBatch` is limited at 100 instances at a time or when the RPC limit for the batch (1MiB) is reached. + * @param batch List of Options when creating an instance including name and params + * @returns A promise that resolves with a list of handles for the created instances. + */ + public createBatch(batch: WorkflowInstanceCreateOptions[]): Promise; +} +type WorkflowDurationLabel = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year'; +type WorkflowSleepDuration = `${number} ${WorkflowDurationLabel}${'s' | ''}` | number; +type WorkflowRetentionDuration = WorkflowSleepDuration; +interface WorkflowInstanceCreateOptions { + /** + * An id for your Workflow instance. Must be unique within the Workflow. + */ + id?: string; + /** + * The event payload the Workflow instance is triggered with + */ + params?: PARAMS; + /** + * The retention policy for Workflow instance. + * Defaults to the maximum retention period available for the owner's account. + */ + retention?: { + successRetention?: WorkflowRetentionDuration; + errorRetention?: WorkflowRetentionDuration; + }; +} +type InstanceStatus = { + status: 'queued' // means that instance is waiting to be started (see concurrency limits) + | 'running' | 'paused' | 'errored' | 'terminated' // user terminated the instance while it was running + | 'complete' | 'waiting' // instance is hibernating and waiting for sleep or event to finish + | 'waitingForPause' // instance is finishing the current work to pause + | 'unknown'; + error?: { + name: string; + message: string; + }; + output?: unknown; +}; +interface WorkflowError { + code?: number; + message: string; +} +declare abstract class WorkflowInstance { + public id: string; + /** + * Pause the instance. + */ + public pause(): Promise; + /** + * Resume the instance. If it is already running, an error will be thrown. + */ + public resume(): Promise; + /** + * Terminate the instance. If it is errored, terminated or complete, an error will be thrown. + */ + public terminate(): Promise; + /** + * Restart the instance. + */ + public restart(): Promise; + /** + * Returns the current status of the instance. + */ + public status(): Promise; + /** + * Send an event to this instance. + */ + public sendEvent({ type, payload, }: { + type: string; + payload: unknown; + }): Promise; +} diff --git a/website/wrangler.jsonc b/website/wrangler.jsonc new file mode 100644 index 00000000..2fa2569b --- /dev/null +++ b/website/wrangler.jsonc @@ -0,0 +1,14 @@ +{ + "name": "dpkit", + "main": "main.ts", + "preview_urls": true, + "compatibility_date": "2025-08-15", + "compatibility_flags": ["nodejs_compat"], + "upload_source_maps": true, + "placement": { + "mode": "smart" + }, + "observability": { + "enabled": true + } +}