diff --git a/config.template.jsonc b/config.template.jsonc index f42a98ddd8..52fc1bf1aa 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -159,17 +159,30 @@ // ── Database ──────────────────────────────────────────────────────── "database": { - // `sqlite` for single-node / dev; `mysql` for self-host with MariaDB. + // `sqlite` for single-node/dev, `mysql` for MariaDB/MySQL, + // or `postgres` for PostgreSQL. + // + // NOTE: `postgres` support is a community contribution and is not + // exercised by Puter.com production. It boots, passes its own + // integration tests against pgmock, and runs the common user/app/ + // fsentry/session/permission/OIDC flows — but less-traveled code + // paths may surface MySQL/SQLite-isms that haven't been ported yet. + // Expect rough edges and please file issues if you hit one. For + // production self-hosting today, `mysql` (MariaDB) and `sqlite` are + // the supported defaults. "engine": "sqlite", // sqlite — file path on disk "path": "volatile/runtime/puter-database.sqlite", "targetVersion": 0, - // mysql — connection details + // mysql/postgres — connection details. PostgreSQL defaults to port + // 5432 when `engine` is `postgres`; MySQL/MariaDB normally use 3306. "host": "", "port": 3306, "user": "", "password": "", "database": "", + // postgres may also use a URL instead of host/user/password fields: + // "connectionString": "postgres://puter:secret@localhost:5432/puter", // Optional read-replica pool. Reads route here when populated. "replica": { "host": "", @@ -178,9 +191,10 @@ "password": "", "database": "" } - // mysql self-host bootstrap: set `migrationPaths` to apply the bundled - // schema on first boot. Idempotent — safe to leave on. + // mysql/postgres self-host bootstrap: set `migrationPaths` to apply the + // bundled schema on first boot. Idempotent — safe to leave on. // "migrationPaths": ["./src/backend/clients/database/migrations/mysql"] + // "migrationPaths": ["./src/backend/clients/database/migrations/postgres"] }, // ── DynamoDB (KV store) ───────────────────────────────────────────── diff --git a/doc/self-hosting.md b/doc/self-hosting.md index c8730f70f6..bbcd57480f 100644 --- a/doc/self-hosting.md +++ b/doc/self-hosting.md @@ -136,7 +136,7 @@ Why these knobs: - `jwt_secret` + `jwt_secret_v2` — Puter signs new auth tokens with `jwt_secret_v2` (v2 token format, `kid: 'v2'` JWT header). `jwt_secret` is verify-only and lets tokens minted before this version (cookies still in users' browsers, stored API tokens) keep working. `allow_v1_tokens: true` keeps that fallback enabled. Fresh installs can leave it as-is; future versions will flip the flag to `false` and retire `jwt_secret` entirely once legacy tokens have drained. - `env: "prod"` — the bundled `config.default.json` ships with `env: "dev"` (matches the source-tree `npm run start=gui` workflow, which expects webpack-dev-server emitting a CSS manifest). Self-host runs against pre-built static bundles, so `env: "prod"` makes the homepage emit the `/dist/bundle.min.css` `` tag instead of waiting on a manifest that doesn't exist. -- `database.migrationPaths` — Puter applies the bundled MySQL schema on boot. `mysql_mig_1.sql` (tables) and `mysql_mig_2.sql` (default apps: editor, viewer, pdf, camera, player, recorder, git, dev-center, puter-linux). Idempotent — safe to re-run. +- `database.migrationPaths` — Puter applies the bundled MySQL/MariaDB schema on boot. The migration files are idempotent, so it is safe to leave this configured across restarts. - `dynamo.bootstrapTables: true` — Puter creates its KV table on boot. **Only set against a local emulator**, never real AWS. - `dynamo.aws` keys are dummies; DynamoDB-local doesn't validate them but the AWS SDK requires _something_. **Note:** DynamoDB uses `access_key` / `secret_key` (snake_case); S3 below uses `accessKeyId` / `secretAccessKey` (camelCase). Not interchangeable. - `providers.ollama.enabled: false` — Puter auto-probes a local Ollama at `127.0.0.1:11434` by default; without one running you'd see `ECONNREFUSED` on every boot. To run a bundled Ollama, see [Optional: local LLM (Ollama)](#optional-local-llm-ollama) below. @@ -232,6 +232,26 @@ Change it in Settings after first login. All optional. Drop any of the blocks below into `puter/config/config.json` and `docker compose restart puter`. See [config.template.jsonc](../config.template.jsonc) for the full list. Per-key documentation lives in [src/backend/types.ts](../src/backend/types.ts). +### PostgreSQL database + +> **Community contribution — expect rough edges.** PostgreSQL support was contributed by the community and is not run by Puter.com production. It boots, applies the bundled schema, and exercises the common user/app/fsentry/session/permission/OIDC flows in the integration tests, but less-traveled SQL call sites may still need porting. If you hit a query that doesn't work on Postgres, please open an issue. For production self-hosting today, MariaDB/MySQL and SQLite are the supported defaults. + +The bundled Docker Compose stack still defaults to MariaDB. To use PostgreSQL instead, run a PostgreSQL service yourself, point Puter at it, and use the PostgreSQL migration path: + +```json +"database": { + "engine": "postgres", + "host": "postgres", + "port": 5432, + "user": "puter", + "password": "...", + "database": "puter", + "migrationPaths": ["/opt/puter/dist/src/backend/clients/database/migrations/postgres"] +} +``` + +You may use `"connectionString": "postgres://puter:...@postgres:5432/puter"` instead of the host/user/password fields. Puter only needs normal PostgreSQL connection details and the migration path; CloudNativePG is one possible Kubernetes operator, but no operator-specific manifests are required by Puter. + ### Email (SMTP) Used for password resets, email confirmation, and notifications. Without it those flows silently fail. diff --git a/extensions/appTelemetry.ts b/extensions/appTelemetry.ts index a5799c1116..48503dbce5 100644 --- a/extensions/appTelemetry.ts +++ b/extensions/appTelemetry.ts @@ -84,7 +84,7 @@ export const handleAppTelemetryUsers = async ( const users = await clients.db.read( `SELECT u.username, u.uuid FROM user_to_app_permissions p - INNER JOIN user u ON p.user_id = u.id + INNER JOIN ${clients.db.quoteIdentifier('user')} u ON p.user_id = u.id WHERE p.permission = 'flag:app-is-authenticated' AND p.app_id = ? ORDER BY (p.dt IS NOT NULL), p.dt, p.user_id LIMIT ? OFFSET ?`, diff --git a/package-lock.json b/package-lock.json index 3bb2565154..779df32bcb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15004,6 +15004,46 @@ "node": "*" } }, + "node_modules/pg": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.21.0.tgz", + "integrity": "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.13.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.14.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.13.0.tgz", + "integrity": "sha512-EMnU9E2fSULdsbErBbMaXJvFeD9B4+nPcM3f+4lsiCR0BHLPrLVjv3DbyM2hgQQviKJaTWIRRTjKjWlHg3p2ig==", + "license": "MIT" + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -15013,10 +15053,19 @@ "node": ">=4.0.0" } }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, "node_modules/pg-protocol": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", - "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.14.0.tgz", + "integrity": "sha512-n5taZ1kO3s9ngDTVxsEznOqCyToTgz0FLuPq0B33COy5pPpuWJpY3/2oRBVETuOgzdqRXfWpM9HIhp2LBBT1BA==", "license": "MIT" }, "node_modules/pg-types": { @@ -15035,6 +15084,22 @@ "node": ">=4" } }, + "node_modules/pgmock": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pgmock/-/pgmock-1.0.3.tgz", + "integrity": "sha512-5Lo17esUvwOq9TvAZMeFZRB69+QiSCpLrdwbsxDqGoj6yVZi+6umMiJvFiWwgo+YlNfWe1kY9Djw360T/J0AWg==", + "dev": true, + "license": "MIT" + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -18411,6 +18476,7 @@ "openai": "^6.34.0", "otpauth": "^9.2.4", "parse-domain": "^8.2.2", + "pg": "^8.21.0", "prompt-sync": "^4.2.0", "replicate": "^1.0.0", "sharp": "^0.34.5", @@ -18427,9 +18493,11 @@ "@types/busboy": "^1.5.4", "@types/jsonwebtoken": "^9.0.10", "@types/node": "^24.0.0", + "@types/pg": "^8.6.1", "@types/validator": "^13.15.10", "chai": "^4.3.7", "nodemon": "^3.1.0", + "pgmock": "^1.0.3", "typescript": "^5.9.3", "vite": "^8.0.0", "vitest": "^4.0.14" diff --git a/package.json b/package.json index 71c03a26a3..ef950c633b 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ }, "scripts": { "test:backend": "vitest run --config src/backend/vitest.config.ts ", + "test:backend:postgres": "PUTER_TEST_DB_ENGINE=postgres vitest run --config src/backend/vitest.config.ts ", "start=gui": "nodemon --exec \"node dev-server.js\" ", "start": "node --enable-source-maps -r ./dist/src/backend/telemetry.js ./dist/src/backend/index.js", "prestart": "npm run build:ts", diff --git a/src/backend/clients/database/DatabaseClient.ts b/src/backend/clients/database/DatabaseClient.ts index df78efea1b..75146395a0 100644 --- a/src/backend/clients/database/DatabaseClient.ts +++ b/src/backend/clients/database/DatabaseClient.ts @@ -31,6 +31,8 @@ export interface BatchEntry { values: unknown[]; } +type SqlJsonPath = readonly [string, ...string[]]; + /** * Base database client. Subclasses must override every method that throws here. * @@ -101,9 +103,10 @@ export class AbstractDatabaseClient extends PuterClient { const cols = Object.keys(data); const values = Object.values(data); const sql = - `INSERT INTO \`${tableName}\` ` + - `(${cols.map((c) => `\`${c}\``).join(', ')}) ` + - `VALUES (${cols.map(() => '?').join(', ')})`; + `INSERT INTO ${this.quoteIdentifier(tableName)} ` + + `(${cols.map((c) => this.quoteIdentifier(c)).join(', ')}) ` + + `VALUES (${cols.map(() => '?').join(', ')})` + + this.returningIdClause(); return this.write(sql, values); } @@ -161,4 +164,88 @@ export class AbstractDatabaseClient extends PuterClient { } return choices.otherwise as T; } + + quoteIdentifier(identifier: string): string { + return identifier + .split('.') + .map((part) => { + if (part === '*') return part; + return `\`${part.replaceAll('`', '``')}\``; + }) + .join('.'); + } + + booleanLiteral(value: boolean): string { + return value ? '1' : '0'; + } + + booleanValue(value: boolean): boolean | 0 | 1 { + return value ? 1 : 0; + } + + insertIgnoreInto(tableName: string): string { + const table = this.quoteIdentifier(tableName); + return this.case({ + sqlite: `INSERT OR IGNORE INTO ${table}`, + postgres: `INSERT INTO ${table}`, + otherwise: `INSERT IGNORE INTO ${table}`, + }); + } + + insertIgnoreSuffix(): string { + return this.case({ + postgres: ' ON CONFLICT DO NOTHING', + otherwise: '', + }); + } + + upsertClause( + conflictColumns: readonly string[], + updateColumns: readonly string[], + ): string { + if (updateColumns.length === 0) { + throw new Error('upsertClause requires at least one update column'); + } + + const updateList = updateColumns + .map((column) => `${this.quoteIdentifier(column)} = ?`) + .join(', '); + + return this.case({ + mysql: `ON DUPLICATE KEY UPDATE ${updateList}`, + otherwise: `ON CONFLICT(${conflictColumns + .map((column) => this.quoteIdentifier(column)) + .join(', ')}) DO UPDATE SET ${updateList}`, + }); + } + + jsonTextExtract(jsonExpression: string, path: SqlJsonPath): string { + const sqlitePath = `$${path.map((part) => `.${part}`).join('')}`; + return this.case({ + sqlite: `json_extract(${jsonExpression}, ${this.sqlStringLiteral(sqlitePath)})`, + mysql: `JSON_UNQUOTE(JSON_EXTRACT(${jsonExpression}, ${this.sqlStringLiteral(sqlitePath)}))`, + postgres: `${jsonExpression} #>> ARRAY[${path + .map((part) => this.sqlStringLiteral(part)) + .join(', ')}]`, + otherwise: `JSON_UNQUOTE(JSON_EXTRACT(${jsonExpression}, ${this.sqlStringLiteral(sqlitePath)}))`, + }); + } + + nullCoalesce(...expressions: readonly string[]): string { + if (expressions.length === 0) { + throw new Error('nullCoalesce requires at least one expression'); + } + return `COALESCE(${expressions.join(', ')})`; + } + + returningIdClause(): string { + return this.case({ + postgres: ' RETURNING id', + otherwise: '', + }); + } + + protected sqlStringLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'`; + } } diff --git a/src/backend/clients/database/MySQLDatabaseClient.ts b/src/backend/clients/database/MySQLDatabaseClient.ts index 6119688f41..09e6159d94 100644 --- a/src/backend/clients/database/MySQLDatabaseClient.ts +++ b/src/backend/clients/database/MySQLDatabaseClient.ts @@ -23,6 +23,7 @@ import { createPool, type Pool } from 'mysql2'; import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient'; import { SQLBatcher } from './SQLBatcher.js'; import { splitMysqlStatements } from './splitMysqlStatements.js'; +import { compareMigrationFilenames } from './migrationFilenames.js'; import type { IConfig } from '../../types'; const RETRIABLE_ERROR_CODES = new Set([ @@ -44,36 +45,7 @@ const RETRIABLE_ERROR_MESSAGES = [ 'ETIMEDOUT', ]; -/** - * Comparator for MySQL migration filenames. - * - * Existing files are named `mysql_mig_.sql` with unpadded N, so a - * plain lexical sort puts `mysql_mig_10.sql` BEFORE `mysql_mig_2.sql`. - * Pull the trailing integer out and sort numerically. Anything that - * doesn't match the `_.sql` shape (one-off, vendor dump) falls - * back to lexical comparison so the order stays deterministic, and - * unmatched names sort *after* numbered files so future numbered - * migrations don't get interleaved into a one-off's namespace. - * - * Exported only for the unit test — the production caller is the - * `runMigrations()` loop inside this file. - */ -export const compareMigrationFilenames = (a: string, b: string): number => { - const numericIndex = (name: string): number => { - const m = /_(\d+)\.sql$/.exec(name); - return m ? Number.parseInt(m[1], 10) : Number.NaN; - }; - const na = numericIndex(a); - const nb = numericIndex(b); - if (Number.isFinite(na) && Number.isFinite(nb)) { - if (na !== nb) return na - nb; - } else if (Number.isFinite(na)) { - return -1; - } else if (Number.isFinite(nb)) { - return 1; - } - return a.localeCompare(b); -}; +export { compareMigrationFilenames }; type PoolConfig = Parameters[0]; diff --git a/src/backend/clients/database/PostgresDatabaseClient.integration.test.ts b/src/backend/clients/database/PostgresDatabaseClient.integration.test.ts new file mode 100644 index 0000000000..468cb1722b --- /dev/null +++ b/src/backend/clients/database/PostgresDatabaseClient.integration.test.ts @@ -0,0 +1,328 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { Pool } from 'pg'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; +import type { IConfig } from '../../types'; +import type { PuterServer } from '../../server'; +import { generateDefaultFsentries } from '../../util/userProvisioning.js'; +import { + createPgMockPostgresDatabaseClient, + POSTGRES_TEST_MIGRATIONS_PATH, + setupTestServer, +} from '../../testUtil.js'; +import { PostgresDatabaseClient } from './PostgresDatabaseClient.js'; + +const postgresUrl = process.env.PUTER_TEST_POSTGRES_URL; +const postgresMigrationsPath = POSTGRES_TEST_MIGRATIONS_PATH; +const postgresTestSchemaPattern = /^puter_test_[a-f0-9]{32}$/u; +const postgresIntegrationTimeoutMs = 180_000; + +let postgresTestSchema: string | undefined; +let postgresTestUrl: string | undefined; + +const postgresConfig = (overrides: Partial = {}): IConfig => { + if (postgresUrl && !postgresTestUrl) { + throw new Error('Postgres test schema was not initialized'); + } + + const { database: databaseOverrides, ...rootOverrides } = overrides; + const database = postgresUrl + ? { + engine: 'postgres' as const, + inMemory: false, + connectionString: postgresTestUrl, + migrationPaths: [postgresMigrationsPath], + } + : { + engine: 'postgres' as const, + inMemory: true, + migrationPaths: [postgresMigrationsPath], + }; + + return { + port: 0, + extensions: [], + database: { ...database, ...(databaseOverrides ?? {}) }, + ...rootOverrides, + }; +}; + +const quoteTestSchemaIdentifier = (schema: string): string => { + if (!postgresTestSchemaPattern.test(schema)) { + throw new Error(`Unsafe Postgres test schema name: ${schema}`); + } + return `"${schema}"`; +}; + +const postgresConnectionStringForSchema = ( + connectionString: string, + schema: string, +): string => { + if (!postgresTestSchemaPattern.test(schema)) { + throw new Error(`Unsafe Postgres test schema name: ${schema}`); + } + + const url = new URL(connectionString); + url.searchParams.set('options', `-c search_path=${schema}`); + return url.toString(); +}; + +const createPostgresTestSchema = async (): Promise => { + if (!postgresUrl) return; + + const schema = `puter_test_${uuidv4().replaceAll('-', '')}`; + const pool = new Pool({ connectionString: postgresUrl }); + try { + await pool.query(`CREATE SCHEMA ${quoteTestSchemaIdentifier(schema)}`); + postgresTestSchema = schema; + postgresTestUrl = postgresConnectionStringForSchema( + postgresUrl, + schema, + ); + } finally { + await pool.end(); + } +}; + +const dropPostgresTestSchema = async (): Promise => { + if (!postgresUrl || !postgresTestSchema) return; + + const schema = postgresTestSchema; + postgresTestSchema = undefined; + postgresTestUrl = undefined; + + const pool = new Pool({ connectionString: postgresUrl }); + try { + await pool.query( + `DROP SCHEMA IF EXISTS ${quoteTestSchemaIdentifier(schema)} CASCADE`, + ); + } finally { + await pool.end(); + } +}; + +describe('PostgresDatabaseClient integration', () => { + let server: PuterServer | undefined; + + beforeEach(async () => { + await createPostgresTestSchema(); + }); + + afterEach(async () => { + try { + await server?.shutdown(); + } finally { + server = undefined; + await dropPostgresTestSchema(); + } + }); + + it( + 'applies the native migrations idempotently to an empty database', + async () => { + const config = postgresConfig(); + const pgMockClient = postgresUrl + ? undefined + : await createPgMockPostgresDatabaseClient(config); + try { + const firstClient = + pgMockClient?.client ?? new PostgresDatabaseClient(config); + try { + await firstClient.onServerStart(); + } finally { + await firstClient.onServerShutdown(); + } + + const secondClient = + pgMockClient?.createClient() ?? + new PostgresDatabaseClient(config); + try { + await secondClient.onServerStart(); + const [systemUser] = await secondClient.read( + 'SELECT `id`, `username` FROM `user` WHERE `username` = ?', + ['system'], + ); + const [devCenter] = await secondClient.read( + 'SELECT `name`, `index_url` FROM `apps` WHERE `name` = ?', + ['dev-center'], + ); + + expect(systemUser).toMatchObject({ + id: 1, + username: 'system', + }); + expect(devCenter).toMatchObject({ + name: 'dev-center', + index_url: + 'https://builtins.namespaces.puter.com/dev-center', + }); + } finally { + await secondClient.onServerShutdown(); + } + } finally { + pgMockClient?.destroy(); + } + }, + postgresIntegrationTimeoutMs, + ); + + it('starts the server and exercises user, app, fsentry, permission, and session flows', async () => { + const userStorageAllowance = 123_456_789; + server = await setupTestServer( + postgresConfig({ + no_default_user: false, + is_storage_limited: true, + }), + ); + + const admin = await server.stores.user.getByUsername('admin'); + expect(admin?.username).toBe('admin'); + + const username = `pg-${uuidv4().slice(0, 8)}`; + const createdUser = await server.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: `${username}@test.local`, + free_storage: userStorageAllowance, + requires_email_confirmation: false, + }); + await generateDefaultFsentries( + server.clients.db, + server.stores.user, + createdUser, + ); + const user = await server.stores.user.getById(createdUser.id); + if (!user) throw new Error('created user was not readable'); + expect(user.username).toBe(username); + await expect( + server.stores.fsEntry.getUserStorageAllowance(user.id), + ).resolves.toMatchObject({ + max: userStorageAllowance, + }); + + const otherUsername = `pg-other-${uuidv4().slice(0, 8)}`; + const otherUser = await server.stores.user.create({ + username: otherUsername, + uuid: uuidv4(), + password: null, + email: `${otherUsername}@test.local`, + free_storage: 100 * 1024 * 1024, + requires_email_confirmation: false, + }); + + const authResult = await server.services.auth.createSessionToken( + user, + { + ip: '127.0.0.1', + user_agent: 'postgres-integration-test', + }, + ); + const authenticated = + await server.services.auth.authenticateFromToken(authResult.token); + expect(authenticated?.user?.id).toBe(user?.id); + + const devCenter = await server.stores.app.getByName('dev-center'); + if (!devCenter) throw new Error('dev-center app was not seeded'); + expect(devCenter.name).toBe('dev-center'); + + const documents = await server.stores.fsEntry.getEntryByPath( + `/${username}/Documents`, + ); + if (!documents) throw new Error('Documents directory was not created'); + expect(documents.isDir).toBe(true); + const folder = await server.stores.fsEntry.createNonFileEntry({ + userId: user.id, + parent: documents, + name: 'postgres-folder', + kind: 'directory', + }); + const renamedFolder = await server.stores.fsEntry.updateEntry( + folder.uuid, + { + name: 'postgres-folder-renamed', + path: `/${username}/Documents/postgres-folder-renamed`, + }, + ); + expect(renamedFolder.name).toBe('postgres-folder-renamed'); + + await server.stores.permission.upsertUserAppPerm( + user.id, + Number(devCenter.id), + 'driver:postgres-integration', + { ok: true }, + ); + await expect( + server.stores.permission.hasUserAppPerm( + user.id, + Number(devCenter.id), + 'driver:postgres-integration', + ), + ).resolves.toBe(true); + + await server.stores.oidc.link( + user.id, + 'postgres-integration-test', + 'subject-1', + null, + ); + await expect( + server.stores.oidc.link( + user.id, + 'postgres-integration-test', + 'subject-1', + null, + ), + ).resolves.toBeUndefined(); + await expect( + server.stores.oidc.link( + otherUser.id, + 'postgres-integration-test', + 'subject-1', + null, + ), + ).rejects.toMatchObject({ + statusCode: 409, + legacyCode: 'conflict', + }); + + const session = await server.stores.session.create(user.id, { + meta: { source: 'postgres-integration-test' }, + }); + const activeSession = await server.stores.session.getByUuid( + session.uuid, + ); + expect(activeSession?.uuid).toBe(session.uuid); + + const workerName = `worker-${uuidv4().slice(0, 8)}`; + const workerSession = await server.stores.session.getOrCreateWorker( + user.id, + { workerName }, + ); + expect(workerSession?.kind).toBe('worker'); + expect(workerSession?.meta?.worker_name).toBe(workerName); + + await server.stores.session.removeByUuid(session.uuid); + await expect( + server.stores.session.getByUuid(session.uuid), + ).resolves.toBeNull(); + }, postgresIntegrationTimeoutMs); +}); diff --git a/src/backend/clients/database/PostgresDatabaseClient.test.ts b/src/backend/clients/database/PostgresDatabaseClient.test.ts new file mode 100644 index 0000000000..7f1bbb81ab --- /dev/null +++ b/src/backend/clients/database/PostgresDatabaseClient.test.ts @@ -0,0 +1,360 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { FieldDef, QueryResult } from 'pg'; +import { describe, expect, it } from 'vitest'; +import type { IConfig } from '../../types'; +import { DatabaseClientFactory } from './index.js'; +import { + mapPostgresWriteResult, + PostgresDatabaseClient, + type PostgresPool, + type PostgresPoolClient, +} from './PostgresDatabaseClient.js'; + +type QueryCall = { + text: string; + values?: unknown[]; +}; + +const postgresConfig = (): IConfig => ({ + port: 0, + extensions: [], + database: { + engine: 'postgres', + migrationPaths: [], + }, +}); + +const postgresReplicaConfig = (): IConfig => ({ + port: 0, + extensions: [], + database: { + engine: 'postgres', + migrationPaths: [], + replica: {}, + }, +}); + +const field = (name: string, dataTypeID: number): FieldDef => ({ + name, + tableID: 0, + columnID: 0, + dataTypeID, + dataTypeSize: -1, + dataTypeModifier: -1, + format: 'text', +}); + +const int8Field = (name: string): FieldDef => field(name, 20); +const textField = (name: string): FieldDef => field(name, 25); + +const queryResult = ( + rows: Record[] = [], + rowCount = rows.length, + fields: FieldDef[] = [], +): QueryResult> => ({ + command: '', + fields, + oid: 0, + rowCount, + rows, +}); + +class RecordingPoolClient implements PostgresPoolClient { + readonly calls: QueryCall[] = []; + released = false; + + constructor(private readonly failOnText?: string) {} + + async query( + text: string, + values?: unknown[], + ): Promise>> { + this.calls.push({ text, values }); + if (this.failOnText && text.includes(this.failOnText)) { + throw new Error(`forced query failure: ${text}`); + } + return queryResult([], text === 'ROLLBACK' ? 0 : 1); + } + + release(): void { + this.released = true; + } +} + +class RecordingPool implements PostgresPool { + readonly calls: QueryCall[] = []; + + constructor( + private readonly client: RecordingPoolClient = + new RecordingPoolClient(), + private readonly nextResult: QueryResult> = + queryResult(), + ) {} + + async query( + text: string, + values?: unknown[], + ): Promise>> { + this.calls.push({ text, values }); + if (text === 'SELECT 1') return queryResult([{ ok: 1 }], 1); + return this.nextResult; + } + + async connect(): Promise { + return this.client; + } + + async end(): Promise {} +} + +describe('PostgresDatabaseClient', () => { + it('is selected by the database factory', () => { + const client = new DatabaseClientFactory(postgresConfig()); + expect(client).toBeInstanceOf(PostgresDatabaseClient); + }); + + it('maps pg write results to the shared WriteResult shape', () => { + expect( + mapPostgresWriteResult(queryResult([{ id: '42' }], 1)), + ).toEqual({ + insertId: 42, + affectedRows: 1, + anyRowsAffected: true, + }); + + expect(mapPostgresWriteResult(queryResult([], 0))).toEqual({ + insertId: 0, + affectedRows: 0, + anyRowsAffected: false, + }); + }); + + it('prepares write SQL at the pg boundary', async () => { + const pool = new RecordingPool( + new RecordingPoolClient(), + queryResult([{ id: '7' }], 1), + ); + const client = new PostgresDatabaseClient(postgresConfig(), () => pool); + await client.onServerStart(); + + const result = await client.write( + 'INSERT INTO `apps` (`name`) VALUES (?) RETURNING id', + ['editor'], + ); + + expect(result.insertId).toBe(7); + expect(pool.calls.at(-1)).toEqual({ + text: 'INSERT INTO "apps" ("name") VALUES ($1) RETURNING id', + values: ['editor'], + }); + }); + + it('normalizes int8 fields on read and primary read rows', async () => { + const pool = new RecordingPool( + new RecordingPoolClient(), + queryResult( + [ + { + uuid: 'session-1', + created_at: '1710000000', + last_activity: '1710000001', + expires_at: '1710000002', + revoked_at: null, + }, + ], + 1, + [ + textField('uuid'), + int8Field('created_at'), + int8Field('last_activity'), + int8Field('expires_at'), + int8Field('revoked_at'), + ], + ), + ); + const client = new PostgresDatabaseClient(postgresConfig(), () => pool); + await client.onServerStart(); + + await expect(client.read('SELECT * FROM `sessions`')).resolves.toEqual( + [ + { + uuid: 'session-1', + created_at: 1710000000, + last_activity: 1710000001, + expires_at: 1710000002, + revoked_at: null, + }, + ], + ); + await expect(client.pread('SELECT * FROM `sessions`')).resolves.toEqual( + [ + { + uuid: 'session-1', + created_at: 1710000000, + last_activity: 1710000001, + expires_at: 1710000002, + revoked_at: null, + }, + ], + ); + }); + + it('rejects unsafe int8 values instead of losing precision', async () => { + const pool = new RecordingPool( + new RecordingPoolClient(), + queryResult( + [{ id: '9007199254740992' }], + 1, + [int8Field('id')], + ), + ); + const client = new PostgresDatabaseClient(postgresConfig(), () => pool); + await client.onServerStart(); + + await expect(client.read('SELECT `id` FROM `sessions`')).rejects.toThrow( + 'safe integer', + ); + }); + + it('normalizes tryHardRead rows returned by a replica', async () => { + const primaryPool = new RecordingPool( + new RecordingPoolClient(), + queryResult([{ created_at: '1' }], 1, [ + int8Field('created_at'), + ]), + ); + const replicaPool = new RecordingPool( + new RecordingPoolClient(), + queryResult([{ created_at: '2' }], 1, [ + int8Field('created_at'), + ]), + ); + const pools = [primaryPool, replicaPool]; + let nextPoolIndex = 0; + const client = new PostgresDatabaseClient( + postgresReplicaConfig(), + () => { + const pool = pools[nextPoolIndex]; + nextPoolIndex += 1; + if (!pool) throw new Error('unexpected pool factory call'); + return pool; + }, + ); + await client.onServerStart(); + + await expect( + client.tryHardRead('SELECT `created_at` FROM `sessions`'), + ).resolves.toEqual([{ created_at: 2 }]); + }); + + it('normalizes tryHardRead rows returned by primary fallback', async () => { + const primaryPool = new RecordingPool( + new RecordingPoolClient(), + queryResult([{ created_at: '3' }], 1, [ + int8Field('created_at'), + ]), + ); + const replicaPool = new RecordingPool( + new RecordingPoolClient(), + queryResult([], 0, [int8Field('created_at')]), + ); + const pools = [primaryPool, replicaPool]; + let nextPoolIndex = 0; + const client = new PostgresDatabaseClient( + postgresReplicaConfig(), + () => { + const pool = pools[nextPoolIndex]; + nextPoolIndex += 1; + if (!pool) throw new Error('unexpected pool factory call'); + return pool; + }, + ); + await client.onServerStart(); + + await expect( + client.tryHardRead('SELECT `created_at` FROM `sessions`'), + ).resolves.toEqual([{ created_at: 3 }]); + }); + + it('runs batch writes in order and commits', async () => { + const conn = new RecordingPoolClient(); + const client = new PostgresDatabaseClient( + postgresConfig(), + () => new RecordingPool(conn), + ); + await client.onServerStart(); + + await client.batchWrite([ + { + statement: + 'UPDATE `user` SET `username` = ? WHERE `id` = ?', + values: ['ada', 1], + }, + { + statement: 'DELETE FROM `sessions` WHERE `uuid` = ?', + values: ['session-1'], + }, + ]); + + expect(conn.calls).toEqual([ + { text: 'BEGIN', values: undefined }, + { + text: 'UPDATE "user" SET "username" = $1 WHERE "id" = $2', + values: ['ada', 1], + }, + { + text: 'DELETE FROM "sessions" WHERE "uuid" = $1', + values: ['session-1'], + }, + { text: 'COMMIT', values: undefined }, + ]); + expect(conn.released).toBe(true); + }); + + it('rolls back and releases the pg connection when a batch write fails', async () => { + const conn = new RecordingPoolClient('UPDATE "user"'); + const client = new PostgresDatabaseClient( + postgresConfig(), + () => new RecordingPool(conn), + ); + await client.onServerStart(); + + await expect( + client.batchWrite([ + { + statement: + 'UPDATE `user` SET `username` = ? WHERE `id` = ?', + values: ['ada', 1], + }, + ]), + ).rejects.toThrow('forced query failure'); + + expect(conn.calls).toEqual([ + { text: 'BEGIN', values: undefined }, + { + text: 'UPDATE "user" SET "username" = $1 WHERE "id" = $2', + values: ['ada', 1], + }, + { text: 'ROLLBACK', values: undefined }, + ]); + expect(conn.released).toBe(true); + }); +}); diff --git a/src/backend/clients/database/PostgresDatabaseClient.ts b/src/backend/clients/database/PostgresDatabaseClient.ts new file mode 100644 index 0000000000..1e6a506936 --- /dev/null +++ b/src/backend/clients/database/PostgresDatabaseClient.ts @@ -0,0 +1,390 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { readdirSync, readFileSync } from 'fs'; +import { isAbsolute, resolve as resolvePath } from 'path'; +import { Pool, type PoolConfig, type QueryResult } from 'pg'; +import { + AbstractDatabaseClient, + type BatchEntry, + type WriteResult, +} from './DatabaseClient'; +import { compareMigrationFilenames } from './migrationFilenames.js'; +import { preparePostgresSql } from './preparePostgresSql.js'; +import { splitPostgresStatements } from './splitPostgresStatements.js'; +import type { IConfig } from '../../types'; + +type PostgresEndpointConfig = { + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + connectionString?: string; + url?: string; +}; + +export interface PostgresQueryable { + query(query: string, values?: unknown[]): Promise; +} + +export interface PostgresPoolClient extends PostgresQueryable { + release(): void; +} + +export interface PostgresPool extends PostgresQueryable { + connect(): Promise; + end(): Promise; +} + +type PostgresPoolFactory = (poolConfig: PoolConfig) => PostgresPool; + +enum Configuration { + SINGLE, + REPLICA, +} + +const POSTGRES_INT8_OID = 20; +const INTEGER_TEXT_PATTERN = /^-?\d+$/u; + +const normalizePostgresInt8 = ( + value: unknown, + columnName: string, +): number | null | undefined => { + if (value === null || value === undefined) return value; + + const parsed = + typeof value === 'bigint' + ? Number(value) + : typeof value === 'number' + ? value + : typeof value === 'string' && INTEGER_TEXT_PATTERN.test(value) + ? Number(value) + : Number.NaN; + + if (!Number.isSafeInteger(parsed)) { + throw new Error( + `[postgres] int8 column ${columnName} is outside JavaScript's safe integer range`, + ); + } + + return parsed; +}; + +const normalizePostgresRows = ( + result: QueryResult, +): Record[] => { + const int8Fields = result.fields.filter( + (field) => field.dataTypeID === POSTGRES_INT8_OID, + ); + if (int8Fields.length === 0) { + return result.rows as Record[]; + } + + return result.rows.map((row) => { + const normalized: Record = { ...row }; + for (const field of int8Fields) { + normalized[field.name] = normalizePostgresInt8( + normalized[field.name], + field.name, + ); + } + return normalized; + }); +}; + +export const mapPostgresWriteResult = (result: QueryResult): WriteResult => { + const affectedRows = result.rowCount ?? 0; + const rowId = result.rows[0]?.id; + const insertId = + typeof rowId === 'bigint' + ? rowId + : typeof rowId === 'number' + ? rowId + : typeof rowId === 'string' && rowId !== '' + ? Number(rowId) + : 0; + const normalizedInsertId = + typeof insertId === 'number' && Number.isNaN(insertId) ? 0 : insertId; + + return { + insertId: normalizedInsertId, + affectedRows, + anyRowsAffected: affectedRows > 0, + }; +}; + +export class PostgresDatabaseClient extends AbstractDatabaseClient { + override readonly engineName = 'postgres'; + + private primaryPool!: PostgresPool; + private replicaPool!: PostgresPool; + private configuration = Configuration.SINGLE; + private shutdownStarted = false; + private shutdownTimer: ReturnType | null = null; + + constructor( + config: IConfig, + private readonly poolFactory: PostgresPoolFactory = (poolConfig) => + new Pool(poolConfig) as unknown as PostgresPool, + ) { + super(config); + } + + override async onServerStart(): Promise { + const dbConf = this.config.database!; + + this.primaryPool = this.createPool(dbConf); + await this.primaryPool.query('SELECT 1'); + console.log('[postgres] connected to primary'); + + if (dbConf.replica) { + this.replicaPool = this.createPool(dbConf.replica); + await this.replicaPool.query('SELECT 1'); + this.configuration = Configuration.REPLICA; + console.log('[postgres] connected to read-replica'); + } else { + this.replicaPool = this.primaryPool; + this.configuration = Configuration.SINGLE; + } + + await this.runMigrations(); + } + + override async onServerPrepareShutdown(): Promise { + if (this.shutdownStarted) return; + this.shutdownStarted = true; + + const drainMs = 60_000; + console.log( + `[postgres] draining in-flight queries (${drainMs}ms) before closing pools`, + ); + + this.shutdownTimer = setTimeout(() => { + this.shutdownTimer = null; + this.closeCurrentPools().catch((e) => + console.error('[postgres] error closing pools after drain', e), + ); + }, drainMs); + + if (typeof this.shutdownTimer.unref === 'function') { + this.shutdownTimer.unref(); + } + } + + override async onServerShutdown(): Promise { + if (this.shutdownTimer) { + clearTimeout(this.shutdownTimer); + this.shutdownTimer = null; + } + await this.closeCurrentPools(); + } + + override quoteIdentifier(identifier: string): string { + return identifier + .split('.') + .map((part) => { + if (part === '*') return part; + return `"${part.replaceAll('"', '""')}"`; + }) + .join('.'); + } + + override booleanLiteral(value: boolean): string { + return value ? 'TRUE' : 'FALSE'; + } + + override booleanValue(value: boolean): boolean { + return value; + } + + override async read( + query: string, + params: unknown[] = [], + ): Promise[]> { + const result = await this.query(this.replicaPool, query, params); + return normalizePostgresRows(result); + } + + override async pread( + query: string, + params: unknown[] = [], + ): Promise[]> { + const result = await this.query(this.primaryPool, query, params); + return normalizePostgresRows(result); + } + + override async write( + query: string, + params: unknown[] = [], + ): Promise { + const result = await this.query(this.primaryPool, query, params); + return mapPostgresWriteResult(result); + } + + override async batchWrite(entries: BatchEntry[]): Promise { + if (entries.length === 0) return; + + const conn = await this.primaryPool.connect(); + try { + await conn.query('BEGIN'); + try { + for (const { statement, values } of entries) { + await this.query(conn, statement, values); + } + await conn.query('COMMIT'); + } catch (err) { + await conn.query('ROLLBACK').catch(() => {}); + throw err; + } + } finally { + conn.release(); + } + } + + override async tryHardRead( + query: string, + params: unknown[] = [], + ): Promise[]> { + if (this.configuration === Configuration.SINGLE) { + return this.read(query, params); + } + + const primaryPromise = this.query(this.primaryPool, query, params); + try { + const replicaResult = await this.query( + this.replicaPool, + query, + params, + ); + if (replicaResult.rows.length > 0) { + primaryPromise.catch(() => {}); + return normalizePostgresRows(replicaResult); + } + } catch { + // fall through to primary + } + + const primaryResult = await primaryPromise; + return normalizePostgresRows(primaryResult); + } + + private async runMigrations(): Promise { + const paths = this.config.database?.migrationPaths; + if (!paths || paths.length === 0) return; + + const conn = await this.primaryPool.connect(); + try { + for (const rawPath of paths) { + const dir = isAbsolute(rawPath) + ? rawPath + : resolvePath(process.cwd(), rawPath); + + let files: string[]; + try { + files = readdirSync(dir) + .filter( + (f) => + f.endsWith('.sql') && f.startsWith('postgres'), + ) + .sort(compareMigrationFilenames); + } catch (e) { + throw new Error( + `[postgres] migration path is unreadable: ${dir}`, + { cause: e }, + ); + } + + if (files.length === 0) { + console.log(`[postgres] no migrations in ${dir}`); + continue; + } + + console.log( + `[postgres] running migrations from ${dir}: ${files.length} file(s)`, + ); + + for (const file of files) { + const filePath = resolvePath(dir, file); + const contents = readFileSync(filePath, 'utf8'); + const statements = splitPostgresStatements(contents); + await conn.query('BEGIN'); + try { + for (let i = 0; i < statements.length; i++) { + try { + await conn.query(statements[i]); + } catch (e) { + throw new Error( + `[postgres] failed to apply ${file} at statement ${i}`, + { cause: e }, + ); + } + } + await conn.query('COMMIT'); + } catch (e) { + await conn.query('ROLLBACK').catch(() => {}); + throw e; + } + console.log( + `[postgres] applied ${file} (${statements.length} statements)`, + ); + } + } + } finally { + conn.release(); + } + } + + private createPool(dbConf: PostgresEndpointConfig): PostgresPool { + const connectionString = dbConf.connectionString ?? dbConf.url; + if (connectionString) { + return this.poolFactory({ + connectionString, + max: 30, + }); + } + + return this.poolFactory({ + host: dbConf.host ?? '127.0.0.1', + port: dbConf.port ?? 5432, + user: dbConf.user ?? 'postgres', + password: dbConf.password ?? '', + database: dbConf.database ?? 'puter', + max: 30, + }); + } + + private async query( + target: PostgresQueryable, + query: string, + params: unknown[] = [], + ): Promise { + const prepared = preparePostgresSql(query); + return target.query(prepared.text, params); + } + + private async closeCurrentPools(): Promise { + const tasks: Promise[] = []; + if (this.primaryPool) tasks.push(this.primaryPool.end()); + if (this.replicaPool && this.replicaPool !== this.primaryPool) { + tasks.push(this.replicaPool.end()); + } + await Promise.all(tasks); + } +} diff --git a/src/backend/clients/database/index.ts b/src/backend/clients/database/index.ts index f801c49103..368e426ce0 100644 --- a/src/backend/clients/database/index.ts +++ b/src/backend/clients/database/index.ts @@ -24,10 +24,12 @@ export { } from './DatabaseClient'; export { SqliteDatabaseClient } from './SqliteDatabaseClient'; export { MySQLDatabaseClient } from './MySQLDatabaseClient'; +export { PostgresDatabaseClient } from './PostgresDatabaseClient'; import type { IConfig } from '../../types'; import { AbstractDatabaseClient } from './DatabaseClient'; import { MySQLDatabaseClient } from './MySQLDatabaseClient'; +import { PostgresDatabaseClient } from './PostgresDatabaseClient'; import { SqliteDatabaseClient } from './SqliteDatabaseClient'; /** @@ -41,6 +43,8 @@ export const DatabaseClientFactory = class DatabaseClientFactory { switch (engine) { case 'mysql': return new MySQLDatabaseClient(config); + case 'postgres': + return new PostgresDatabaseClient(config); case 'sqlite': return new SqliteDatabaseClient(config); default: diff --git a/src/backend/clients/database/migrationFilenames.ts b/src/backend/clients/database/migrationFilenames.ts new file mode 100644 index 0000000000..2f3cc31ec0 --- /dev/null +++ b/src/backend/clients/database/migrationFilenames.ts @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Comparator for migration filenames named `_mig_.sql`. + * + * Existing MySQL files use unpadded integers, so lexical sorting places + * `*_mig_10.sql` before `*_mig_2.sql`. Pull the trailing integer out and sort + * numerically. Anything that does not match the `_.sql` shape falls + * back to lexical comparison and sorts after numbered files. + */ +export const compareMigrationFilenames = (a: string, b: string): number => { + const numericIndex = (name: string): number => { + const m = /_(\d+)\.sql$/.exec(name); + return m ? Number.parseInt(m[1], 10) : Number.NaN; + }; + const na = numericIndex(a); + const nb = numericIndex(b); + if (Number.isFinite(na) && Number.isFinite(nb)) { + if (na !== nb) return na - nb; + } else if (Number.isFinite(na)) { + return -1; + } else if (Number.isFinite(nb)) { + return 1; + } + return a.localeCompare(b); +}; diff --git a/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql b/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql new file mode 100644 index 0000000000..177d4475d9 --- /dev/null +++ b/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql @@ -0,0 +1,724 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +CREATE TABLE IF NOT EXISTS "user" ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uuid varchar(36) NOT NULL UNIQUE, + username varchar(50) UNIQUE, + email varchar(256), + password varchar(225), + free_storage bigint, + max_subdomains integer, + taskbar_items text, + desktop_uuid varchar(36), + appdata_uuid varchar(36), + documents_uuid varchar(36), + pictures_uuid varchar(36), + videos_uuid varchar(36), + trash_uuid varchar(36), + trash_id integer, + appdata_id integer, + desktop_id integer, + documents_id integer, + pictures_id integer, + videos_id integer, + referrer varchar(64), + desktop_bg_url text, + desktop_bg_color varchar(20), + desktop_bg_fit varchar(16), + pass_recovery_token varchar(36), + requires_email_confirmation boolean NOT NULL DEFAULT FALSE, + email_confirm_code varchar(8), + email_confirm_token varchar(36), + email_confirmed boolean NOT NULL DEFAULT FALSE, + dev_first_name varchar(100), + dev_last_name varchar(100), + dev_paypal varchar(100), + dev_approved_for_incentive_program boolean DEFAULT FALSE, + dev_joined_incentive_program boolean DEFAULT FALSE, + suspended boolean, + unsubscribed boolean NOT NULL DEFAULT FALSE, + "timestamp" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_activity_ts timestamp, + referral_code varchar(16) UNIQUE, + referred_by integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + unconfirmed_change_email varchar(256), + change_email_confirm_token varchar(256), + otp_secret text, + otp_enabled boolean DEFAULT FALSE, + otp_recovery_codes text, + stripe_customer_id varchar(40), + public_uuid varchar(36), + public_id integer, + clean_email varchar(256), + audit_metadata jsonb, + signup_ip varchar(45), + signup_ip_forwarded varchar(45), + signup_user_agent varchar(512), + signup_origin varchar(255), + signup_server varchar(255), + metadata jsonb DEFAULT '{}'::jsonb, + reputation smallint DEFAULT 100 +); + +CREATE INDEX IF NOT EXISTS idx_user_email ON "user" (email); +CREATE INDEX IF NOT EXISTS idx_user_pass_recovery_token ON "user" (pass_recovery_token); +CREATE INDEX IF NOT EXISTS idx_user_referrer ON "user" (referrer); +CREATE INDEX IF NOT EXISTS idx_user_email_confirm_token ON "user" (email_confirm_token); +CREATE INDEX IF NOT EXISTS idx_user_last_activity_ts ON "user" (last_activity_ts); +CREATE INDEX IF NOT EXISTS idx_user_referred_by ON "user" (referred_by); +CREATE INDEX IF NOT EXISTS idx_user_stripe_customer_id ON "user" (stripe_customer_id); +CREATE INDEX IF NOT EXISTS idx_user_clean_email ON "user" (clean_email); +CREATE INDEX IF NOT EXISTS idx_user_signup_ip ON "user" (signup_ip); +CREATE INDEX IF NOT EXISTS idx_user_signup_ip_forwarded ON "user" (signup_ip_forwarded); +CREATE INDEX IF NOT EXISTS idx_user_signup_user_agent ON "user" (signup_user_agent); +CREATE INDEX IF NOT EXISTS idx_user_signup_origin ON "user" (signup_origin); +CREATE INDEX IF NOT EXISTS idx_user_signup_server ON "user" (signup_server); + +CREATE TABLE IF NOT EXISTS apps ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uid varchar(40) NOT NULL UNIQUE, + owner_user_id integer REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + icon text, + name varchar(100) NOT NULL UNIQUE, + title varchar(100) NOT NULL, + description text, + godmode boolean DEFAULT FALSE, + maximize_on_start boolean DEFAULT FALSE, + index_url text NOT NULL, + approved_for_listing boolean DEFAULT FALSE, + approved_for_opening_items boolean DEFAULT FALSE, + approved_for_incentive_program boolean DEFAULT FALSE, + "timestamp" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_review timestamp, + tags varchar(255), + app_owner integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + background boolean DEFAULT FALSE, + metadata jsonb, + protected boolean DEFAULT FALSE, + is_private boolean DEFAULT FALSE +); + +CREATE INDEX IF NOT EXISTS idx_apps_owner_user_id ON apps (owner_user_id); +CREATE INDEX IF NOT EXISTS idx_apps_app_owner ON apps (app_owner); +CREATE INDEX IF NOT EXISTS idx_apps_owner_timestamp ON apps (owner_user_id, "timestamp" DESC); +CREATE INDEX IF NOT EXISTS idx_apps_listing_timestamp ON apps (approved_for_listing, "timestamp" DESC); + +CREATE TABLE IF NOT EXISTS "group" ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uid varchar(40) UNIQUE, + owner_user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + extra jsonb, + metadata jsonb, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_group_owner_user_id ON "group" (owner_user_id); + +CREATE TABLE IF NOT EXISTS app_filetype_association ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + app_id integer NOT NULL REFERENCES apps (id) ON DELETE CASCADE ON UPDATE CASCADE, + type varchar(60) NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_app_filetype_association_app_id ON app_filetype_association (app_id); +CREATE INDEX IF NOT EXISTS idx_app_filetype_association_type ON app_filetype_association (type); + +CREATE TABLE IF NOT EXISTS app_opens ( + _id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + app_uid varchar(40) NOT NULL REFERENCES apps (uid) ON DELETE CASCADE ON UPDATE CASCADE, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + ts integer NOT NULL, + human_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_app_opens_user_id ON app_opens (user_id); +CREATE INDEX IF NOT EXISTS idx_app_opens_app_uid ON app_opens (app_uid); +CREATE INDEX IF NOT EXISTS idx_app_opens_uid_ts ON app_opens (app_uid, ts); +CREATE INDEX IF NOT EXISTS idx_app_opens_app_user ON app_opens (app_uid, user_id); + +CREATE TABLE IF NOT EXISTS fsentries ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uuid varchar(36) NOT NULL UNIQUE, + bucket varchar(50), + bucket_region varchar(30), + public_token varchar(36) UNIQUE, + file_request_token varchar(36) UNIQUE, + is_shortcut boolean DEFAULT FALSE, + shortcut_to integer REFERENCES fsentries (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + parent_id integer REFERENCES fsentries (id) ON DELETE CASCADE ON UPDATE CASCADE, + associated_app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + is_dir boolean DEFAULT FALSE, + layout varchar(30), + sort_by varchar(20), + sort_order varchar(10), + is_public boolean, + thumbnail text, + immutable boolean NOT NULL DEFAULT FALSE, + name varchar(767) NOT NULL, + metadata text, + modified integer NOT NULL, + created integer, + accessed integer, + size bigint, + symlink_path varchar(260), + is_symlink boolean DEFAULT FALSE, + parent_uid varchar(36), + path varchar(4096), + CONSTRAINT fsentries_parent_name_unique UNIQUE (parent_id, name) +); + +CREATE INDEX IF NOT EXISTS idx_fsentries_name ON fsentries (name); +CREATE INDEX IF NOT EXISTS idx_fsentries_modified ON fsentries (modified); +CREATE INDEX IF NOT EXISTS idx_fsentries_parent_id ON fsentries (parent_id); +CREATE INDEX IF NOT EXISTS idx_fsentries_is_dir ON fsentries (is_dir); +CREATE INDEX IF NOT EXISTS idx_fsentries_user_id ON fsentries (user_id); +CREATE INDEX IF NOT EXISTS idx_fsentries_shortcut_to ON fsentries (shortcut_to); +CREATE INDEX IF NOT EXISTS idx_fsentries_associated_app_id ON fsentries (associated_app_id); +CREATE INDEX IF NOT EXISTS idx_fsentries_bucket ON fsentries (bucket); +CREATE INDEX IF NOT EXISTS idx_fsentries_bucket_region ON fsentries (bucket_region); +CREATE INDEX IF NOT EXISTS idx_fsentries_parent_uid ON fsentries (parent_uid); +CREATE INDEX IF NOT EXISTS idx_fsentries_path ON fsentries (path); +CREATE INDEX IF NOT EXISTS idx_fsentries_accessed ON fsentries (accessed); +CREATE INDEX IF NOT EXISTS idx_fsentries_user_parent_name ON fsentries (user_id, parent_uid, name); +CREATE INDEX IF NOT EXISTS idx_fsentries_parent_uid_name ON fsentries (parent_uid, name); + +CREATE TABLE IF NOT EXISTS fsentry_versions ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + fsentry_id integer NOT NULL REFERENCES fsentries (id) ON DELETE CASCADE ON UPDATE CASCADE, + fsentry_uuid varchar(36) NOT NULL, + version_id varchar(60) NOT NULL, + user_id integer REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + message text, + ts_epoch integer +); + +CREATE INDEX IF NOT EXISTS idx_fsentry_versions_fsentry_id ON fsentry_versions (fsentry_id); +CREATE INDEX IF NOT EXISTS idx_fsentry_versions_fsentry_uuid ON fsentry_versions (fsentry_uuid); +CREATE INDEX IF NOT EXISTS idx_fsentry_versions_user_id ON fsentry_versions (user_id); + +CREATE TABLE IF NOT EXISTS subdomains ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uuid varchar(40) UNIQUE, + subdomain varchar(64) NOT NULL UNIQUE, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + root_dir_id integer REFERENCES fsentries (id) ON DELETE SET NULL ON UPDATE CASCADE, + associated_app_id integer REFERENCES apps (id) ON DELETE CASCADE ON UPDATE CASCADE, + ts timestamp DEFAULT CURRENT_TIMESTAMP, + app_owner integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + protected boolean DEFAULT FALSE, + domain varchar(265), + database_id varchar(40), + preamble_version varchar(64) +); + +CREATE INDEX IF NOT EXISTS idx_subdomains_user_id ON subdomains (user_id); +CREATE INDEX IF NOT EXISTS idx_subdomains_root_dir_id ON subdomains (root_dir_id); +CREATE INDEX IF NOT EXISTS idx_subdomains_associated_app_id ON subdomains (associated_app_id); +CREATE INDEX IF NOT EXISTS idx_subdomains_app_owner ON subdomains (app_owner); +CREATE INDEX IF NOT EXISTS idx_subdomains_domain ON subdomains (domain); +CREATE INDEX IF NOT EXISTS idx_subdomains_root_user ON subdomains (root_dir_id, user_id); +CREATE INDEX IF NOT EXISTS idx_subdomains_app_user ON subdomains (associated_app_id, user_id); + +CREATE TABLE IF NOT EXISTS kv ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + app varchar(40), + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + kkey_hash bigint NOT NULL, + kkey text NOT NULL, + value text, + migrated boolean DEFAULT FALSE, + "expireAt" timestamp, + CONSTRAINT kv_app_user_kkey_hash_unique UNIQUE (app, user_id, kkey_hash) +); + +CREATE INDEX IF NOT EXISTS idx_kv_app ON kv (app); +CREATE INDEX IF NOT EXISTS idx_kv_user_id ON kv (user_id); +CREATE INDEX IF NOT EXISTS idx_kv_kkey_hash ON kv (kkey_hash); + +CREATE TABLE IF NOT EXISTS sessions ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + uuid varchar(40) NOT NULL, + meta jsonb, + created_at bigint DEFAULT 0, + last_activity bigint DEFAULT 0, + kind varchar(32) NOT NULL DEFAULT 'web' + CHECK (kind IN ('web', 'app', 'access_token', 'asset', 'worker')), + label varchar(255), + parent_session_id varchar(64), + last_ip varchar(64), + last_user_agent varchar(512), + revoked_at bigint, + expires_at bigint, + app_uid varchar(64), + legacy_token_uid varchar(64), + created_via varchar(32), + auth_id varchar(64), + access_token_uid varchar(64) +); + +CREATE INDEX IF NOT EXISTS idx_sessions_user_id ON sessions (user_id); +CREATE INDEX IF NOT EXISTS idx_sessions_uuid ON sessions (uuid); +CREATE INDEX IF NOT EXISTS idx_sessions_user_revoked ON sessions (user_id, revoked_at); +CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions (parent_session_id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_user_app_active + ON sessions (user_id, app_uid) + WHERE kind = 'app' AND revoked_at IS NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_legacy_token_active + ON sessions (legacy_token_uid) + WHERE legacy_token_uid IS NOT NULL AND revoked_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_sessions_kind_user ON sessions (kind, user_id); +CREATE INDEX IF NOT EXISTS idx_sessions_access_token_uid + ON sessions (access_token_uid) + WHERE access_token_uid IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_user_worker_active + ON sessions (user_id, COALESCE(app_uid, ''), (meta #>> ARRAY['worker_name'])) + WHERE kind = 'worker' AND revoked_at IS NULL; + +CREATE TABLE IF NOT EXISTS user_to_app_permissions ( + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + app_id integer NOT NULL REFERENCES apps (id) ON DELETE CASCADE ON UPDATE CASCADE, + permission varchar(255) NOT NULL, + extra jsonb, + dt timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (user_id, app_id, permission) +); + +CREATE INDEX IF NOT EXISTS idx_utap_user_permission ON user_to_app_permissions (user_id, permission); +CREATE INDEX IF NOT EXISTS idx_utap_app_permission ON user_to_app_permissions (app_id, permission); + +CREATE TABLE IF NOT EXISTS user_to_group_permissions ( + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + group_id integer NOT NULL REFERENCES "group" (id) ON DELETE CASCADE ON UPDATE CASCADE, + permission varchar(255) NOT NULL, + extra jsonb, + PRIMARY KEY (user_id, group_id, permission) +); + +CREATE INDEX IF NOT EXISTS idx_user_to_group_permissions_group_id ON user_to_group_permissions (group_id); + +CREATE TABLE IF NOT EXISTS user_to_user_permissions ( + issuer_user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + holder_user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + permission varchar(255) NOT NULL, + extra jsonb, + PRIMARY KEY (issuer_user_id, holder_user_id, permission) +); + +CREATE INDEX IF NOT EXISTS idx_user_to_user_permissions_holder_user_id ON user_to_user_permissions (holder_user_id); + +CREATE TABLE IF NOT EXISTS dev_to_app_permissions ( + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + app_id integer NOT NULL REFERENCES apps (id) ON DELETE CASCADE ON UPDATE CASCADE, + permission varchar(255) NOT NULL, + extra jsonb, + PRIMARY KEY (user_id, app_id, permission) +); + +CREATE INDEX IF NOT EXISTS idx_dev_to_app_permissions_app_id ON dev_to_app_permissions (app_id); +CREATE INDEX IF NOT EXISTS idx_dev_to_app_permissions_permission ON dev_to_app_permissions (permission); + +CREATE TABLE IF NOT EXISTS access_token_permissions ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_uid varchar(40) NOT NULL, + authorizer_user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + authorizer_app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + permission varchar(255) NOT NULL, + extra jsonb, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_access_token_permissions_token_uid ON access_token_permissions (token_uid); + +CREATE TABLE IF NOT EXISTS old_app_names ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + app_uid varchar(40) NOT NULL REFERENCES apps (uid) ON DELETE CASCADE, + name varchar(255) NOT NULL, + "timestamp" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT unique_old_app_names_app_uid_name UNIQUE (app_uid, name) +); + +CREATE INDEX IF NOT EXISTS idx_old_app_names_app_uid ON old_app_names (app_uid); +CREATE INDEX IF NOT EXISTS idx_old_app_names_name ON old_app_names (name); + +CREATE TABLE IF NOT EXISTS jct_user_group ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + group_id integer NOT NULL REFERENCES "group" (id) ON DELETE CASCADE ON UPDATE CASCADE, + extra jsonb, + metadata jsonb, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_jct_user_group_user_id ON jct_user_group (user_id); +CREATE INDEX IF NOT EXISTS idx_jct_user_group_group_id ON jct_user_group (group_id); + +CREATE TABLE IF NOT EXISTS notification ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + uid varchar(40) UNIQUE, + value jsonb NOT NULL, + acknowledged bigint, + shown bigint, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_notification_user_id ON notification (user_id); + +CREATE TABLE IF NOT EXISTS share ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uid varchar(40) UNIQUE, + issuer_user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + recipient_email varchar(255) NOT NULL, + data jsonb, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_share_issuer_user_id ON share (issuer_user_id); +CREATE INDEX IF NOT EXISTS idx_share_recipient_email ON share (recipient_email); + +CREATE TABLE IF NOT EXISTS feedback ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + message text, + ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_feedback_user_id ON feedback (user_id); + +CREATE TABLE IF NOT EXISTS thread ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uid varchar(40) NOT NULL UNIQUE, + parent_uid varchar(40) REFERENCES thread (uid) ON DELETE CASCADE ON UPDATE CASCADE, + owner_user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + "schema" text, + text text NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_thread_parent_uid ON thread (parent_uid); +CREATE INDEX IF NOT EXISTS idx_thread_owner_user_id ON thread (owner_user_id); + +CREATE TABLE IF NOT EXISTS user_comments ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + uid varchar(40) NOT NULL UNIQUE, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + metadata jsonb, + text text NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_user_comments_user_id ON user_comments (user_id); +CREATE INDEX IF NOT EXISTS idx_user_comments_uid ON user_comments (uid); + +CREATE TABLE IF NOT EXISTS user_fsentry_comments ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_comment_id integer NOT NULL REFERENCES user_comments (id) ON DELETE CASCADE ON UPDATE CASCADE, + fsentry_id integer NOT NULL REFERENCES fsentries (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_user_fsentry_comments_user_comment_id ON user_fsentry_comments (user_comment_id); +CREATE INDEX IF NOT EXISTS idx_user_fsentry_comments_fsentry_id ON user_fsentry_comments (fsentry_id); + +CREATE TABLE IF NOT EXISTS user_fsentry_version_comments ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_comment_id integer NOT NULL REFERENCES user_comments (id) ON DELETE CASCADE ON UPDATE CASCADE, + fsentry_version_id integer NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_user_fsentry_version_comments_user_comment_id ON user_fsentry_version_comments (user_comment_id); +CREATE INDEX IF NOT EXISTS idx_user_fsentry_version_comments_version_id ON user_fsentry_version_comments (fsentry_version_id); + +CREATE TABLE IF NOT EXISTS user_group_comments ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_comment_id integer NOT NULL REFERENCES user_comments (id) ON DELETE CASCADE ON UPDATE CASCADE, + group_id integer NOT NULL REFERENCES "group" (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_user_group_comments_user_comment_id ON user_group_comments (user_comment_id); +CREATE INDEX IF NOT EXISTS idx_user_group_comments_group_id ON user_group_comments (group_id); + +CREATE TABLE IF NOT EXISTS user_user_comments ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_comment_id integer NOT NULL REFERENCES user_comments (id) ON DELETE CASCADE ON UPDATE CASCADE, + commented_user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_user_user_comments_user_comment_id ON user_user_comments (user_comment_id); +CREATE INDEX IF NOT EXISTS idx_user_user_comments_commented_user_id ON user_user_comments (commented_user_id); + +CREATE TABLE IF NOT EXISTS user_oidc_providers ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + provider varchar(64) NOT NULL, + provider_sub varchar(255) NOT NULL, + refresh_token text, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_user_oidc_providers_user_id ON user_oidc_providers (user_id); +CREATE INDEX IF NOT EXISTS idx_user_oidc_providers_provider ON user_oidc_providers (provider); +CREATE UNIQUE INDEX IF NOT EXISTS idx_user_oidc_providers_provider_sub_unique ON user_oidc_providers (provider, provider_sub); + +CREATE TABLE IF NOT EXISTS app_update_audit ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + app_id_keep integer NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + old_name varchar(50), + new_name varchar(50), + reason varchar(255) +); + +CREATE INDEX IF NOT EXISTS idx_app_update_audit_app_id ON app_update_audit (app_id); + +CREATE TABLE IF NOT EXISTS user_update_audit ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id_keep integer NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + old_email varchar(256), + new_email varchar(256), + old_username varchar(50), + new_username varchar(50), + reason varchar(255) +); + +CREATE INDEX IF NOT EXISTS idx_user_update_audit_user_id ON user_update_audit (user_id); + +CREATE TABLE IF NOT EXISTS storage_audit ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id_keep integer NOT NULL, + is_subtract boolean NOT NULL DEFAULT FALSE, + amount bigint NOT NULL, + field_a varchar(16), + field_b varchar(16), + reason varchar(255), + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_storage_audit_user_id ON storage_audit (user_id); + +CREATE TABLE IF NOT EXISTS audit_user_to_app_permissions ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id_keep integer NOT NULL, + app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + app_id_keep integer NOT NULL, + permission varchar(255) NOT NULL, + extra jsonb, + action varchar(16), + reason varchar(255), + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_audit_user_to_app_permissions_user_id ON audit_user_to_app_permissions (user_id); +CREATE INDEX IF NOT EXISTS idx_audit_user_to_app_permissions_app_id ON audit_user_to_app_permissions (app_id); + +CREATE TABLE IF NOT EXISTS audit_dev_to_app_permissions ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id_keep integer NOT NULL, + app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + app_id_keep integer NOT NULL, + permission varchar(255) NOT NULL, + extra jsonb, + action varchar(16), + reason varchar(255), + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_audit_dev_to_app_permissions_user_id ON audit_dev_to_app_permissions (user_id); +CREATE INDEX IF NOT EXISTS idx_audit_dev_to_app_permissions_app_id ON audit_dev_to_app_permissions (app_id); + +CREATE TABLE IF NOT EXISTS audit_user_to_group_permissions ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id_keep integer NOT NULL, + group_id integer REFERENCES "group" (id) ON DELETE SET NULL ON UPDATE CASCADE, + group_id_keep integer NOT NULL, + permission varchar(255) NOT NULL, + extra jsonb, + action varchar(255), + reason varchar(255), + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_audit_user_to_group_permissions_user_id ON audit_user_to_group_permissions (user_id); +CREATE INDEX IF NOT EXISTS idx_audit_user_to_group_permissions_group_id ON audit_user_to_group_permissions (group_id); + +CREATE TABLE IF NOT EXISTS audit_user_to_user_permissions ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + issuer_user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + issuer_user_id_keep integer NOT NULL, + holder_user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + holder_user_id_keep integer NOT NULL, + permission varchar(255) NOT NULL, + extra jsonb, + action varchar(16), + reason varchar(255), + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_audit_user_to_user_permissions_issuer_user_id ON audit_user_to_user_permissions (issuer_user_id); +CREATE INDEX IF NOT EXISTS idx_audit_user_to_user_permissions_holder_user_id ON audit_user_to_user_permissions (holder_user_id); + +CREATE TABLE IF NOT EXISTS ai_usage ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + service_name varchar(64), + model_name varchar(128), + price_modifier varchar(40), + cost integer, + value_uint_1 integer, + value_uint_2 integer, + value_uint_3 integer, + value_uint_4 integer, + value_uint_5 integer, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_ai_usage_app_id ON ai_usage (app_id); +CREATE INDEX IF NOT EXISTS idx_ai_usage_service_name ON ai_usage (service_name); +CREATE INDEX IF NOT EXISTS idx_ai_usage_model_name ON ai_usage (model_name); +CREATE INDEX IF NOT EXISTS idx_ai_usage_price_modifier ON ai_usage (price_modifier); +CREATE INDEX IF NOT EXISTS idx_ai_usage_created_at ON ai_usage (created_at); +CREATE INDEX IF NOT EXISTS idx_ai_usage_user_timestamp ON ai_usage (user_id, created_at); + +CREATE TABLE IF NOT EXISTS general_analytics ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + uid varchar(40) NOT NULL, + trace_id varchar(40), + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + user_id_keep integer, + app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + app_id_keep integer, + server_id varchar(40), + actor_type varchar(40), + tags jsonb, + fields jsonb +); + +CREATE INDEX IF NOT EXISTS idx_general_analytics_user_id ON general_analytics (user_id); +CREATE INDEX IF NOT EXISTS idx_general_analytics_app_id ON general_analytics (app_id); + +CREATE TABLE IF NOT EXISTS monthly_usage_counts ( + year integer NOT NULL, + month integer NOT NULL, + service_type varchar(40) NOT NULL, + service_name varchar(40) NOT NULL, + actor_key varchar(255) NOT NULL, + pricing_category jsonb NOT NULL, + pricing_category_hash bytea NOT NULL, + "count" integer DEFAULT 0, + value_uint_1 integer, + value_uint_2 integer, + value_uint_3 integer, + PRIMARY KEY (year, month, service_type, service_name, actor_key, pricing_category_hash) +); + +CREATE TABLE IF NOT EXISTS service_usage_monthly ( + "key" varchar(255) NOT NULL, + year integer NOT NULL, + month integer NOT NULL, + user_id integer REFERENCES "user" (id) ON DELETE SET NULL ON UPDATE CASCADE, + app_id integer REFERENCES apps (id) ON DELETE SET NULL ON UPDATE CASCADE, + "count" integer NOT NULL, + extra jsonb, + PRIMARY KEY ("key", year, month) +); + +CREATE INDEX IF NOT EXISTS idx_service_usage_monthly_user_id ON service_usage_monthly (user_id); +CREATE INDEX IF NOT EXISTS idx_service_usage_monthly_app_id ON service_usage_monthly (app_id); + +CREATE TABLE IF NOT EXISTS per_user_credit ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id integer NOT NULL UNIQUE REFERENCES "user" (id) ON DELETE CASCADE ON UPDATE CASCADE, + amount bigint NOT NULL, + last_updated_at bigint NOT NULL +); + +INSERT INTO "user" (id, uuid, username, metadata) +VALUES (1, '5d4adce0-a381-4982-9c02-6e2540026238', 'system', '{}'::jsonb) +ON CONFLICT (uuid) DO UPDATE SET username = EXCLUDED.username; + +SELECT setval(pg_get_serial_sequence('"user"', 'id'), COALESCE((SELECT MAX(id) FROM "user"), 1), true); + +INSERT INTO "group" (uid, owner_user_id, extra, metadata) +VALUES + ('26bfb1fb-421f-45bc-9aa4-d81ea569e7a5', 1, '{"critical": true, "type": "default", "name": "system"}'::jsonb, '{"title": "System", "color": "#000000"}'::jsonb), + ('ca342a5e-b13d-4dee-9048-58b11a57cc55', 1, '{"critical": true, "type": "default", "name": "admin"}'::jsonb, '{"title": "Admin", "color": "#a83232"}'::jsonb), + ('78b1b1dd-c959-44d2-b02c-8735671f9997', 1, '{"critical": true, "type": "default", "name": "user"}'::jsonb, '{"title": "User", "color": "#3254a8"}'::jsonb), + ('b7220104-7905-4985-b996-649fdcdb3c8f', 1, '{"critical": true, "type": "default", "name": "temp"}'::jsonb, '{"title": "Temp", "color": "#888888"}'::jsonb), + ('3c2dfff7-d22a-41aa-a193-59a61dac4b64', 1, '{"type": "default", "name": "moderator"}'::jsonb, '{"title": "Moderator", "color": "#a432a8"}'::jsonb), + ('5e8f251d-3382-4b0d-932c-7bb82f48652f', 1, '{"type": "default", "name": "developer"}'::jsonb, '{"title": "Developer", "color": "#32a852"}'::jsonb) +ON CONFLICT (uid) DO UPDATE SET + owner_user_id = EXCLUDED.owner_user_id, + extra = EXCLUDED.extra, + metadata = EXCLUDED.metadata; + +INSERT INTO apps ( + uid, owner_user_id, icon, name, title, description, index_url, + godmode, maximize_on_start, background, + approved_for_listing, approved_for_opening_items, + approved_for_incentive_program, tags, "timestamp" +) +VALUES + ('app-3920851d-bda8-479b-9407-8517293c7d44', 1, NULL, 'pdf', 'PDF', '', 'https://pdf.puter.com', FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-5584fbf7-ed69-41fc-99cd-85da21b1ef51', 1, NULL, 'camera', 'Camera', 'Camera in the browser.', 'https://online-camera.com', FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-11edfba2-1ed3-4e22-8573-47e88fb87d70', 1, NULL, 'player', 'Player', 'A free video player app in the browser.', 'https://simple-player.puter.com', FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-7bdca1a4-6373-4c98-ad97-03ff2d608ca1', 1, NULL, 'recorder', 'Recorder', 'Online voice recorder in the browser with cloud storage.', 'https://voice-recorder.com', FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-d7e9471f-e441-4d72-a5ab-75e96573b76b', 1, NULL, 'music-player', 'Music Player', 'A free music player app in the browser.', 'https://player.puter.com', FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, NULL, '2026-05-10 00:00:00'), + ('app-e3ac5486-da8c-42ad-8377-8728086e0980', 1, NULL, 'git', 'Git', 'Puter Git client', 'https://builtins.namespaces.puter.com/git', FALSE, FALSE, TRUE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-0b37f054-07d4-4627-8765-11bd23e889d4', 1, NULL, 'dev-center', 'Dev Center', 'This is the app that makes apps', 'https://builtins.namespaces.puter.com/dev-center', TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-fbbdb72b-ad08-4cb4-86a1-de0f27cf2e1e', 1, NULL, 'puter-linux', 'Puter Linux', 'Linux emulator for Puter', 'https://builtins.namespaces.puter.com/emulator', TRUE, FALSE, FALSE, TRUE, TRUE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-838dfbc4-bf8b-48c2-b47b-c4adc77fab58', 1, NULL, 'editor', 'Editor', 'Text editor', 'https://online-notepad.com', TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-58282b08-990a-4906-95f7-fa37ff92452b', 1, NULL, 'draw', 'Draw', 'Image editor', 'https://draw.puter.com', TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, 'graphics', '2020-01-01 00:00:00'), + ('app-0bef044f-918f-4cbf-a0c0-b4a17ee81085', 1, NULL, 'about', 'About', 'About Puter', 'https://about.puter.com', TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-a2ae72a4-1ba3-4a29-b5c0-6de1be5cf178', 1, NULL, 'app-center', 'App Center', 'Discover apps for Puter', 'https://app-center.puter.com', TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, NULL, '2020-01-01 00:00:00'), + ('app-93005ce0-80d1-50d9-9b1e-9c453c375d56', 1, NULL, 'markus', 'Markus', 'Markdown editor', 'https://markus.puter.com', TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-6f79ef7b-52b7-4b31-91c6-fc07b62e9396', 1, NULL, 'code', 'Code', 'Code editor', 'https://code.puter.com', TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-862fc09e-c7b8-4c30-b5b4-47b3cc5f5232', 1, NULL, 'memos', 'Memos', 'Notes and memos', 'https://memos.puter.com', FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-aeb8c03e-1144-4c57-bff5-3a1a7c17f9f0', 1, NULL, 'word-processor', 'Word Processor', 'Write documents', 'https://word-processor.puter.com', FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-c10a1999-22f5-4644-9960-9aaac0d4934e', 1, NULL, 'spreadsheet', 'Spreadsheet', 'Work with spreadsheets', 'https://spreadsheet.puter.com', FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-e9c1d58d-3d8d-4f0a-a85c-e383ef63bc29', 1, NULL, 'presentation', 'Presentation', 'Create presentations', 'https://presentation.puter.com', FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00'), + ('app-fb7d9e42-8207-4fa0-b680-f0239327097f', 1, NULL, 'pdf-editor', 'PDF Editor', 'Edit PDF documents', 'https://pdf-editor.puter.com', FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, 'productivity', '2020-01-01 00:00:00') +ON CONFLICT (uid) DO UPDATE SET + owner_user_id = EXCLUDED.owner_user_id, + name = EXCLUDED.name, + title = EXCLUDED.title, + description = EXCLUDED.description, + index_url = EXCLUDED.index_url, + godmode = EXCLUDED.godmode, + maximize_on_start = EXCLUDED.maximize_on_start, + background = EXCLUDED.background, + approved_for_listing = EXCLUDED.approved_for_listing, + approved_for_opening_items = EXCLUDED.approved_for_opening_items, + approved_for_incentive_program = EXCLUDED.approved_for_incentive_program, + tags = EXCLUDED.tags; + +INSERT INTO user_to_group_permissions (user_id, group_id, permission, extra) +SELECT u.id, g.id, 'driver', '{}'::jsonb +FROM "user" u, "group" g +WHERE u.username = 'system' + AND g.uid = 'ca342a5e-b13d-4dee-9048-58b11a57cc55' +ON CONFLICT (user_id, group_id, permission) DO UPDATE SET extra = EXCLUDED.extra; diff --git a/src/backend/clients/database/preparePostgresSql.test.ts b/src/backend/clients/database/preparePostgresSql.test.ts new file mode 100644 index 0000000000..d52d89219c --- /dev/null +++ b/src/backend/clients/database/preparePostgresSql.test.ts @@ -0,0 +1,85 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; +import { preparePostgresSql } from './preparePostgresSql.js'; + +describe('preparePostgresSql', () => { + it('converts placeholders and backtick identifiers outside SQL literals', () => { + const prepared = preparePostgresSql( + 'SELECT `user`.`id` FROM `user` WHERE `email` = ? AND `username` = ?', + ); + + expect(prepared).toEqual({ + text: 'SELECT "user"."id" FROM "user" WHERE "email" = $1 AND "username" = $2', + parameterCount: 2, + }); + }); + + it('leaves question marks inside strings and comments untouched', () => { + const prepared = preparePostgresSql(` + SELECT '?' AS literal, ? + -- ? in a comment + FROM \`apps\` + WHERE \`description\` = 'is this ok?' + /* and ? in a block comment */ + AND \`name\` = ? + `); + + expect(prepared.text).toContain("SELECT '?' AS literal, $1"); + expect(prepared.text).toContain('-- ? in a comment'); + expect(prepared.text).toContain('"description" = \'is this ok?\''); + expect(prepared.text).toContain('/* and ? in a block comment */'); + expect(prepared.text).toContain('"name" = $2'); + expect(prepared.parameterCount).toBe(2); + }); + + it('escapes double quotes inside converted identifiers', () => { + const prepared = preparePostgresSql( + 'SELECT `odd"name` FROM `odd``table` WHERE `id` = ?', + ); + + expect(prepared).toEqual({ + text: 'SELECT "odd""name" FROM "odd`table" WHERE "id" = $1', + parameterCount: 1, + }); + }); + + it('does not scan placeholders inside dollar-quoted strings', () => { + const prepared = preparePostgresSql( + "SELECT $$?$$, $tag$`not_ident` ?$tag$, `id` FROM `user` WHERE `id` = ?", + ); + + expect(prepared).toEqual({ + text: 'SELECT $$?$$, $tag$`not_ident` ?$tag$, "id" FROM "user" WHERE "id" = $1', + parameterCount: 1, + }); + }); + + it('does not treat Postgres JSON operators as comments', () => { + const prepared = preparePostgresSql( + "SELECT * FROM `sessions` WHERE `user_id` = ? AND `meta` #>> ARRAY['worker_name'] = ? AND `expires_at` > ?", + ); + + expect(prepared).toEqual({ + text: 'SELECT * FROM "sessions" WHERE "user_id" = $1 AND "meta" #>> ARRAY[\'worker_name\'] = $2 AND "expires_at" > $3', + parameterCount: 3, + }); + }); +}); diff --git a/src/backend/clients/database/preparePostgresSql.ts b/src/backend/clients/database/preparePostgresSql.ts new file mode 100644 index 0000000000..dab129132b --- /dev/null +++ b/src/backend/clients/database/preparePostgresSql.ts @@ -0,0 +1,169 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +export interface PreparedPostgresSql { + text: string; + parameterCount: number; +} + +type ScannerState = + | { kind: 'normal' } + | { kind: 'singleQuote' } + | { kind: 'doubleQuote' } + | { kind: 'backtickIdentifier' } + | { kind: 'lineComment' } + | { kind: 'blockComment' } + | { kind: 'dollarQuote'; tag: string }; + +const isLineCommentStart = (sql: string, index: number): boolean => { + if (sql[index] !== '-' || sql[index + 1] !== '-') return false; + const after = sql[index + 2]; + return after === undefined || /\s/u.test(after); +}; + +const readDollarQuoteTag = (sql: string, index: number): string | null => { + const match = /^\$[A-Za-z_][A-Za-z0-9_]*\$|^\$\$/u.exec(sql.slice(index)); + return match?.[0] ?? null; +}; + +export const preparePostgresSql = (sql: string): PreparedPostgresSql => { + let state: ScannerState = { kind: 'normal' }; + let parameterCount = 0; + let out = ''; + + for (let i = 0; i < sql.length; i++) { + const char = sql[i]!; + const next = sql[i + 1]; + + switch (state.kind) { + case 'normal': { + const dollarTag = + char === '$' ? readDollarQuoteTag(sql, i) : null; + if (dollarTag) { + out += dollarTag; + i += dollarTag.length - 1; + state = { kind: 'dollarQuote', tag: dollarTag }; + break; + } + if (char === "'") { + out += char; + state = { kind: 'singleQuote' }; + break; + } + if (char === '"') { + out += char; + state = { kind: 'doubleQuote' }; + break; + } + if (char === '`') { + out += '"'; + state = { kind: 'backtickIdentifier' }; + break; + } + if (isLineCommentStart(sql, i)) { + out += char; + if (char === '-') { + out += next ?? ''; + i += 1; + } + state = { kind: 'lineComment' }; + break; + } + if (char === '/' && next === '*') { + out += '/*'; + i += 1; + state = { kind: 'blockComment' }; + break; + } + if (char === '?') { + parameterCount += 1; + out += `$${parameterCount}`; + break; + } + out += char; + break; + } + + case 'singleQuote': + out += char; + if (char === "'" && next === "'") { + out += next; + i += 1; + } else if (char === '\\' && next !== undefined) { + out += next; + i += 1; + } else if (char === "'") { + state = { kind: 'normal' }; + } + break; + + case 'doubleQuote': + out += char; + if (char === '"' && next === '"') { + out += next; + i += 1; + } else if (char === '"') { + state = { kind: 'normal' }; + } + break; + + case 'backtickIdentifier': + if (char === '`' && next === '`') { + out += '`'; + i += 1; + } else if (char === '`') { + out += '"'; + state = { kind: 'normal' }; + } else if (char === '"') { + out += '""'; + } else { + out += char; + } + break; + + case 'lineComment': + out += char; + if (char === '\n') { + state = { kind: 'normal' }; + } + break; + + case 'blockComment': + out += char; + if (char === '*' && next === '/') { + out += '/'; + i += 1; + state = { kind: 'normal' }; + } + break; + + case 'dollarQuote': + if (sql.startsWith(state.tag, i)) { + out += state.tag; + i += state.tag.length - 1; + state = { kind: 'normal' }; + } else { + out += char; + } + break; + } + } + + return { text: out, parameterCount }; +}; diff --git a/src/backend/clients/database/splitPostgresStatements.ts b/src/backend/clients/database/splitPostgresStatements.ts new file mode 100644 index 0000000000..1d47d46222 --- /dev/null +++ b/src/backend/clients/database/splitPostgresStatements.ts @@ -0,0 +1,120 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +type ScannerState = + | { kind: 'normal' } + | { kind: 'singleQuote' } + | { kind: 'doubleQuote' } + | { kind: 'lineComment' } + | { kind: 'blockComment' } + | { kind: 'dollarQuote'; tag: string }; + +const readDollarQuoteTag = (sql: string, index: number): string | null => { + const match = /^\$[A-Za-z_][A-Za-z0-9_]*\$|^\$\$/u.exec(sql.slice(index)); + return match?.[0] ?? null; +}; + +export const splitPostgresStatements = (sql: string): string[] => { + const statements: string[] = []; + let state: ScannerState = { kind: 'normal' }; + let start = 0; + + for (let i = 0; i < sql.length; i++) { + const char = sql[i]!; + const next = sql[i + 1]; + + switch (state.kind) { + case 'normal': { + const dollarTag = + char === '$' ? readDollarQuoteTag(sql, i) : null; + if (dollarTag) { + i += dollarTag.length - 1; + state = { kind: 'dollarQuote', tag: dollarTag }; + break; + } + if (char === "'") { + state = { kind: 'singleQuote' }; + break; + } + if (char === '"') { + state = { kind: 'doubleQuote' }; + break; + } + if (char === '-' && next === '-') { + i += 1; + state = { kind: 'lineComment' }; + break; + } + if (char === '/' && next === '*') { + i += 1; + state = { kind: 'blockComment' }; + break; + } + if (char === ';') { + const statement = sql.slice(start, i).trim(); + if (statement !== '') statements.push(statement); + start = i + 1; + } + break; + } + + case 'singleQuote': + if (char === "'" && next === "'") { + i += 1; + } else if (char === '\\' && next !== undefined) { + i += 1; + } else if (char === "'") { + state = { kind: 'normal' }; + } + break; + + case 'doubleQuote': + if (char === '"' && next === '"') { + i += 1; + } else if (char === '"') { + state = { kind: 'normal' }; + } + break; + + case 'lineComment': + if (char === '\n') { + state = { kind: 'normal' }; + } + break; + + case 'blockComment': + if (char === '*' && next === '/') { + i += 1; + state = { kind: 'normal' }; + } + break; + + case 'dollarQuote': + if (sql.startsWith(state.tag, i)) { + i += state.tag.length - 1; + state = { kind: 'normal' }; + } + break; + } + } + + const finalStatement = sql.slice(start).trim(); + if (finalStatement !== '') statements.push(finalStatement); + return statements; +}; diff --git a/src/backend/controllers/auth/AuthController.ts b/src/backend/controllers/auth/AuthController.ts index 002bf97ab0..361f56d4fb 100644 --- a/src/backend/controllers/auth/AuthController.ts +++ b/src/backend/controllers/auth/AuthController.ts @@ -2335,8 +2335,8 @@ export class AuthController extends PuterController { } await this.clients.db.write( - 'UPDATE `user` SET `otp_enabled` = 1 WHERE `uuid` = ?', - [user.uuid], + 'UPDATE `user` SET `otp_enabled` = ? WHERE `uuid` = ?', + [this.clients.db.booleanValue(true), user.uuid], ); await this.stores.user.invalidateById(user.id); @@ -2371,8 +2371,8 @@ export class AuthController extends PuterController { }); await this.clients.db.write( - 'UPDATE `user` SET `otp_enabled` = 0, `otp_recovery_codes` = NULL, `otp_secret` = NULL WHERE `uuid` = ?', - [user.uuid], + 'UPDATE `user` SET `otp_enabled` = ?, `otp_recovery_codes` = NULL, `otp_secret` = NULL WHERE `uuid` = ?', + [this.clients.db.booleanValue(false), user.uuid], ); await this.stores.user.invalidateById(user.id); diff --git a/src/backend/controllers/static/StaticPagesController.ts b/src/backend/controllers/static/StaticPagesController.ts index 92b423f72a..a53e84d176 100644 --- a/src/backend/controllers/static/StaticPagesController.ts +++ b/src/backend/controllers/static/StaticPagesController.ts @@ -165,7 +165,7 @@ export class StaticPagesController extends PuterController { const domain = this.config.domain ?? req.hostname; const origin = `${req.protocol}://${domain}`; const apps = (await this.clients.db.read( - 'SELECT `name` FROM `apps` WHERE `approved_for_listing` = 1', + `SELECT \`name\` FROM \`apps\` WHERE \`approved_for_listing\` = ${this.clients.db.booleanLiteral(true)}`, )) as Array<{ name: string }>; const urls = [ `${req.protocol}://docs.${domain}/`, @@ -252,7 +252,7 @@ export class StaticPagesController extends PuterController { const [dupe] = (await this.clients.db.read( `SELECT EXISTS( SELECT 1 FROM \`user\` WHERE (\`email\` = ? OR \`clean_email\` = ?) - AND \`email_confirmed\` = 1 + AND \`email_confirmed\` = ${this.clients.db.booleanLiteral(true)} AND \`password\` IS NOT NULL ) AS email_exists`, [user.email, cleanEmail], diff --git a/src/backend/core/http/middleware/privateAppGate.ts b/src/backend/core/http/middleware/privateAppGate.ts index f660463f8b..ba1e6cfe5c 100644 --- a/src/backend/core/http/middleware/privateAppGate.ts +++ b/src/backend/core/http/middleware/privateAppGate.ts @@ -243,7 +243,9 @@ export async function resolveOwnedAppForHostedSite(opts: { const uniqueCandidates = [...new Set(urlCandidates)]; if (uniqueCandidates.length === 0) return null; - const privateFilter = opts.requirePrivate ? 'AND `is_private` = 1 ' : ''; + const privateFilter = opts.requirePrivate + ? `AND \`is_private\` = ${opts.db.booleanLiteral(true)} ` + : ''; const placeholders = uniqueCandidates.map(() => '?').join(', '); const rows = await opts.db.read( `SELECT * FROM apps WHERE owner_user_id = ? ${privateFilter}AND index_url IN (${placeholders}) LIMIT 2`, diff --git a/src/backend/package.json b/src/backend/package.json index f46e7e5b6b..adf1697a9b 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -61,6 +61,7 @@ "openai": "^6.34.0", "otpauth": "^9.2.4", "parse-domain": "^8.2.2", + "pg": "^8.21.0", "prompt-sync": "^4.2.0", "replicate": "^1.0.0", "sharp": "^0.34.5", @@ -77,9 +78,11 @@ "@types/busboy": "^1.5.4", "@types/jsonwebtoken": "^9.0.10", "@types/node": "^24.0.0", + "@types/pg": "^8.6.1", "@types/validator": "^13.15.10", "chai": "^4.3.7", "nodemon": "^3.1.0", + "pgmock": "^1.0.3", "typescript": "^5.9.3", "vite": "^8.0.0", "vitest": "^4.0.14" diff --git a/src/backend/services/fs/FSService.ts b/src/backend/services/fs/FSService.ts index b84c8ad72f..6485ddd5f0 100644 --- a/src/backend/services/fs/FSService.ts +++ b/src/backend/services/fs/FSService.ts @@ -3080,11 +3080,12 @@ export class FSService extends PuterService { */ async removeAllForUser(userId: number): Promise { const pageSize = 5000; + const falseLiteral = this.clients.db.booleanLiteral(false); // Files-first loop: delete backing S3 objects in batches, then DB rows. for (;;) { const files = (await this.clients.db.read( `SELECT uuid, bucket, bucket_region FROM fsentries - WHERE user_id = ? AND is_dir = 0 AND (is_shortcut = 0 OR is_shortcut IS NULL) AND (is_symlink = 0 OR is_symlink IS NULL) + WHERE user_id = ? AND is_dir = ${falseLiteral} AND (is_shortcut = ${falseLiteral} OR is_shortcut IS NULL) AND (is_symlink = ${falseLiteral} OR is_symlink IS NULL) LIMIT ${pageSize}`, [userId], )) as Array<{ diff --git a/src/backend/stores/app/AppStore.js b/src/backend/stores/app/AppStore.js index d7106efa94..c114b54475 100644 --- a/src/backend/stores/app/AppStore.js +++ b/src/backend/stores/app/AppStore.js @@ -106,6 +106,16 @@ const READ_ONLY_COLUMNS = new Set([ 'protected', 'is_private', ]); +const APP_BOOLEAN_COLUMNS = new Set([ + 'godmode', + 'maximize_on_start', + 'approved_for_listing', + 'approved_for_opening_items', + 'approved_for_incentive_program', + 'background', + 'protected', + 'is_private', +]); export class AppStore extends PuterStore { #appStatsInterval; @@ -341,7 +351,7 @@ export class AppStore extends PuterStore { const colList = columns.map((c) => `\`${c}\``).join(', '); const result = await this.clients.db.write( - `INSERT INTO \`apps\` (${colList}) VALUES (${placeholders})`, + `INSERT INTO \`apps\` (${colList}) VALUES (${placeholders})${this.clients.db.returningIdClause()}`, values, ); const insertId = result?.insertId; @@ -380,7 +390,7 @@ export class AppStore extends PuterStore { const colList = columns.map((c) => `\`${c}\``).join(', '); const result = await this.clients.db.write( - `INSERT INTO \`apps\` (${colList}) VALUES (${placeholders})`, + `INSERT INTO \`apps\` (${colList}) VALUES (${placeholders})${this.clients.db.returningIdClause()}`, values, ); const insertId = result?.insertId; @@ -698,6 +708,7 @@ export class AppStore extends PuterStore { async #resolveByOldName(name) { const cutoffClause = this.clients.db.case({ sqlite: `datetime('now', '-${OLD_APP_NAME_TTL_MONTHS} months')`, + postgres: `(NOW() - INTERVAL '${OLD_APP_NAME_TTL_MONTHS} months')`, otherwise: `(NOW() - INTERVAL ${OLD_APP_NAME_TTL_MONTHS} MONTH)`, }); @@ -842,7 +853,10 @@ export class AppStore extends PuterStore { const out = {}; for (const [k, v] of Object.entries(fields)) { if (READ_ONLY_COLUMNS.has(k)) continue; - out[k] = v; + out[k] = + APP_BOOLEAN_COLUMNS.has(k) && v !== null && v !== undefined + ? this.clients.db.booleanValue(Boolean(v)) + : v; } return out; } @@ -1136,6 +1150,7 @@ export class AppStore extends PuterStore { const periodExpr = this.clients.db.case({ mysql: `DATE_FORMAT(FROM_UNIXTIME(ts), '${timeFormat}')`, sqlite: `STRFTIME('${timeFormat}', datetime(ts, 'unixepoch'))`, + postgres: `TO_CHAR(TO_TIMESTAMP(ts), '${this.#postgresDateFormat(grouping)}')`, otherwise: `DATE_FORMAT(FROM_UNIXTIME(ts), '${timeFormat}')`, }); const rows = await this.clients.db.read( @@ -1156,6 +1171,23 @@ export class AppStore extends PuterStore { return this.#assembleGroupedResult(processed, allPeriods, grouping); } + #postgresDateFormat(grouping) { + switch (grouping) { + case 'hour': + return 'YYYY-MM-DD HH24:00:00'; + case 'day': + return 'YYYY-MM-DD'; + case 'week': + return 'YYYY-WW'; + case 'month': + return 'YYYY-MM'; + case 'year': + return 'YYYY'; + default: + return 'YYYY-MM-DD'; + } + } + #assembleGroupedResult(rows, allPeriods, grouping) { // Totals come from raw rows so they survive even if a row's // period key fails to match an `allPeriods` entry (e.g. week diff --git a/src/backend/stores/fs/FSEntryStore.test.ts b/src/backend/stores/fs/FSEntryStore.test.ts new file mode 100644 index 0000000000..8566a96bee --- /dev/null +++ b/src/backend/stores/fs/FSEntryStore.test.ts @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; +import type { IConfig } from '../../types'; +import { FSEntryStore } from './FSEntryStore.js'; + +describe('FSEntryStore', () => { + it('quotes camelCase storage allowance aliases for Postgres', async () => { + const queries: string[] = []; + const config: IConfig = { + port: 0, + extensions: [], + is_storage_limited: true, + storage_capacity: 1000, + }; + const clients = { + db: { + quoteIdentifier: (identifier: string) => `"${identifier}"`, + read: async (query: string) => { + queries.push(query); + if (query.includes('SUM(size)')) { + return [{ totalUsage: 321 }]; + } + return [{ freeStorage: 654 }]; + }, + }, + event: { + emitAndWait: async () => undefined, + }, + }; + const store = new FSEntryStore( + config, + clients as ConstructorParameters[1], + {} as ConstructorParameters[2], + ); + + await expect(store.getUserStorageAllowance(42)).resolves.toEqual({ + curr: 321, + max: 654, + }); + expect(queries).toEqual([ + 'SELECT COALESCE(SUM(size), 0) AS "totalUsage" FROM fsentries WHERE user_id = ?', + 'SELECT free_storage AS "freeStorage" FROM "user" WHERE id = ? LIMIT 1', + ]); + }); +}); diff --git a/src/backend/stores/fs/FSEntryStore.ts b/src/backend/stores/fs/FSEntryStore.ts index fc595cbc41..46d065548b 100644 --- a/src/backend/stores/fs/FSEntryStore.ts +++ b/src/backend/stores/fs/FSEntryStore.ts @@ -60,10 +60,7 @@ export class FSEntryStore extends PuterStore { declare protected stores: LayerInstances; #insertIgnoreIntoFsentriesSql(): string { - return this.clients.db.case({ - sqlite: 'INSERT OR IGNORE INTO fsentries', - otherwise: 'INSERT IGNORE INTO fsentries', - }); + return this.clients.db.insertIgnoreInto('fsentries'); } // JSON aggregation of associated subdomain rows, keyed on fsentries.id. @@ -86,6 +83,16 @@ export class FSEntryStore extends PuterStore { FROM subdomains sd WHERE sd.root_dir_id = fsentries.id )`, + postgres: `( + SELECT COALESCE( + json_agg( + json_build_object('uuid', sd.uuid, 'subdomain', sd.subdomain) + ), + '[]'::json + ) + FROM subdomains sd + WHERE sd.root_dir_id = fsentries.id + )`, }); } @@ -659,6 +666,8 @@ export class FSEntryStore extends PuterStore { const insertRows: unknown[] = []; const valuePlaceholders: string[] = []; const expectedUuidByPath = new Map(); + const trueLiteral = this.clients.db.booleanLiteral(true); + const falseLiteral = this.clients.db.booleanLiteral(false); for (const dirPath of pathsAtDepth) { const parentPath = pathPosix.dirname(dirPath); const parentEntry = @@ -679,7 +688,7 @@ export class FSEntryStore extends PuterStore { const expectedUuid = uuidv4(); expectedUuidByPath.set(dirPath, expectedUuid); valuePlaceholders.push( - '(?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 0, 0)', + `(?, ?, ?, ?, ?, ?, ${trueLiteral}, ?, ?, ?, ${falseLiteral}, 0)`, ); insertRows.push( expectedUuid, @@ -709,7 +718,7 @@ export class FSEntryStore extends PuterStore { accessed, immutable, size - ) VALUES ${valuePlaceholders.join(', ')}`, + ) VALUES ${valuePlaceholders.join(', ')}${this.clients.db.insertIgnoreSuffix()}`, insertRows, ); } catch { @@ -812,6 +821,8 @@ export class FSEntryStore extends PuterStore { : await this.#ensureDirectoryPath(parentPath, userId, true); const dirName = pathPosix.basename(normalizedPath); const now = Math.floor(Date.now() / 1000); + const trueLiteral = this.clients.db.booleanLiteral(true); + const falseLiteral = this.clients.db.booleanLiteral(false); let insertError: unknown = null; try { @@ -829,7 +840,7 @@ export class FSEntryStore extends PuterStore { accessed, immutable, size - ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 0, 0)`, + ) VALUES (?, ?, ?, ?, ?, ?, ${trueLiteral}, ?, ?, ?, ${falseLiteral}, 0)${this.clients.db.insertIgnoreSuffix()}`, [ uuidv4(), userId, @@ -1765,11 +1776,13 @@ export class FSEntryStore extends PuterStore { entry.input.associatedAppId ?? null, entry.input.isPublic === undefined ? null - : entry.input.isPublic - ? 1 - : 0, + : this.clients.db.booleanValue( + entry.input.isPublic, + ), entry.input.thumbnail ?? null, - entry.input.immutable ? 1 : 0, + this.clients.db.booleanValue( + Boolean(entry.input.immutable), + ), entry.fileName, entry.targetPath, entry.metadataJson, @@ -1850,7 +1863,7 @@ export class FSEntryStore extends PuterStore { } valuePlaceholders.push( - '(?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + `(?, ?, ?, ?, ?, ?, ?, ${this.clients.db.booleanLiteral(false)}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ); values.push( entry.input.uuid, @@ -1862,11 +1875,13 @@ export class FSEntryStore extends PuterStore { entry.input.associatedAppId ?? null, entry.input.isPublic === undefined ? null - : entry.input.isPublic - ? 1 - : 0, + : this.clients.db.booleanValue( + entry.input.isPublic, + ), entry.input.thumbnail ?? null, - entry.input.immutable ? 1 : 0, + this.clients.db.booleanValue( + Boolean(entry.input.immutable), + ), entry.fileName, entry.targetPath, entry.metadataJson, @@ -1903,7 +1918,7 @@ export class FSEntryStore extends PuterStore { created, accessed, size - ) VALUES ${valuePlaceholders.join(', ')}`, + ) VALUES ${valuePlaceholders.join(', ')}${this.clients.db.insertIgnoreSuffix()}`, values, ); }, @@ -2055,11 +2070,13 @@ export class FSEntryStore extends PuterStore { entry.input.associatedAppId ?? null, entry.input.isPublic === undefined ? null - : entry.input.isPublic - ? 1 - : 0, + : this.clients.db.booleanValue( + entry.input.isPublic, + ), entry.input.thumbnail ?? null, - entry.input.immutable ? 1 : 0, + this.clients.db.booleanValue( + Boolean(entry.input.immutable), + ), entry.fileName, entry.targetPath, entry.metadataJson, @@ -2273,9 +2290,13 @@ export class FSEntryStore extends PuterStore { ? `/${input.name}` : `${parentPath}/${input.name}`; - const isDir = input.kind === 'directory' ? 1 : 0; - const isShortcut = input.kind === 'shortcut' ? 1 : 0; - const isSymlink = input.kind === 'symlink' ? 1 : 0; + const isDir = this.clients.db.booleanValue(input.kind === 'directory'); + const isShortcut = this.clients.db.booleanValue( + input.kind === 'shortcut', + ); + const isSymlink = this.clients.db.booleanValue( + input.kind === 'symlink', + ); await this.clients.db.write( `INSERT INTO fsentries ( @@ -2315,12 +2336,10 @@ export class FSEntryStore extends PuterStore { input.associatedAppId ?? null, input.metadata ?? null, input.thumbnail ?? null, - input.immutable ? 1 : 0, + this.clients.db.booleanValue(Boolean(input.immutable)), input.isPublic === undefined || input.isPublic === null ? null - : input.isPublic - ? 1 - : 0, + : this.clients.db.booleanValue(input.isPublic), now, now, now, @@ -2578,10 +2597,12 @@ export class FSEntryStore extends PuterStore { if (patch.isPublic !== undefined) push( 'is_public', - patch.isPublic === null ? null : patch.isPublic ? 1 : 0, + patch.isPublic === null + ? null + : this.clients.db.booleanValue(patch.isPublic), ); if (patch.immutable !== undefined) - push('immutable', patch.immutable ? 1 : 0); + push('immutable', this.clients.db.booleanValue(patch.immutable)); if (patch.associatedAppId !== undefined) push('associated_app_id', patch.associatedAppId); if (patch.layout !== undefined) push('layout', patch.layout); @@ -2629,7 +2650,7 @@ export class FSEntryStore extends PuterStore { async getRootEntryForUser(userId: number): Promise { const rows = (await this.clients.db.read( `SELECT ${this.#selectFsentriesColumns()} FROM fsentries - WHERE user_id = ? AND parent_uid IS NULL AND is_dir = 1 + WHERE user_id = ? AND parent_uid IS NULL AND is_dir = ${this.clients.db.booleanLiteral(true)} ORDER BY id ASC LIMIT 1`, [userId], )) as unknown as FSEntryRow[]; @@ -2669,9 +2690,13 @@ export class FSEntryStore extends PuterStore { if (oldPath && oldPath !== '/' && oldPath !== newPath) { const likePattern = `${this.#escapeLikePattern(oldPath)}/%`; const oldLen = oldPath.length; + const rewrittenPath = this.clients.db.case({ + postgres: '? || SUBSTR(path, ?)', + otherwise: 'CONCAT(?, SUBSTR(path, ?))', + }); await this.clients.db.write( `UPDATE fsentries - SET path = CONCAT(?, SUBSTR(path, ?)), + SET path = ${rewrittenPath}, modified = ? WHERE user_id = ? AND path LIKE ? ESCAPE '!'`, [newPath, oldLen + 1, now, userId, likePattern], @@ -2720,9 +2745,13 @@ export class FSEntryStore extends PuterStore { // CONCAT(?, SUBSTR(path, ? + 1)) to rewrite just the prefix portion. const oldPrefixLen = normalizedOld.length; + const rewrittenPath = this.clients.db.case({ + postgres: '? || SUBSTR(path, ?)', + otherwise: 'CONCAT(?, SUBSTR(path, ?))', + }); const result = await this.clients.db.write( `UPDATE fsentries - SET path = CONCAT(?, SUBSTR(path, ?)), + SET path = ${rewrittenPath}, modified = ? WHERE user_id = ? AND path LIKE ? ESCAPE '!'`, [normalizedNew, oldPrefixLen + 1, now, userId, likePattern], @@ -2771,11 +2800,11 @@ export class FSEntryStore extends PuterStore { ): Promise<{ curr: number; max: number }> { const [usageRows, userRows] = await Promise.all([ this.clients.db.read( - 'SELECT COALESCE(SUM(size), 0) AS totalUsage FROM fsentries WHERE user_id = ?', + `SELECT COALESCE(SUM(size), 0) AS ${this.clients.db.quoteIdentifier('totalUsage')} FROM fsentries WHERE user_id = ?`, [userId], ) as Promise<{ totalUsage: number }[]>, this.clients.db.read( - 'SELECT free_storage AS freeStorage FROM user WHERE id = ? LIMIT 1', + `SELECT free_storage AS ${this.clients.db.quoteIdentifier('freeStorage')} FROM ${this.clients.db.quoteIdentifier('user')} WHERE id = ? LIMIT 1`, [userId], ) as Promise<{ freeStorage: number | null }[]>, ]); diff --git a/src/backend/stores/group/GroupStore.ts b/src/backend/stores/group/GroupStore.ts index 8436f55af0..589d43b98a 100644 --- a/src/backend/stores/group/GroupStore.ts +++ b/src/backend/stores/group/GroupStore.ts @@ -159,6 +159,7 @@ export class GroupStore extends PuterStore { }): Promise { const windowClause = this.clients.db.case({ sqlite: "datetime('now', '-1 hour')", + postgres: "NOW() - INTERVAL '1 hour'", otherwise: 'NOW() - INTERVAL 1 HOUR', }); const [countRow] = await this.clients.db.read( diff --git a/src/backend/stores/oidc/OIDCStore.js b/src/backend/stores/oidc/OIDCStore.js index 5f59e98a1b..baa780539d 100644 --- a/src/backend/stores/oidc/OIDCStore.js +++ b/src/backend/stores/oidc/OIDCStore.js @@ -20,6 +20,15 @@ import { PuterStore } from '../types'; import { HttpError } from '../../core/http/HttpError.js'; +const isUniqueConstraintError = (e) => { + return ( + e?.message?.includes('UNIQUE') || + e?.code === 'SQLITE_CONSTRAINT' || + e?.code === 'ER_DUP_ENTRY' || + e?.code === '23505' + ); +}; + /** * CRUD over the `user_oidc_providers` table. * @@ -54,11 +63,7 @@ export class OIDCStore extends PuterStore { ); return; } catch (e) { - const isUnique = - e.message?.includes('UNIQUE') || - e.code === 'SQLITE_CONSTRAINT' || - e.code === 'ER_DUP_ENTRY'; - if (!isUnique) throw e; + if (!isUniqueConstraintError(e)) throw e; } // UNIQUE(provider, provider_sub) collision — either we're re-linking diff --git a/src/backend/stores/oidc/OIDCStore.test.ts b/src/backend/stores/oidc/OIDCStore.test.ts new file mode 100644 index 0000000000..b792424eae --- /dev/null +++ b/src/backend/stores/oidc/OIDCStore.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, vi } from 'vitest'; +import { OIDCStore } from './OIDCStore.js'; + +type OidcRow = { + user_id: number; + provider: string; + provider_sub: string; +}; + +const createPostgresUniqueError = (): Error & { code: string } => { + return Object.assign( + new Error( + 'duplicate key value violates unique constraint "idx_user_oidc_providers_provider_sub_unique"', + ), + { code: '23505' }, + ); +}; + +const createStore = (rows: readonly OidcRow[]) => { + const db = { + write: vi.fn( + async (_sql: string, _params: readonly unknown[]): Promise => { + throw createPostgresUniqueError(); + }, + ), + read: vi.fn( + async ( + _sql: string, + _params: readonly unknown[], + ): Promise => rows, + ), + }; + const store = new OIDCStore({}, { db }); + + return { db, store }; +}; + +describe('OIDCStore', () => { + it('treats a Postgres unique violation as idempotent for an existing same-user link', async () => { + const { db, store } = createStore([ + { + user_id: 123, + provider: 'test-provider', + provider_sub: 'subject-1', + }, + ]); + + await expect( + store.link(123, 'test-provider', 'subject-1'), + ).resolves.toBeUndefined(); + + expect(db.read).toHaveBeenCalledWith( + expect.stringContaining('user_oidc_providers'), + ['test-provider', 'subject-1'], + ); + }); + + it('rejects a Postgres unique violation for an existing different-user link', async () => { + const { store } = createStore([ + { + user_id: 456, + provider: 'test-provider', + provider_sub: 'subject-1', + }, + ]); + + await expect( + store.link(123, 'test-provider', 'subject-1'), + ).rejects.toMatchObject({ + statusCode: 409, + legacyCode: 'conflict', + }); + }); +}); diff --git a/src/backend/stores/permission/PermissionStore.ts b/src/backend/stores/permission/PermissionStore.ts index e0c46a7c58..26c7e2a17b 100644 --- a/src/backend/stores/permission/PermissionStore.ts +++ b/src/backend/stores/permission/PermissionStore.ts @@ -169,11 +169,10 @@ export class PermissionStore extends PuterStore { permission: string, extra: Record, ): Promise { - const upsertClause = this.clients.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: - 'ON CONFLICT(`holder_user_id`, `issuer_user_id`, `permission`) DO UPDATE SET `extra` = ?', - }); + const upsertClause = this.clients.db.upsertClause( + ['holder_user_id', 'issuer_user_id', 'permission'], + ['extra'], + ); await this.clients.db.write( 'INSERT INTO `user_to_user_permissions` (`holder_user_id`, `issuer_user_id`, `permission`, `extra`) ' + `VALUES (?, ?, ?, ?) ${upsertClause}`, @@ -262,11 +261,10 @@ export class PermissionStore extends PuterStore { permission: string, extra: Record, ): Promise { - const upsertClause = this.clients.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: - 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?', - }); + const upsertClause = this.clients.db.upsertClause( + ['user_id', 'app_id', 'permission'], + ['extra'], + ); await this.clients.db.write( 'INSERT INTO `user_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' + `VALUES (?, ?, ?, ?) ${upsertClause}`, @@ -353,11 +351,10 @@ export class PermissionStore extends PuterStore { permission: string, extra: Record, ): Promise { - const upsertClause = this.clients.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: - 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?', - }); + const upsertClause = this.clients.db.upsertClause( + ['user_id', 'app_id', 'permission'], + ['extra'], + ); await this.clients.db.write( 'INSERT INTO `dev_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' + `VALUES (?, ?, ?, ?) ${upsertClause}`, @@ -443,11 +440,10 @@ export class PermissionStore extends PuterStore { permission: string, extra: Record, ): Promise { - const upsertClause = this.clients.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: - 'ON CONFLICT(`user_id`, `group_id`, `permission`) DO UPDATE SET `extra` = ?', - }); + const upsertClause = this.clients.db.upsertClause( + ['user_id', 'group_id', 'permission'], + ['extra'], + ); await this.clients.db.write( 'INSERT INTO `user_to_group_permissions` (`user_id`, `group_id`, `permission`, `extra`) ' + `VALUES (?, ?, ?, ?) ${upsertClause}`, diff --git a/src/backend/stores/session/SessionStore.js b/src/backend/stores/session/SessionStore.js index 9265e36c91..35e0f42c4b 100644 --- a/src/backend/stores/session/SessionStore.js +++ b/src/backend/stores/session/SessionStore.js @@ -55,14 +55,15 @@ export const WORKER_WINDOW_SECONDS = 99 * 365 * 24 * 60 * 60; // 99y (virtually // FK / type violations must bubble up; otherwise the caller caches a // row that was never inserted. // better-sqlite3 surfaces SqliteError.code; mysql2 surfaces .code and -// .errno (1062 = ER_DUP_ENTRY). +// .errno (1062 = ER_DUP_ENTRY); pg surfaces SQLSTATE 23505. function isUniqueViolation(err) { if (!err) return false; const code = err.code; if ( code === 'SQLITE_CONSTRAINT_UNIQUE' || code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || - code === 'ER_DUP_ENTRY' + code === 'ER_DUP_ENTRY' || + code === '23505' ) { return true; } @@ -462,7 +463,7 @@ export class SessionStore extends PuterStore { * * `appUid` is allowed null for user-scoped workers — the partial * unique index treats those distinctly (SQLite via NULL-distinct - * semantics, MySQL via IFNULL in the generated key). + * semantics, MySQL/Postgres via COALESCE in the generated key). * * @param userId - User row id (numeric). * @param opts.appUid - App UID or null for user-scoped workers. @@ -598,9 +599,16 @@ export class SessionStore extends PuterStore { return cached; } + const coalesceLastIp = this.clients.db.nullCoalesce('`last_ip`', "''"); + const coalesceBoundIp = this.clients.db.nullCoalesce('?', "''"); + const coalesceLastUserAgent = this.clients.db.nullCoalesce( + '`last_user_agent`', + "''", + ); + const coalesceBoundUserAgent = this.clients.db.nullCoalesce('?', "''"); const selectOldest = () => this.clients.db.read( - "SELECT * FROM `sessions` WHERE `kind` = 'web' AND `user_id` = ? AND `created_via` = 'legacy_backfill' AND IFNULL(`last_ip`, '') = IFNULL(?, '') AND IFNULL(`last_user_agent`, '') = IFNULL(?, '') AND `revoked_at` IS NULL AND (`expires_at` IS NULL OR `expires_at` > ?) ORDER BY `id` ASC LIMIT 1", + `SELECT * FROM \`sessions\` WHERE \`kind\` = 'web' AND \`user_id\` = ? AND \`created_via\` = 'legacy_backfill' AND ${coalesceLastIp} = ${coalesceBoundIp} AND ${coalesceLastUserAgent} = ${coalesceBoundUserAgent} AND \`revoked_at\` IS NULL AND (\`expires_at\` IS NULL OR \`expires_at\` > ?) ORDER BY \`id\` ASC LIMIT 1`, [opts.userId, ip, ua, now], ); @@ -713,7 +721,7 @@ export class SessionStore extends PuterStore { /** Update user-level last activity timestamp. */ async updateUserActivity(userId, lastActivityTs) { await this.clients.db.write( - 'UPDATE `user` SET `last_activity_ts` = ? WHERE `id` = ? AND (`last_activity_ts` IS NULL OR `last_activity_ts` < ?) LIMIT 1', + 'UPDATE `user` SET `last_activity_ts` = ? WHERE `id` = ? AND (`last_activity_ts` IS NULL OR `last_activity_ts` < ?)', [lastActivityTs, userId, lastActivityTs], ); } @@ -915,23 +923,18 @@ export class SessionStore extends PuterStore { /** * Active worker session for (userId, appUid, workerName). Matches * the partial unique index `idx_sessions_user_worker_active`. - * `appUid` is allowed null for user-scoped workers; IFNULL keeps + * `appUid` is allowed null for user-scoped workers; COALESCE keeps * the comparison correct since SQL `= NULL` doesn't match. - * - * MySQL's `JSON_EXTRACT` returns a JSON-typed value with embedded - * quoting (`"name"` rather than `name`), which never equals a - * plain string bind. Use `JSON_UNQUOTE(JSON_EXTRACT(...))` on - * MySQL to strip that. SQLite's `json_extract` already returns the - * unwrapped scalar so the literal form works there. */ async #selectWorkerRow(userId, appUid, workerName) { const now = nowSeconds(); - const workerNameExpr = this.clients.db.case({ - sqlite: "json_extract(`meta`, '$.worker_name')", - otherwise: "JSON_UNQUOTE(JSON_EXTRACT(`meta`, '$.worker_name'))", - }); + const workerNameExpr = this.clients.db.jsonTextExtract('`meta`', [ + 'worker_name', + ]); + const appUidExpr = this.clients.db.nullCoalesce('`app_uid`', "''"); + const appUidBound = this.clients.db.nullCoalesce('?', "''"); const rows = await this.clients.db.read( - `SELECT * FROM \`sessions\` WHERE \`kind\` = 'worker' AND \`user_id\` = ? AND IFNULL(\`app_uid\`, '') = IFNULL(?, '') AND ${workerNameExpr} = ? AND \`revoked_at\` IS NULL AND (\`expires_at\` IS NULL OR \`expires_at\` > ?) LIMIT 1`, + `SELECT * FROM \`sessions\` WHERE \`kind\` = 'worker' AND \`user_id\` = ? AND ${appUidExpr} = ${appUidBound} AND ${workerNameExpr} = ? AND \`revoked_at\` IS NULL AND (\`expires_at\` IS NULL OR \`expires_at\` > ?) LIMIT 1`, [userId, appUid ?? null, workerName, now], ); return this.#normalizeRow(rows[0]); diff --git a/src/backend/stores/user/UserStore.test.ts b/src/backend/stores/user/UserStore.test.ts new file mode 100644 index 0000000000..1fc80e87bb --- /dev/null +++ b/src/backend/stores/user/UserStore.test.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; +import { setupTestServer } from '../../testUtil.ts'; +import { PuterServer } from '../../server.ts'; + +describe('UserStore', () => { + let server: PuterServer; + + beforeAll(async () => { + server = await setupTestServer(); + }); + + afterAll(async () => { + await server?.shutdown(); + }); + + it('keeps cached booleans normalized after update', async () => { + const username = `us-${Math.random().toString(36).slice(2, 10)}`; + const user = await server.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: `${username}@test.local`, + }); + + await server.stores.user.update(user.id, { + email_confirmed: false, + requires_email_confirmation: true, + }); + + const cachedUser = await server.stores.user.getById(user.id); + + expect(cachedUser?.email_confirmed).toBe(false); + expect(cachedUser?.requires_email_confirmation).toBe(true); + expect(typeof cachedUser?.email_confirmed).toBe('boolean'); + expect(typeof cachedUser?.requires_email_confirmation).toBe('boolean'); + }); +}); diff --git a/src/backend/stores/user/UserStore.ts b/src/backend/stores/user/UserStore.ts index 7b27679115..efee906f6e 100644 --- a/src/backend/stores/user/UserStore.ts +++ b/src/backend/stores/user/UserStore.ts @@ -85,6 +85,15 @@ const LATIN1_USER_COLUMNS: ReadonlySet = new Set([ 'username', 'clean_email', ]); +const USER_BOOLEAN_COLUMNS: ReadonlySet = new Set([ + 'requires_email_confirmation', + 'email_confirmed', + 'dev_approved_for_incentive_program', + 'dev_joined_incentive_program', + 'suspended', + 'unsubscribed', + 'otp_enabled', +]); const assertLatin1Writable = (fields: Record): void => { for (const [key, value] of Object.entries(fields)) { @@ -336,7 +345,7 @@ export class UserStore extends PuterStore { signup_server, referrer, last_activity_ts) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)${this.clients.db.returningIdClause()}`, [ fields.username, fields.email, @@ -344,7 +353,9 @@ export class UserStore extends PuterStore { fields.password, fields.uuid, fields.free_storage ?? null, - fields.requires_email_confirmation ? 1 : 0, + this.clients.db.booleanValue( + Boolean(fields.requires_email_confirmation), + ), fields.email_confirm_code ?? null, fields.email_confirm_token ?? null, fields.audit_metadata @@ -380,13 +391,23 @@ export class UserStore extends PuterStore { userId: number, patch: Record, ): Promise { - const keys = Object.keys(patch); + const dbPatch: Record = {}; + for (const [key, value] of Object.entries(patch)) { + dbPatch[key] = + USER_BOOLEAN_COLUMNS.has(key) && + value !== null && + value !== undefined + ? this.clients.db.booleanValue(Boolean(value)) + : value; + } + + const keys = Object.keys(dbPatch); if (keys.length === 0) return; - assertLatin1Writable(patch); + assertLatin1Writable(dbPatch); const setClause = keys.map((k) => `\`${k}\` = ?`).join(', '); - const values = keys.map((k) => patch[k]); + const values = keys.map((k) => dbPatch[k]); await this.clients.db.write( `UPDATE \`user\` SET ${setClause} WHERE \`id\` = ?`, @@ -395,7 +416,7 @@ export class UserStore extends PuterStore { const fresh = await this.getByProperty('id', userId, { force: true }); if (fresh) { - await this.#refreshCache({ ...fresh, ...patch }); + await this.#refreshCache(fresh); } else { await this.invalidateById(userId); } @@ -428,13 +449,19 @@ export class UserStore extends PuterStore { `UPDATE \`user\` SET \`email\` = NULL, \`clean_email\` = NULL, - \`email_confirmed\` = 0, - \`requires_email_confirmation\` = 0, + \`email_confirmed\` = ?, + \`requires_email_confirmation\` = ?, \`email_confirm_code\` = NULL, \`email_confirm_token\` = NULL WHERE \`id\` != ? AND (\`email\` = ? OR \`clean_email\` = ?)`, - [userId, email, cleanEmailValue], + [ + this.clients.db.booleanValue(false), + this.clients.db.booleanValue(false), + userId, + email, + cleanEmailValue, + ], ); } diff --git a/src/backend/testUtil.ts b/src/backend/testUtil.ts index 77e149e9fb..496ac1d248 100644 --- a/src/backend/testUtil.ts +++ b/src/backend/testUtil.ts @@ -1,8 +1,96 @@ import { deepMerge } from '../../tools/lib/configMigration.mjs'; import { PuterServer } from './server'; import { IConfig } from './types'; +import { puterClients } from './clients'; +import { + PostgresDatabaseClient, + type PostgresPool, +} from './clients/database/PostgresDatabaseClient'; +import type { PoolConfig } from 'pg'; -export const setupTestServer = async (configOverrides?: IConfig) => { +export const POSTGRES_TEST_MIGRATIONS_PATH = + 'src/backend/clients/database/migrations/postgres'; + +type PgMockPostgresHarness = { + client: PostgresDatabaseClient; + createClient: () => PostgresDatabaseClient; + destroy: () => void; +}; + +const usesPgMockPostgres = (config: IConfig): boolean => { + const database = config.database; + if (database?.engine !== 'postgres' || database.inMemory !== true) { + return false; + } + + return !( + database.connectionString || + database.url || + database.host || + database.port || + database.user || + database.password || + database.database || + database.replica + ); +}; + +export const createPgMockPostgresDatabaseClient = async ( + config: IConfig, +): Promise => { + const [{ PostgresMock }, { Pool }] = await Promise.all([ + import('pgmock'), + import('pg'), + ]); + const mock = await PostgresMock.create(); + const pgMockConfig = mock.getNodePostgresConfig(); + const createPgMockStream = pgMockConfig.stream; + const poolFactory = (poolConfig: PoolConfig): PostgresPool => + new Pool({ + ...poolConfig, + database: 'postgres', + ...pgMockConfig, + stream: () => { + const stream = createPgMockStream(); + stream.ref = () => stream; + stream.unref = () => stream; + return stream; + }, + }) as unknown as PostgresPool; + const createClient = () => new PostgresDatabaseClient(config, poolFactory); + + return { + client: createClient(), + createClient, + destroy: () => mock.destroy(), + }; +}; + +/** + * When `PUTER_TEST_DB_ENGINE=postgres` is set, `setupTestServer` swaps its + * default sqlite test database for an in-memory Postgres backed by pgmock + * (with the bundled Postgres migrations applied on boot). Tests that + * explicitly override `database` still win — the env var only affects the + * implicit default used by callers that don't pass any DB overrides. + * + * Recognized values: `postgres` → pgmock. Anything else (including unset) → + * the original sqlite-in-memory default. + */ +const testDatabaseDefault = (): IConfig['database'] => { + const engine = (process.env.PUTER_TEST_DB_ENGINE ?? '').toLowerCase(); + if (engine === 'postgres') { + return { + engine: 'postgres', + inMemory: true, + migrationPaths: [POSTGRES_TEST_MIGRATIONS_PATH], + }; + } + return { engine: 'sqlite', inMemory: true }; +}; + +export const setupTestServer = async ( + configOverrides?: IConfig, +): Promise => { // read default config json const defaultConfig = await import('../../config.default.json', { with: { @@ -14,7 +102,7 @@ export const setupTestServer = async (configOverrides?: IConfig) => { deepMerge(defaultConfig, { extensions: [], port: 0, - database: { engine: 'sqlite', inMemory: true }, + database: testDatabaseDefault(), dynamo: { inMemory: true, bootstrapTables: true }, redis: { useMock: true }, s3: { localConfig: { inMemory: true } }, @@ -22,8 +110,41 @@ export const setupTestServer = async (configOverrides?: IConfig) => { no_devwatch: true, }), configOverrides ?? {}, + ) as IConfig; + + let pgMockClient: PgMockPostgresHarness | undefined; + if (usesPgMockPostgres(config)) { + const database = config.database; + if (!database) { + throw new Error('Postgres test database config is missing'); + } + database.migrationPaths ??= [POSTGRES_TEST_MIGRATIONS_PATH]; + pgMockClient = await createPgMockPostgresDatabaseClient(config); + } + + const server = new PuterServer( + config, + pgMockClient ? { ...puterClients, db: pgMockClient.client } : undefined, ); - const server = new PuterServer(config); - await server.start(true); + + if (pgMockClient) { + const originalShutdown = server.shutdown.bind(server); + let destroyPgMock: (() => void) | undefined = pgMockClient.destroy; + server.shutdown = async () => { + try { + await originalShutdown(); + } finally { + destroyPgMock?.(); + destroyPgMock = undefined; + } + }; + } + + try { + await server.start(true); + } catch (e) { + pgMockClient?.destroy(); + throw e; + } return server; }; diff --git a/src/backend/types.ts b/src/backend/types.ts index 2cf945e6bd..7fd57a071a 100644 --- a/src/backend/types.ts +++ b/src/backend/types.ts @@ -306,7 +306,7 @@ export interface IS3Config { } export interface IDatabaseConfig { - engine: 'sqlite' | 'mysql'; + engine: 'sqlite' | 'mysql' | 'postgres'; // sqlite /** * SQLite database file path. Defaults to `':memory:'` (the @@ -317,7 +317,8 @@ export interface IDatabaseConfig { /** * Force in-memory SQLite (ignores `path`). Equivalent to * `path: ':memory:'`. Intended for tests so each suite gets a - * pristine in-process database. + * pristine in-process database. Test utilities also use + * `engine: 'postgres'` with `inMemory: true` to run against pgmock. */ inMemory?: boolean; targetVersion?: number; @@ -327,17 +328,21 @@ export interface IDatabaseConfig { user?: string; password?: string; database?: string; + connectionString?: string; + url?: string; replica?: { host?: string; port?: number; user?: string; password?: string; database?: string; + connectionString?: string; + url?: string; }; /** * Ordered list of directories whose `.sql` files are run sequentially at - * server start (mysql engine only). Files within a directory are sorted - * lexically; directories are processed in array order. Files MUST be + * server start (mysql/postgres engines). Numbered migration filenames sort + * numerically; directories are processed in array order. Files MUST be * idempotent — there is no per-file applied-state tracking. * Relative paths resolve from `process.cwd()`. */ diff --git a/src/backend/util/userProvisioning.ts b/src/backend/util/userProvisioning.ts index cdb2689e73..e64180c2d9 100644 --- a/src/backend/util/userProvisioning.ts +++ b/src/backend/util/userProvisioning.ts @@ -81,10 +81,11 @@ export async function generateDefaultFsentries( ]), ]; - // Each row: uuid, parent_uid, user_id, name, path, created, modified - // is_dir and immutable are hardcoded to 1. + // Each row: uuid, parent_uid, user_id, name, path, created, modified. + // is_dir and immutable are hardcoded to the database's true literal. + const trueLiteral = db.booleanLiteral(true); const placeholders = rows - .map(() => '(?, ?, ?, ?, ?, 1, ?, ?, 1)') + .map(() => `(?, ?, ?, ?, ?, ${trueLiteral}, ?, ?, ${trueLiteral})`) .join(', '); const params: unknown[] = []; for (const [uuid, parent, name, path] of rows) { diff --git a/src/backend/vitest.config.ts b/src/backend/vitest.config.ts index 92bebab1d3..8b7801caf2 100644 --- a/src/backend/vitest.config.ts +++ b/src/backend/vitest.config.ts @@ -27,6 +27,16 @@ const isCi = process.env.CI === 'true'; const backendDir = __dirname; const repoRoot = path.resolve(backendDir, '../..'); +// pgmock boots a WASM-emulated Postgres on every `setupTestServer()` call — +// migrations alone take ~60s, and the WASM VM serializes badly across +// concurrent emulator instances. So in pgmock mode we (a) bump hook/test +// timeouts well beyond the 10s default and (b) disable file parallelism so +// only one pgmock VM runs at a time. Tests still race their own logic +// internally; we only serialize the *files*. +const isPgmockMode = + (process.env.PUTER_TEST_DB_ENGINE ?? '').toLowerCase() === 'postgres'; +const pgmockTimeoutMs = 600_000; + // Vite 8's oxc transform leaves TC39 stage-3 decorators in place // (used by `@Controller`/`@Post`), so they reach Node verbatim and // crash with "SyntaxError: Invalid or unexpected token". Pre-transform @@ -72,6 +82,13 @@ export default defineConfig(({ mode }) => ({ }, test: { globals: true, + ...(isPgmockMode + ? { + testTimeout: pgmockTimeoutMs, + hookTimeout: pgmockTimeoutMs, + fileParallelism: false, + } + : {}), coverage: { provider: 'v8', reporter: isCi