From 3f7d60293a823fad858f90574498467ea45ecfe7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:04:02 -0600 Subject: [PATCH 01/39] Serialize same-path opens with database destruction --- AGENTS.md | 7 + src/binding/core/test_seam.h | 4 +- src/binding/database/db_registry.cpp | 227 ++++++++++++++++--------- src/binding/database/db_registry.h | 3 + test/destroy.test.ts | 62 ++++++- test/fixtures/fork-destroy-failure.mts | 25 +++ test/fixtures/fork-destroy-open.mts | 48 ++++++ test/workers/destroy-open-worker.mts | 17 ++ 8 files changed, 308 insertions(+), 85 deletions(-) create mode 100644 test/fixtures/fork-destroy-failure.mts create mode 100644 test/fixtures/fork-destroy-open.mts create mode 100644 test/workers/destroy-open-worker.mts diff --git a/AGENTS.md b/AGENTS.md index 5d76ca0cc..6ab2e7fc2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -210,6 +210,8 @@ sufficient (env teardown does not honor tsfn acquire counts); see ambiguous "disable the bound") falls back to the default like any malformed value. There is no opt-out: a deployment that would rather wait than fail a legitimately slow holder raises the value instead +- `ROCKSDB_JS_DESTROY_DELAY_MS` - Test-only: delay after descriptor teardown and + before physical database destruction (widens same-path reopen races) ## Test Structure @@ -307,6 +309,11 @@ sufficient (env teardown does not honor tsfn acquire counts); see racing close skip the purge (`use_count > 1`), so their state destructors re-run `PurgeIfUnreferenced` after releasing the ref — without that retry the skipped purge is permanent and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). + Database destruction is path-global rather than `(path, readOnly)`-scoped: `DestroyDB` closes every + descriptor for the path and keeps the path in `destroyingPaths` until both `rocksdb::DestroyDB` and + directory cleanup finish. `OpenDB` waits on that state before resolving a registry entry and must + re-resolve the map after every condition-variable wake; retaining a map-node reference across an + unlocked wait is a use-after-free when the closer erases that node. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index fa3172e4d..c7e3c15fb 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -9,8 +9,8 @@ // production where the env var is unset. // // Pass the env var name to testDelayMs() at the call site; see -// EventEmitter::notify, TransactionHandle::close, and TransactionHandle::get -// for usage. +// EventEmitter::notify, TransactionHandle::close, TransactionHandle::get, and +// DBRegistry::DestroyDB for usage. inline int testDelayMs(const char* envName) { const char* value = ::getenv(envName); return value ? ::atoi(value) : 0; diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 1900fd7b3..f8e350a1a 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -2,15 +2,54 @@ #include #include "database/db_registry.h" #include "transaction/transaction_handle.h" +#include "core/test_seam.h" #include "napi/macros.h" #include "core/platform.h" #include "core/compression.h" #include "napi/helpers.h" #include "napi/async.h" #include "rocksdb/table.h" +#include +#include namespace rocksdb_js { +namespace { + +constexpr std::chrono::seconds DATABASE_LIFECYCLE_WAIT{30}; + +class DestroyPathGuard final { +public: + DestroyPathGuard( + std::mutex& mutex, + std::condition_variable& condition, + std::unordered_set& destroyingPaths, + std::string path + ) : mutex(mutex), condition(condition), destroyingPaths(destroyingPaths), path(std::move(path)) {} + + ~DestroyPathGuard() { + { + std::lock_guard lock(this->mutex); + this->destroyingPaths.erase(this->path); + } + this->condition.notify_all(); + } + +private: + std::mutex& mutex; + std::condition_variable& condition; + std::unordered_set& destroyingPaths; + std::string path; +}; + +struct ClosingDescriptor final { + DBKey key; + std::shared_ptr descriptor; + std::shared_ptr condition; +}; + +} // namespace + // Initialize the static instance std::unique_ptr DBRegistry::instance; @@ -110,9 +149,8 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { std::lock_guard lock(instance->databasesMutex); auto eraseIt = instance->databases.find(key); - // Only erase the entry we claimed. OpenDB's wait predicate may have - // reset the map's descriptor ref to null while we closed; a brand-new - // descriptor cannot appear because OpenDB blocks until we notify below. + // Only erase the entry we claimed. A brand-new descriptor cannot appear + // because OpenDB blocks until we notify below. if (eraseIt != instance->databases.end() && (!eraseIt->second.descriptor || eraseIt->second.descriptor == descriptor)) { instance->databases.erase(eraseIt); @@ -150,76 +188,87 @@ void DBRegistry::DestroyDB(const std::string& path) { } DEBUG_LOG("%p DBRegistry::DestroyDB Destroying \"%s\"\n", instance.get(), path.c_str()); - - std::shared_ptr descriptor; - std::shared_ptr condition; - - // Claim the descriptor under the lock but leave the entry in the map until - // the close completes (same discipline as CloseDB): the entry is how the - // env-cleanup hooks (RemoveListenersByEnv / ReleaseCommitCompletionsByEnv) - // find shared descriptors, so erasing before close would let a worker env - // tear down in that window without scrubbing its tsfns from this - // descriptor — the close's own release pass would then touch freed tsfns. - // It also keeps a concurrent OpenDB waiting on the entry's condition - // instead of re-opening the path while its files are being destroyed. + const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; { - std::lock_guard lock(instance->databasesMutex); - for (auto& [key, entry] : instance->databases) { - if (key.path == path && entry.descriptor) { + std::unique_lock lock(instance->databasesMutex); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); + })) { + throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": another destroy is still in progress"); + } + instance->destroyingPaths.insert(path); + } + // A physical destroy has several throwing stages; the path gate must never + // survive one of them and permanently block every later open. + DestroyPathGuard pathGuard( + instance->databasesMutex, + instance->lifecycleCondition, + instance->destroyingPaths, + path + ); + + while (true) { + std::vector claimed; + std::vector alreadyClosing; + { + std::lock_guard lock(instance->databasesMutex); + for (auto& [key, entry] : instance->databases) { + if (key.path != path || !entry.descriptor) { + continue; + } + ClosingDescriptor closing{key, entry.descriptor, entry.condition}; if (entry.descriptor->beginClose()) { - descriptor = entry.descriptor; - condition = entry.condition; - DEBUG_LOG("%p DBRegistry::DestroyDB Claimed descriptor close (ref count = %ld)\n", - instance.get(), descriptor.use_count()); + claimed.push_back(std::move(closing)); + } else { + alreadyClosing.push_back(std::move(closing)); } - break; } } - } - if (descriptor) { - // Close all closables (iterators, transactions, handles) attached to this descriptor - // This should release all DBHandle references - DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor and all attached resources (ref count = %zu)\n", - instance.get(), descriptor.use_count()); - descriptor->finishClose(); + for (auto& closing : claimed) { + DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", + instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); + closing.descriptor->finishClose(); + } - // Now that the close is complete, remove the path's entries and wake - // any OpenDB waiting on this path. { std::lock_guard lock(instance->databasesMutex); - for (auto it = instance->databases.begin(); it != instance->databases.end(); ) { - if (it->first.path == path) { - it = instance->databases.erase(it); - } else { - ++it; + for (const auto& closing : claimed) { + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + instance->databases.erase(entry); } } } - if (condition) { - condition->notify_all(); + for (const auto& closing : claimed) { + closing.condition->notify_all(); + const size_t refCountAfterClose = closing.descriptor.use_count(); + if (refCountAfterClose > 1) { + std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + + " reference(s) still held after closing all handles. This may indicate handles not properly closed or JavaScript objects not yet garbage collected."; + DEBUG_LOG("%p DBRegistry::DestroyDB Error: %s\n", instance.get(), errorMsg.c_str()); + throw rocksdb_js::DBException(errorMsg); + } } - // After closing, check if there are still lingering references - // Should only be our local reference (= 1) at this point - size_t refCountAfterClose = descriptor.use_count(); - if (refCountAfterClose > 1) { - std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + - " reference(s) still held after closing all handles. This may indicate handles not properly closed or JavaScript objects not yet garbage collected."; - DEBUG_LOG("%p DBRegistry::DestroyDB Error: %s\n", instance.get(), errorMsg.c_str()); - throw rocksdb_js::DBException(errorMsg); + if (alreadyClosing.empty()) { + break; + } + for (const auto& closing : alreadyClosing) { + std::unique_lock lock(instance->databasesMutex); + if (!closing.condition->wait_until(lock, deadline, [&]() { + auto entry = instance->databases.find(closing.key); + return entry == instance->databases.end() || entry->second.descriptor != closing.descriptor; + })) { + throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); + } } + } - // Release our reference to the descriptor - // This will trigger the destructor which properly closes the DB - DEBUG_LOG("%p DBRegistry::DestroyDB Releasing descriptor reference\n", instance.get()); - descriptor.reset(); - } else { - // No open descriptor claimed; remove any placeholder entries for the - // path (an entry mid-close is erased by its closer's guarded erase). + { std::lock_guard lock(instance->databasesMutex); - for (auto it = instance->databases.begin(); it != instance->databases.end(); ) { - if (it->first.path == path && !it->second.descriptor) { + for (auto it = instance->databases.begin(); it != instance->databases.end();) { + if (it->first.path == path) { it = instance->databases.erase(it); } else { ++it; @@ -228,6 +277,10 @@ void DBRegistry::DestroyDB(const std::string& path) { } // Now the database lock should be released, safe to destroy + const int destroyDelayMs = testDelayMs("ROCKSDB_JS_DESTROY_DELAY_MS"); + if (destroyDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); + } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); if (!status.ok()) { @@ -276,34 +329,42 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons std::string name = options.name.empty() ? "default" : options.name; std::shared_ptr descriptor; std::unique_lock lock(instance->databasesMutex); - - // get or create entry for this path + mode + readOnly combination + const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; DBKey key{path, options.readOnly}; - auto entryIterator = instance->databases.find(key); - if (entryIterator == instance->databases.end()) { - // create entry with empty descriptor and new condition variable - auto [it, inserted] = instance->databases.emplace(key, DBRegistryEntry()); - entryIterator = it; - } + decltype(instance->databases)::iterator entryIterator; + while (true) { + if (!instance->destroyingPaths.empty() && + instance->destroyingPaths.find(path) != instance->destroyingPaths.end() + ) { + DEBUG_LOG("%p DBRegistry::OpenDB Database \"%s\" is being destroyed, waiting\n", instance.get(), path.c_str()); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": destruction is still in progress"); + } + continue; + } - auto& entry = entryIterator->second; + entryIterator = instance->databases.find(key); + if (entryIterator == instance->databases.end()) { + entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; + } + if (!entryIterator->second.descriptor || !entryIterator->second.descriptor->isClosing()) { + break; + } - // wait for any closing database on this specific path to be fully removed before proceeding - entry.condition->wait(lock, [&]() { - if (entry.descriptor) { - if (entry.descriptor->isClosing()) { - DEBUG_LOG("%p DBRegistry::OpenDB Database \"%s\" is closing, waiting for removal\n", instance.get(), path.c_str()); - entry.descriptor.reset(); - return false; // keep waiting - } - return true; // database exists and is not closing + DEBUG_LOG("%p DBRegistry::OpenDB Database \"%s\" is closing, waiting for removal\n", instance.get(), path.c_str()); + auto condition = entryIterator->second.condition; + if (!condition->wait_until(lock, deadline, [&]() { + auto current = instance->databases.find(key); + return current == instance->databases.end() || + !current->second.descriptor || !current->second.descriptor->isClosing(); + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": the previous instance is still closing"); } - return true; // database doesn't exist, can proceed - }); + } - // at this point, either: - // 1. descriptor is set to a valid, non-closing database, or - // 2. descriptor is nullptr (database doesn't exist) + auto& entry = entryIterator->second; if (entry.descriptor) { // database exists and is not closing, proceed with existing logic @@ -467,6 +528,10 @@ void DBRegistry::PurgeAll() { uint32_t i = 0; #endif for (auto it = instance->databases.begin(); it != instance->databases.end();) { + if (instance->destroyingPaths.find(it->first.path) != instance->destroyingPaths.end()) { + ++it; + continue; + } auto descriptor = it->second.descriptor; if (descriptor) { DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); @@ -727,7 +792,9 @@ void DBRegistry::Shutdown() { // Collect all descriptors to close for (auto& [_key, entry] : instance->databases) { - if (entry.descriptor) { + if (entry.descriptor && + instance->destroyingPaths.find(entry.descriptor->path) == instance->destroyingPaths.end() + ) { descriptorsToClose.push_back(entry.descriptor); } } diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index 18c010f24..a8444c2fd 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -5,6 +5,7 @@ #include #include #include +#include #include "database/db_descriptor.h" #include "database/db_handle.h" #include "transaction/transaction.h" @@ -78,6 +79,8 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; + std::condition_variable lifecycleCondition; + std::unordered_set destroyingPaths; /** * The singleton instance of the registry. diff --git a/test/destroy.test.ts b/test/destroy.test.ts index ca3c529a1..0f7cd891a 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,7 +1,41 @@ -import { dbRunner } from './lib/util.ts'; +import { dbRunner, generateDBPath } from './lib/util.ts'; +import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; +const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); +const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); + +function runDestroyFixture( + fixture: string, + dbPath: string, + env?: NodeJS.ProcessEnv +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [fixture, dbPath], { + env: { ...process.env, ...env }, + }); + let stderr = ''; + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + const timeout = setTimeout(() => { + child.kill(); + reject(new Error(`Destroy fixture timed out\n${stderr}`)); + }, 10_000); + child.on('error', reject); + child.on('close', (code, signal) => { + clearTimeout(timeout); + if (code === 0 && signal === null) { + resolve(); + } else { + reject(new Error(`Destroy fixture failed (code=${code}, signal=${signal})\n${stderr}`)); + } + }); + }); +} + describe('Destroy', () => { it('should destroy a closed database', () => dbRunner(async ({ db, dbPath }) => { @@ -25,19 +59,41 @@ describe('Destroy', () => { it('should destroy all related instances', () => dbRunner( - { dbOptions: [{}, { name: 'test' }] }, - async ({ db: db1, dbPath: dbPath1 }, { db: db2, dbPath: dbPath2 }) => { + { dbOptions: [{}, { name: 'test' }, { readOnly: true }] }, + async ( + { db: db1, dbPath: dbPath1 }, + { db: db2, dbPath: dbPath2 }, + { db: readOnly, dbPath: readOnlyPath } + ) => { expect(existsSync(dbPath1)).toBe(true); expect(existsSync(dbPath2)).toBe(true); + expect(existsSync(readOnlyPath)).toBe(true); expect(db1.isOpen()).toBe(true); expect(db2.isOpen()).toBe(true); + expect(readOnly.isOpen()).toBe(true); db1.destroy(); expect(existsSync(dbPath1)).toBe(false); expect(existsSync(dbPath2)).toBe(false); + expect(existsSync(readOnlyPath)).toBe(false); expect(db1.isOpen()).toBe(false); expect(db2.isOpen()).toBe(false); + expect(readOnly.isOpen()).toBe(false); } )); + + it('waits for physical destruction before reopening the same path', async () => { + await runDestroyFixture(destroyOpenFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_DELAY_MS: '1000', + }); + }, 15_000); + + it.skipIf(process.platform === 'win32')( + 'releases the path gate when physical destruction fails', + async () => { + await runDestroyFixture(destroyFailureFixture, generateDBPath()); + }, + 15_000 + ); }); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts new file mode 100644 index 000000000..6b0d31f07 --- /dev/null +++ b/test/fixtures/fork-destroy-failure.mts @@ -0,0 +1,25 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { chmodSync } from 'node:fs'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); +db.close(); + +chmodSync(path, 0o500); +let destroyFailed = false; +try { + db.destroy(); +} catch { + destroyFailed = true; +} finally { + chmodSync(path, 0o700); +} +if (!destroyFailed) + throw new Error('Expected destroy to fail for a non-writable database directory'); + +try { + RocksDatabase.open(path).close(); +} catch { + // Physical destruction may have partially completed; only gate release is asserted. +} diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts new file mode 100644 index 000000000..3bfcfb515 --- /dev/null +++ b/test/fixtures/fork-destroy-open.mts @@ -0,0 +1,48 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { setTimeout as delay } from 'node:timers/promises'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const original = RocksDatabase.open(path); +original.putSync('before-destroy', 'present'); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Destroy worker failed to initialize: ${JSON.stringify(ready)}`); +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Destroy worker did not start: ${JSON.stringify(destroying)}`); + +const registryDeadline = Date.now() + 5_000; +while (registryStatus().some((entry) => entry.path === path)) { + if (Date.now() >= registryDeadline) throw new Error('Timed out waiting for the destroy window'); + await delay(1); +} + +const destroyResult = nextMessage(); +const startedAt = Date.now(); +const reopened = RocksDatabase.open(path); +const openDuration = Date.now() - startedAt; +const destroyed = await destroyResult; +if (!destroyed.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyed)}`); +if (openDuration < 500) throw new Error(`Reopen did not wait for destroy (${openDuration}ms)`); +if (reopened.getSync('before-destroy') !== undefined) + throw new Error('Reopen observed pre-destroy data'); +reopened.putSync('after-destroy', 'present'); +if (reopened.getSync('after-destroy') !== 'present') + throw new Error('Reopened database is not usable'); +reopened.close(); +await worker.terminate(); diff --git a/test/workers/destroy-open-worker.mts b/test/workers/destroy-open-worker.mts new file mode 100644 index 000000000..7fe2c0585 --- /dev/null +++ b/test/workers/destroy-open-worker.mts @@ -0,0 +1,17 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { parentPort, workerData } from 'node:worker_threads'; + +const db = RocksDatabase.open(workerData.path); +if (!parentPort) throw new Error('Destroy/open worker requires a parent port'); +const port = parentPort; +port.postMessage({ ready: true }); + +port.once('message', () => { + port.postMessage({ destroying: true }); + try { + db.destroy(); + port.postMessage({ destroyed: true }); + } catch (error) { + port.postMessage({ error: error instanceof Error ? error.message : String(error) }); + } +}); From 8eead8b3d2bac2498a64ee34eee0f916a7c59d23 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:33:10 -0600 Subject: [PATCH 02/39] Address lifecycle review findings --- AGENTS.md | 5 -- README.md | 3 + src/binding/core/test_seam.h | 4 ++ src/binding/database/db_registry.cpp | 93 +++++++++++++++++++++----- src/binding/database/db_settings.cpp | 17 +++++ src/binding/database/db_settings.h | 7 +- src/load-binding.ts | 5 ++ test/destroy.test.ts | 22 +++--- test/fixtures/fork-destroy-failure.mts | 24 +++---- 9 files changed, 133 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6ab2e7fc2..2939e9058 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -309,11 +309,6 @@ sufficient (env teardown does not honor tsfn acquire counts); see racing close skip the purge (`use_count > 1`), so their state destructors re-run `PurgeIfUnreferenced` after releasing the ref — without that retry the skipped purge is permanent and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). - Database destruction is path-global rather than `(path, readOnly)`-scoped: `DestroyDB` closes every - descriptor for the path and keeps the path in `destroyingPaths` until both `rocksdb::DestroyDB` and - directory cleanup finish. `OpenDB` waits on that state before resolving a registry entry and must - re-resolve the map after every condition-variable wake; retaining a map-node reference across an - unlocked wait is a use-after-free when the closer erases that node. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/README.md b/README.md index 834f0f536..02b8717a3 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,9 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. + - `lifecycleWaitSeconds: number` Maximum time a synchronous open or destroy waits for another + lifecycle operation on the same path before throwing a retryable timeout error. Defaults to + `30` seconds and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global [Verification Table](#verification-table). Each slot is 8 bytes, so the default of `131072` (128K) slots is 1 MB. Set to `0` to disable the verification table. This must be configured diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index c7e3c15fb..24126370d 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -16,6 +16,10 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } +inline bool testFailureEnabled(const char* envName) { + return ::getenv(envName) != nullptr; +} + // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index f8e350a1a..bd06f2e92 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -2,6 +2,7 @@ #include #include "database/db_registry.h" #include "transaction/transaction_handle.h" +#include "database/db_settings.h" #include "core/test_seam.h" #include "napi/macros.h" #include "core/platform.h" @@ -10,22 +11,21 @@ #include "napi/async.h" #include "rocksdb/table.h" #include +#include #include namespace rocksdb_js { namespace { -constexpr std::chrono::seconds DATABASE_LIFECYCLE_WAIT{30}; - class DestroyPathGuard final { public: DestroyPathGuard( std::mutex& mutex, std::condition_variable& condition, std::unordered_set& destroyingPaths, - std::string path - ) : mutex(mutex), condition(condition), destroyingPaths(destroyingPaths), path(std::move(path)) {} + const std::string& path + ) : mutex(mutex), condition(condition), destroyingPaths(destroyingPaths), path(path) {} ~DestroyPathGuard() { { @@ -39,13 +39,14 @@ class DestroyPathGuard final { std::mutex& mutex; std::condition_variable& condition; std::unordered_set& destroyingPaths; - std::string path; + const std::string& path; }; struct ClosingDescriptor final { DBKey key; std::shared_ptr descriptor; std::shared_ptr condition; + bool closed = false; }; } // namespace @@ -188,7 +189,8 @@ void DBRegistry::DestroyDB(const std::string& path) { } DEBUG_LOG("%p DBRegistry::DestroyDB Destroying \"%s\"\n", instance.get(), path.c_str()); - const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { @@ -212,6 +214,8 @@ void DBRegistry::DestroyDB(const std::string& path) { std::vector alreadyClosing; { std::lock_guard lock(instance->databasesMutex); + claimed.reserve(instance->databases.size()); + alreadyClosing.reserve(instance->databases.size()); for (auto& [key, entry] : instance->databases) { if (key.path != path || !entry.descriptor) { continue; @@ -225,15 +229,28 @@ void DBRegistry::DestroyDB(const std::string& path) { } } + // Keep entries discoverable while finishClose runs: env cleanup uses the + // registry to remove callbacks owned by a worker that exits mid-close. + std::exception_ptr closeError; for (auto& closing : claimed) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); - closing.descriptor->finishClose(); + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (...) { + if (!closeError) { + closeError = std::current_exception(); + } + } } { std::lock_guard lock(instance->databasesMutex); for (const auto& closing : claimed) { + if (!closing.closed) { + continue; + } auto entry = instance->databases.find(closing.key); if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { instance->databases.erase(entry); @@ -241,7 +258,17 @@ void DBRegistry::DestroyDB(const std::string& path) { } } for (const auto& closing : claimed) { - closing.condition->notify_all(); + if (closing.closed) { + closing.condition->notify_all(); + } + } + if (closeError) { + std::rethrow_exception(closeError); + } + for (const auto& closing : claimed) { + if (!closing.closed) { + continue; + } const size_t refCountAfterClose = closing.descriptor.use_count(); if (refCountAfterClose > 1) { std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + @@ -281,6 +308,9 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } + if (testFailureEnabled("ROCKSDB_JS_DESTROY_FAILURE")) { + throw rocksdb_js::DBException("Injected database destruction failure"); + } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); if (!status.ok()) { @@ -329,7 +359,8 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons std::string name = options.name.empty() ? "default" : options.name; std::shared_ptr descriptor; std::unique_lock lock(instance->databasesMutex); - const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); DBKey key{path, options.readOnly}; decltype(instance->databases)::iterator entryIterator; while (true) { @@ -784,26 +815,52 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { - std::vector> descriptorsToClose; + std::vector descriptorsToClose; { std::lock_guard lock(instance->databasesMutex); DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); + descriptorsToClose.reserve(instance->databases.size()); - // Collect all descriptors to close - for (auto& [_key, entry] : instance->databases) { - if (entry.descriptor && - instance->destroyingPaths.find(entry.descriptor->path) == instance->destroyingPaths.end() + // Claim each close while holding the registry lock so DestroyDB can + // safely wait on the matching erase-and-notify below. + for (auto& [key, entry] : instance->databases) { + if (!entry.descriptor || + instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end() ) { - descriptorsToClose.push_back(entry.descriptor); + continue; + } + ClosingDescriptor closing{key, entry.descriptor, entry.condition}; + if (entry.descriptor->beginClose()) { + descriptorsToClose.push_back(std::move(closing)); } } } // Close all descriptors without holding the lock - for (auto& descriptor : descriptorsToClose) { - DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), descriptor->path.c_str()); - descriptor->close(); + std::exception_ptr closeError; + for (auto& closing : descriptorsToClose) { + DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (...) { + if (!closeError) { + closeError = std::current_exception(); + } + continue; + } + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + instance->databases.erase(entry); + } + } + closing.condition->notify_all(); + } + if (closeError) { + std::rethrow_exception(closeError); } // Purge the registry diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index 660b50308..ee53c89a1 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -1,4 +1,5 @@ #include "database/db_settings.h" +#include #include #include "napi/macros.h" #include "core/platform.h" @@ -30,6 +31,7 @@ DBSettings::DBSettings(): writeBufferManagerAllowStall(false), writeBufferManager(nullptr), compactOnClose(false), + lifecycleWaitSeconds(30), verificationTableEntries(128 * 1024), // 128K slots = 1 MB at 8 bytes per slot verificationTableSeed(generateSeed()), verificationTable(nullptr) @@ -213,6 +215,21 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { NAPI_STATUS_THROWS(rocksdb_js::getProperty(env, params, "compactOnClose", settings.compactOnClose, false)); + int64_t lifecycleWaitSeconds = 0; + status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); + if (status == napi_ok) { + if (lifecycleWaitSeconds <= 0 || + static_cast(lifecycleWaitSeconds) > std::numeric_limits::max() + ) { + ::napi_throw_range_error(env, nullptr, "Lifecycle wait seconds must be a positive integer"); + return nullptr; + } + settings.lifecycleWaitSeconds.store( + static_cast(lifecycleWaitSeconds), + std::memory_order_relaxed + ); + } + int64_t verificationTableEntries = 0; status = rocksdb_js::getProperty(env, params, "verificationTableEntries", verificationTableEntries, true); if (status == napi_ok) { diff --git a/src/binding/database/db_settings.h b/src/binding/database/db_settings.h index 63106ebce..d308ab643 100644 --- a/src/binding/database/db_settings.h +++ b/src/binding/database/db_settings.h @@ -49,6 +49,7 @@ class DBSettings final { std::mutex writeBufferManagerMutex; bool compactOnClose; + std::atomic lifecycleWaitSeconds; // Number of slots requested for the verification table. Default 128K // (1 MB at 8 bytes per slot). 0 disables the table. Configurable via @@ -98,6 +99,10 @@ class DBSettings final { return compactOnClose; } + uint32_t getLifecycleWaitSeconds() const { + return lifecycleWaitSeconds.load(std::memory_order_relaxed); + } + /** * Returns the global verification table, materializing it on first call. * After the first call, the table size is fixed for the process lifetime. @@ -122,4 +127,4 @@ class DBSettings final { } // namespace rocksdb_js -#endif \ No newline at end of file +#endif diff --git a/src/load-binding.ts b/src/load-binding.ts index 04ff17a3e..22dfa2d96 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -510,6 +510,11 @@ export type RocksDatabaseConfig = { */ verificationTableEntries?: number; compactOnClose?: boolean; + /** + * Maximum seconds an open or destroy call waits for another lifecycle + * operation on the same path. Defaults to 30. + */ + lifecycleWaitSeconds?: number; /** * Total memtable memory limit (bytes) shared across every database opened * in this process. When set, RocksDB uses a single `WriteBufferManager` so diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 0f7cd891a..90366e7ed 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,3 +1,4 @@ +import { RocksDatabase } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; @@ -37,6 +38,13 @@ function runDestroyFixture( } describe('Destroy', () => { + it('validates the lifecycle wait configuration', () => { + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 0 })).toThrow( + 'Lifecycle wait seconds must be a positive integer' + ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); + }); + it('should destroy a closed database', () => dbRunner(async ({ db, dbPath }) => { expect(db.isOpen()).toBe(true); @@ -85,15 +93,13 @@ describe('Destroy', () => { it('waits for physical destruction before reopening the same path', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { - ROCKSDB_JS_DESTROY_DELAY_MS: '1000', + ROCKSDB_JS_DESTROY_DELAY_MS: '2000', }); }, 15_000); - it.skipIf(process.platform === 'win32')( - 'releases the path gate when physical destruction fails', - async () => { - await runDestroyFixture(destroyFailureFixture, generateDBPath()); - }, - 15_000 - ); + it('releases the path gate when physical destruction fails', async () => { + await runDestroyFixture(destroyFailureFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_FAILURE: '1', + }); + }, 15_000); }); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 6b0d31f07..c2cc60a97 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -1,25 +1,19 @@ import { RocksDatabase } from '../../src/index.ts'; -import { chmodSync } from 'node:fs'; const path = process.argv[2]; const db = RocksDatabase.open(path); db.putSync('key', 'value'); -db.close(); -chmodSync(path, 0o500); -let destroyFailed = false; +let destroyError: unknown; try { db.destroy(); -} catch { - destroyFailed = true; -} finally { - chmodSync(path, 0o700); +} catch (error) { + destroyError = error; } -if (!destroyFailed) - throw new Error('Expected destroy to fail for a non-writable database directory'); +if (!String(destroyError).includes('Injected database destruction failure')) + throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); -try { - RocksDatabase.open(path).close(); -} catch { - // Physical destruction may have partially completed; only gate release is asserted. -} +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Reopen after failed destruction did not preserve the database'); +reopened.close(); From 1586b42e7cfe790acad0821ab4b5158e6f4ca4e2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:44:52 -0600 Subject: [PATCH 03/39] Contain lifecycle close failures --- src/binding/binding.cpp | 17 +++++-- src/binding/core/test_seam.h | 4 -- src/binding/database/db_registry.cpp | 66 ++++++++++++++++++++++---- src/binding/database/db_registry.h | 4 ++ src/binding/database/db_settings.cpp | 8 ++-- test/destroy.test.ts | 9 ++++ test/fixtures/fork-destroy-failure.mts | 24 +++++++++- 7 files changed, 111 insertions(+), 21 deletions(-) diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 00b15dd38..efb3f5745 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -40,7 +40,12 @@ namespace rocksdb_js { */ napi_value Shutdown(napi_env env, napi_callback_info info) { GlobalEvents::Shutdown(); - DBRegistry::Shutdown(); + try { + DBRegistry::Shutdown(); + } catch (const std::exception& error) { + ::napi_throw_error(env, nullptr, error.what()); + return nullptr; + } napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; @@ -216,9 +221,13 @@ NAPI_MODULE_INIT() { int32_t newRefCount = --moduleRefCount; if (newRefCount == 0) { DEBUG_LOG("Binding::Init Cleaning up last instance, shutting down all databases\n"); - rocksdb_js::GlobalEvents::Shutdown(); - rocksdb_js::TransactionLogStoreRegistry::Shutdown(); - rocksdb_js::DBRegistry::Shutdown(); + try { + rocksdb_js::GlobalEvents::Shutdown(); + rocksdb_js::TransactionLogStoreRegistry::Shutdown(); + rocksdb_js::DBRegistry::Shutdown(); + } catch (const std::exception& error) { + ::fprintf(stderr, "rocksdb-js cleanup failed: %s\n", error.what()); + } DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { DEBUG_LOG("Binding::Init WARNING: Module ref count went negative!\n"); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index 24126370d..c7e3c15fb 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -16,10 +16,6 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } -inline bool testFailureEnabled(const char* envName) { - return ::getenv(envName) != nullptr; -} - // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index bd06f2e92..086a44e3e 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -47,6 +47,13 @@ struct ClosingDescriptor final { std::shared_ptr descriptor; std::shared_ptr condition; bool closed = false; + std::string closeError; + + ClosingDescriptor( + const DBKey& key, + std::shared_ptr descriptor, + std::shared_ptr condition + ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; } // namespace @@ -208,6 +215,9 @@ void DBRegistry::DestroyDB(const std::string& path) { instance->destroyingPaths, path ); + // Test-only failure modes: 1 fails before physical deletion; 2 fails the + // descriptor-close stage so quarantine and waiter behavior are testable. + const int destroyFailureMode = testDelayMs("ROCKSDB_JS_DESTROY_FAILURE"); while (true) { std::vector claimed; @@ -216,6 +226,13 @@ void DBRegistry::DestroyDB(const std::string& path) { std::lock_guard lock(instance->databasesMutex); claimed.reserve(instance->databases.size()); alreadyClosing.reserve(instance->databases.size()); + for (const auto& [key, entry] : instance->databases) { + if (key.path == path && !entry.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot destroy database \"" + path + "\": previous close failed: " + entry.closeError + ); + } + } for (auto& [key, entry] : instance->databases) { if (key.path != path || !entry.descriptor) { continue; @@ -236,9 +253,18 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { + if (destroyFailureMode == 2) { + throw rocksdb_js::DBException("Injected database close failure"); + } closing.descriptor->finishClose(); closing.closed = true; + } catch (const std::exception& error) { + closing.closeError = error.what(); + if (!closeError) { + closeError = std::current_exception(); + } } catch (...) { + closing.closeError = "unknown native close failure"; if (!closeError) { closeError = std::current_exception(); } @@ -248,19 +274,19 @@ void DBRegistry::DestroyDB(const std::string& path) { { std::lock_guard lock(instance->databasesMutex); for (const auto& closing : claimed) { - if (!closing.closed) { + auto entry = instance->databases.find(closing.key); + if (entry == instance->databases.end() || entry->second.descriptor != closing.descriptor) { continue; } - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; } } } for (const auto& closing : claimed) { - if (closing.closed) { - closing.condition->notify_all(); - } + closing.condition->notify_all(); } if (closeError) { std::rethrow_exception(closeError); @@ -308,7 +334,7 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } - if (testFailureEnabled("ROCKSDB_JS_DESTROY_FAILURE")) { + if (destroyFailureMode == 1) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); @@ -380,6 +406,12 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (entryIterator == instance->databases.end()) { entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; } + if (!entryIterator->second.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot open database \"" + path + "\": previous close failed: " + + entryIterator->second.closeError + ); + } if (!entryIterator->second.descriptor || !entryIterator->second.descriptor->isClosing()) { break; } @@ -389,6 +421,7 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (!condition->wait_until(lock, deadline, [&]() { auto current = instance->databases.find(key); return current == instance->databases.end() || + !current->second.closeError.empty() || !current->second.descriptor || !current->second.descriptor->isClosing(); })) { throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": the previous instance is still closing"); @@ -563,12 +596,18 @@ void DBRegistry::PurgeAll() { ++it; continue; } + if (!it->second.closeError.empty()) { + ++it; + continue; + } + auto condition = it->second.condition; auto descriptor = it->second.descriptor; if (descriptor) { DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); descriptor->close(); } it = instance->databases.erase(it); + condition->notify_all(); #ifdef DEBUG ++i; #endif @@ -844,17 +883,26 @@ void DBRegistry::Shutdown() { try { closing.descriptor->finishClose(); closing.closed = true; + } catch (const std::exception& error) { + closing.closeError = error.what(); + if (!closeError) { + closeError = std::current_exception(); + } } catch (...) { + closing.closeError = "unknown native close failure"; if (!closeError) { closeError = std::current_exception(); } - continue; } { std::lock_guard lock(instance->databasesMutex); auto entry = instance->databases.find(closing.key); if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - instance->databases.erase(entry); + if (closing.closed) { + instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + } } } closing.condition->notify_all(); diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index a8444c2fd..abdcf533a 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -41,6 +41,7 @@ struct DBKeyHash { struct DBRegistryEntry final { std::shared_ptr descriptor; std::shared_ptr condition; + std::string closeError; // Default constructor DBRegistryEntry() : condition(std::make_shared()) {} @@ -79,6 +80,9 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; + // Destruction owns a physical path across every (path, readOnly) entry. + // Waiters must re-resolve databases after every wake because the closer can + // erase the node while the mutex is released. std::condition_variable lifecycleCondition; std::unordered_set destroyingPaths; diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index ee53c89a1..2b6099bcc 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -1,4 +1,5 @@ #include "database/db_settings.h" +#include #include #include #include "napi/macros.h" @@ -215,11 +216,12 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { NAPI_STATUS_THROWS(rocksdb_js::getProperty(env, params, "compactOnClose", settings.compactOnClose, false)); - int64_t lifecycleWaitSeconds = 0; + double lifecycleWaitSeconds = 0; status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); if (status == napi_ok) { - if (lifecycleWaitSeconds <= 0 || - static_cast(lifecycleWaitSeconds) > std::numeric_limits::max() + if (!std::isfinite(lifecycleWaitSeconds) || lifecycleWaitSeconds <= 0 || + std::trunc(lifecycleWaitSeconds) != lifecycleWaitSeconds || + lifecycleWaitSeconds > std::numeric_limits::max() ) { ::napi_throw_range_error(env, nullptr, "Lifecycle wait seconds must be a positive integer"); return nullptr; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 90366e7ed..9d2aa9158 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -42,6 +42,9 @@ describe('Destroy', () => { expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 0 })).toThrow( 'Lifecycle wait seconds must be a positive integer' ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 1.5 })).toThrow( + 'Lifecycle wait seconds must be a positive integer' + ); expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); }); @@ -102,4 +105,10 @@ describe('Destroy', () => { ROCKSDB_JS_DESTROY_FAILURE: '1', }); }, 15_000); + + it('quarantines a descriptor whose native close fails', async () => { + await runDestroyFixture(destroyFailureFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_FAILURE: '2', + }); + }, 15_000); }); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index c2cc60a97..15d8d0704 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -1,6 +1,7 @@ import { RocksDatabase } from '../../src/index.ts'; const path = process.argv[2]; +const failureMode = process.env.ROCKSDB_JS_DESTROY_FAILURE; const db = RocksDatabase.open(path); db.putSync('key', 'value'); @@ -10,9 +11,30 @@ try { } catch (error) { destroyError = error; } -if (!String(destroyError).includes('Injected database destruction failure')) +const expectedError = + failureMode === '2' ? 'Injected database close failure' : 'Injected database destruction failure'; +if (!String(destroyError).includes(expectedError)) throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); +if (failureMode === '2') { + const startedAt = Date.now(); + try { + RocksDatabase.open(path); + throw new Error('Expected the failed descriptor to remain quarantined'); + } catch (error) { + if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; + } + if (Date.now() - startedAt >= 1_000) + throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); + try { + db.destroy(); + throw new Error('Expected repeated destroy to report the previous close failure'); + } catch (error) { + if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; + } + process.exit(0); +} + const reopened = RocksDatabase.open(path); if (reopened.getSync('key') !== 'value') throw new Error('Reopen after failed destruction did not preserve the database'); From 9974ccdff35400c30dcebc802cf85f131fc2e4ad Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:51:35 -0600 Subject: [PATCH 04/39] Close remaining lifecycle race windows --- src/binding/binding.cpp | 7 ++- src/binding/database/db_registry.cpp | 63 ++++++++++++++++++------- src/binding/database/db_registry.h | 1 + test/destroy.test.ts | 16 ++++++- test/fixtures/fork-close-failure.mts | 22 +++++++++ test/fixtures/fork-destroy-failure.mts | 9 ++-- test/fixtures/fork-shutdown-failure.mts | 28 +++++++++++ 7 files changed, 124 insertions(+), 22 deletions(-) create mode 100644 test/fixtures/fork-close-failure.mts create mode 100644 test/fixtures/fork-shutdown-failure.mts diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index efb3f5745..2a85b06ae 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,13 +39,16 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { - GlobalEvents::Shutdown(); try { DBRegistry::Shutdown(); } catch (const std::exception& error) { ::napi_throw_error(env, nullptr, error.what()); return nullptr; + } catch (...) { + ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); + return nullptr; } + GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; @@ -227,6 +230,8 @@ NAPI_MODULE_INIT() { rocksdb_js::DBRegistry::Shutdown(); } catch (const std::exception& error) { ::fprintf(stderr, "rocksdb-js cleanup failed: %s\n", error.what()); + } catch (...) { + ::fprintf(stderr, "rocksdb-js cleanup failed: unknown native error\n"); } DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 086a44e3e..388017bc5 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -56,6 +56,13 @@ struct ClosingDescriptor final { ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; +void finishCloseWithTestSeam(const std::shared_ptr& descriptor) { + if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { + throw rocksdb_js::DBException("Injected database close failure"); + } + descriptor->finishClose(); +} + } // namespace // Initialize the static instance @@ -153,15 +160,27 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (descriptor) { // We claimed the close under the lock via beginClose(); run the actual // teardown now. The local copy keeps the descriptor alive throughout. - descriptor->finishClose(); + std::string closeError; + try { + finishCloseWithTestSeam(descriptor); + } catch (const std::exception& error) { + closeError = error.what(); + } catch (...) { + closeError = "unknown native close failure"; + } std::lock_guard lock(instance->databasesMutex); auto eraseIt = instance->databases.find(key); // Only erase the entry we claimed. A brand-new descriptor cannot appear // because OpenDB blocks until we notify below. - if (eraseIt != instance->databases.end() - && (!eraseIt->second.descriptor || eraseIt->second.descriptor == descriptor)) { - instance->databases.erase(eraseIt); + if (eraseIt != instance->databases.end() && eraseIt->second.descriptor == descriptor) { + if (closeError.empty()) { + instance->databases.erase(eraseIt); + } else { + eraseIt->second.closeError = closeError; + DEBUG_LOG("%p DBRegistry::PurgeIfUnreferenced Quarantined \"%s\": %s\n", + instance.get(), path.c_str(), closeError.c_str()); + } } } @@ -215,10 +234,6 @@ void DBRegistry::DestroyDB(const std::string& path) { instance->destroyingPaths, path ); - // Test-only failure modes: 1 fails before physical deletion; 2 fails the - // descriptor-close stage so quarantine and waiter behavior are testable. - const int destroyFailureMode = testDelayMs("ROCKSDB_JS_DESTROY_FAILURE"); - while (true) { std::vector claimed; std::vector alreadyClosing; @@ -253,10 +268,7 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { - if (destroyFailureMode == 2) { - throw rocksdb_js::DBException("Injected database close failure"); - } - closing.descriptor->finishClose(); + finishCloseWithTestSeam(closing.descriptor); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); @@ -311,7 +323,8 @@ void DBRegistry::DestroyDB(const std::string& path) { std::unique_lock lock(instance->databasesMutex); if (!closing.condition->wait_until(lock, deadline, [&]() { auto entry = instance->databases.find(closing.key); - return entry == instance->databases.end() || entry->second.descriptor != closing.descriptor; + return entry == instance->databases.end() || + !entry->second.closeError.empty() || entry->second.descriptor != closing.descriptor; })) { throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); } @@ -334,7 +347,7 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } - if (destroyFailureMode == 1) { + if (testDelayMs("ROCKSDB_JS_DESTROY_FAILURE") > 0) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); @@ -402,6 +415,15 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } + for (const auto& [registeredKey, registeredEntry] : instance->databases) { + if (registeredKey.path == path && !registeredEntry.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot open database \"" + path + "\": previous close failed: " + + registeredEntry.closeError + ); + } + } + entryIterator = instance->databases.find(key); if (entryIterator == instance->databases.end()) { entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; @@ -603,6 +625,10 @@ void DBRegistry::PurgeAll() { auto condition = it->second.condition; auto descriptor = it->second.descriptor; if (descriptor) { + if (descriptor->isClosing()) { + ++it; + continue; + } DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); descriptor->close(); } @@ -870,7 +896,11 @@ void DBRegistry::Shutdown() { continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; - if (entry.descriptor->beginClose()) { + if (!entry.closeError.empty() && !entry.closeRetrying) { + entry.closeRetrying = true; + descriptorsToClose.push_back(std::move(closing)); + } else if (entry.closeError.empty() && entry.descriptor->beginClose()) { + entry.closeRetrying = true; descriptorsToClose.push_back(std::move(closing)); } } @@ -881,7 +911,7 @@ void DBRegistry::Shutdown() { for (auto& closing : descriptorsToClose) { DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); try { - closing.descriptor->finishClose(); + finishCloseWithTestSeam(closing.descriptor); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); @@ -902,6 +932,7 @@ void DBRegistry::Shutdown() { instance->databases.erase(entry); } else { entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; } } } diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index abdcf533a..b2cd7b9d1 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -42,6 +42,7 @@ struct DBRegistryEntry final { std::shared_ptr descriptor; std::shared_ptr condition; std::string closeError; + bool closeRetrying = false; // Default constructor DBRegistryEntry() : condition(std::make_shared()) {} diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 9d2aa9158..abc9607e3 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -7,6 +7,8 @@ import { describe, expect, it } from 'vitest'; const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); +const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); +const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); function runDestroyFixture( fixture: string, @@ -108,7 +110,19 @@ describe('Destroy', () => { it('quarantines a descriptor whose native close fails', async () => { await runDestroyFixture(destroyFailureFixture, generateDBPath(), { - ROCKSDB_JS_DESTROY_FAILURE: '2', + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); + + it('surfaces shutdown close failures and quarantines the whole path', async () => { + await runDestroyFixture(shutdownFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); + + it('quarantines a failed automatic last-handle close', async () => { + await runDestroyFixture(closeFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', }); }, 15_000); }); diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts new file mode 100644 index 000000000..765caad7d --- /dev/null +++ b/test/fixtures/fork-close-failure.mts @@ -0,0 +1,22 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); +db.close(); + +const startedAt = Date.now(); +try { + RocksDatabase.open(path); + throw new Error('Expected the failed automatic close to quarantine the path'); +} catch (error) { + if (!String(error).includes('previous close failed: Injected database close failure')) + throw error; +} +if (Date.now() - startedAt >= 1_000) + throw new Error('Open waited instead of reporting the failed automatic close immediately'); + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +shutdown(); +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Shutdown retry did not clear the quarantined automatic close'); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 15d8d0704..ca3d99eaf 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -1,7 +1,7 @@ import { RocksDatabase } from '../../src/index.ts'; const path = process.argv[2]; -const failureMode = process.env.ROCKSDB_JS_DESTROY_FAILURE; +const closeFailure = process.env.ROCKSDB_JS_CLOSE_FAILURE === '1'; const db = RocksDatabase.open(path); db.putSync('key', 'value'); @@ -11,12 +11,13 @@ try { } catch (error) { destroyError = error; } -const expectedError = - failureMode === '2' ? 'Injected database close failure' : 'Injected database destruction failure'; +const expectedError = closeFailure + ? 'Injected database close failure' + : 'Injected database destruction failure'; if (!String(destroyError).includes(expectedError)) throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); -if (failureMode === '2') { +if (closeFailure) { const startedAt = Date.now(); try { RocksDatabase.open(path); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts new file mode 100644 index 000000000..66fc09d23 --- /dev/null +++ b/test/fixtures/fork-shutdown-failure.mts @@ -0,0 +1,28 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); + +try { + shutdown(); + throw new Error('Expected shutdown to surface the injected close failure'); +} catch (error) { + if (!String(error).includes('Injected database close failure')) throw error; +} + +const startedAt = Date.now(); +try { + RocksDatabase.open(path, { readOnly: true }); + throw new Error('Expected the failed path to remain quarantined across open modes'); +} catch (error) { + if (!String(error).includes('previous close failed: Injected database close failure')) + throw error; +} +if (Date.now() - startedAt >= 1_000) + throw new Error('Cross-mode open waited instead of reporting the quarantined path immediately'); + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +shutdown(); +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Shutdown retry did not clear the quarantined descriptor'); From 98d007be093a981a99a174d4ebf9aed16b7eaa5b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:01:09 -0600 Subject: [PATCH 05/39] Make descriptor close retries resumable --- src/binding/binding.cpp | 21 ++++--- src/binding/database/database.cpp | 9 ++- src/binding/database/db_descriptor.cpp | 73 +++++++++++++------------ src/binding/database/db_descriptor.h | 5 ++ src/binding/database/db_registry.cpp | 76 ++++++++++++++++---------- src/binding/database/db_registry.h | 5 +- src/load-binding.ts | 1 + test/fixtures/fork-close-failure.mts | 12 +++- 8 files changed, 127 insertions(+), 75 deletions(-) diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 2a85b06ae..19aecb435 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -224,15 +224,18 @@ NAPI_MODULE_INIT() { int32_t newRefCount = --moduleRefCount; if (newRefCount == 0) { DEBUG_LOG("Binding::Init Cleaning up last instance, shutting down all databases\n"); - try { - rocksdb_js::GlobalEvents::Shutdown(); - rocksdb_js::TransactionLogStoreRegistry::Shutdown(); - rocksdb_js::DBRegistry::Shutdown(); - } catch (const std::exception& error) { - ::fprintf(stderr, "rocksdb-js cleanup failed: %s\n", error.what()); - } catch (...) { - ::fprintf(stderr, "rocksdb-js cleanup failed: unknown native error\n"); - } + auto cleanup = [](const char* name, auto shutdown) { + try { + shutdown(); + } catch (const std::exception& error) { + ::fprintf(stderr, "rocksdb-js %s cleanup failed: %s\n", name, error.what()); + } catch (...) { + ::fprintf(stderr, "rocksdb-js %s cleanup failed: unknown native error\n", name); + } + }; + cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); + cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); + cleanup("database registry", []() { rocksdb_js::DBRegistry::Shutdown(); }); DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { DEBUG_LOG("Binding::Init WARNING: Module ref count went negative!\n"); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5da78bc40..8885a6764 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -190,7 +190,11 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { if (*dbHandle) { DEBUG_LOG("%p Database::Close Closing database: \"%s\"\n", dbHandle->get(), (*dbHandle)->path.c_str()); - DBRegistry::CloseDB(*dbHandle); + std::string closeError = DBRegistry::CloseDB(*dbHandle); + if (!closeError.empty()) { + ::napi_throw_error(env, nullptr, closeError.c_str()); + return nullptr; + } DEBUG_LOG("%p Database::Close Closed database\n", dbHandle->get()); } else { DEBUG_LOG("%p Database::Close Database not opened\n", dbHandle->get()); @@ -447,6 +451,9 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { DEBUG_LOG("%p Database::Destroy Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); return nullptr; + } catch (...) { + ::napi_throw_error(env, nullptr, "Unknown native database destruction failure"); + return nullptr; } } else { ::napi_throw_error(env, nullptr, "Invalid database handle"); diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index f11076ff1..15f631814 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -1,5 +1,6 @@ #include "core/background_error.h" #include "core/platform.h" +#include "core/test_seam.h" #include "database/db_descriptor.h" #include "database/db_settings.h" #include "napi/helpers.h" @@ -382,41 +383,42 @@ void DBDescriptor::finishClose() { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); - // Wait for all in-flight operations to complete before cleanup. - // The closing flag is already set, so new operations will fail with "Database is closing". - // Existing operations will decrement operationsInFlight and notify us when done. - DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); - uint32_t current; - while ((current = this->operationsInFlight.load()) != 0) { - this->operationsInFlight.wait(current); - } - DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); - - // Drain the commit pipeline before flushing so its data is included in - // the flush. The log lane feeds the commit lane, so it must drain first; - // its final tasks enqueue onto the still-running commit lane (or run - // inline once that lane stops). - this->logWorker.shutdown(); - this->commitWorker.shutdown(); - - // Release any remaining per-env commit-completion tsfns. An in-flight - // commit pins this descriptor (state -> txnHandle -> dbHandle -> descriptor), - // so reaching here means no commit is in flight; only idle (unref'd) tsfns - // for still-living envs can remain, and those envs will issue no further - // commits to this descriptor. Queued completions already handed to a tsfn - // are still delivered (napi_tsfn_release, not abort). - { - std::lock_guard lock(this->commitMutex); - for (auto& [env, completion] : this->commitCompletions) { - if (completion.tsfn) { - ::napi_release_threadsafe_function(completion.tsfn, napi_tsfn_release); + if (!this->closeWorkersStopped) { + // Wait for all in-flight operations to complete before cleanup. + // The closing flag is already set, so new operations will fail with "Database is closing". + // Existing operations will decrement operationsInFlight and notify us when done. + DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); + uint32_t current; + while ((current = this->operationsInFlight.load()) != 0) { + this->operationsInFlight.wait(current); + } + DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); + + // Drain the commit pipeline before flushing so its data is included in + // the flush. The log lane feeds the commit lane, so it must drain first. + this->logWorker.shutdown(); + this->commitWorker.shutdown(); + + { + std::lock_guard lock(this->commitMutex); + for (auto& [env, completion] : this->commitCompletions) { + if (completion.tsfn) { + ::napi_release_threadsafe_function(completion.tsfn, napi_tsfn_release); + } } + this->commitCompletions.clear(); + this->commitCompletionsClosed = true; } - this->commitCompletions.clear(); - // Block any later registerCommitCompletion (a commit racing this close - // from another env) from re-creating a tsfn that would never be - // released; such commits fall back to the legacy libuv path. - this->commitCompletionsClosed = true; + this->closeWorkersStopped = true; + } + + // Inject after the one-shot pipeline shutdown so retry coverage exercises + // a genuinely partially-completed close rather than an untouched descriptor. + if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { + throw rocksdb_js::DBException("Injected database close failure"); + } + if (!this->db) { + return; } // We want to ensure that all in-memory data is written to disk. Keeps the waiting default on @@ -488,7 +490,10 @@ void DBDescriptor::finishClose() { // Unregister from transaction log store registry - this will clean up stores // when the last descriptor for this path is closed - TransactionLogStoreRegistry::Unregister(this->path); + if (!this->transactionLogsUnregistered) { + TransactionLogStoreRegistry::Unregister(this->path); + this->transactionLogsUnregistered = true; + } this->transactions.clear(); { diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 0444e4ed7..4bf7f2ff2 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -286,6 +286,11 @@ struct DBDescriptor final : public std::enable_shared_from_this { * descriptor. */ std::atomic closing{false}; + // finishClose() can be retried after a quarantined failure. These guards + // prevent its one-shot stages from running twice while later idempotent + // cleanup resumes from the failed point. + bool closeWorkersStopped = false; + bool transactionLogsUnregistered = false; /** * Counter tracking in-flight database operations. close() uses diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 388017bc5..a26fabda4 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -56,13 +56,6 @@ struct ClosingDescriptor final { ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; -void finishCloseWithTestSeam(const std::shared_ptr& descriptor) { - if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { - throw rocksdb_js::DBException("Injected database close failure"); - } - descriptor->finishClose(); -} - } // namespace // Initialize the static instance @@ -71,15 +64,15 @@ std::unique_ptr DBRegistry::instance; /** * Close a RocksDB database handle. */ -void DBRegistry::CloseDB(const std::shared_ptr handle) { +std::string DBRegistry::CloseDB(const std::shared_ptr handle) { if (!instance) { DEBUG_LOG("%p DBRegistry::CloseDB Registry not initialized\n", instance.get()); - return; + return {}; } if (!handle) { DEBUG_LOG("%p DBRegistry::CloseDB Invalid handle\n", instance.get()); - return; + return {}; } #ifdef DEBUG @@ -88,7 +81,7 @@ void DBRegistry::CloseDB(const std::shared_ptr handle) { if (!handle->descriptor) { DEBUG_LOG("%p DBRegistry::CloseDB Database not opened\n", instance.get()); - return; + return {}; } DBKey key{handle->descriptor->path, handle->descriptor->readOnly}; @@ -98,7 +91,7 @@ void DBRegistry::CloseDB(const std::shared_ptr handle) { // close the handle, decrements the descriptor ref count handle->close(); - DBRegistry::PurgeIfUnreferenced(key.path, key.readOnly); + return DBRegistry::PurgeIfUnreferenced(key.path, key.readOnly); } /** @@ -133,14 +126,15 @@ void DBRegistry::CloseDB(const std::shared_ptr handle) { * the duration of finishClose(), so a concurrent OpenDB keeps waiting on * the condition rather than re-opening the path mid-close. */ -void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { +std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (!instance) { - return; + return {}; } DBKey key{path, readOnly}; std::shared_ptr descriptor; std::shared_ptr condition; + std::string closeError; { std::lock_guard lock(instance->databasesMutex); auto entryIterator = instance->databases.find(key); @@ -160,9 +154,8 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (descriptor) { // We claimed the close under the lock via beginClose(); run the actual // teardown now. The local copy keeps the descriptor alive throughout. - std::string closeError; try { - finishCloseWithTestSeam(descriptor); + descriptor->finishClose(); } catch (const std::exception& error) { closeError = error.what(); } catch (...) { @@ -188,6 +181,7 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (condition) { condition->notify_all(); } + return closeError; } /** @@ -244,7 +238,8 @@ void DBRegistry::DestroyDB(const std::string& path) { for (const auto& [key, entry] : instance->databases) { if (key.path == path && !entry.closeError.empty()) { throw rocksdb_js::DBException( - "Cannot destroy database \"" + path + "\": previous close failed: " + entry.closeError + "Cannot destroy database \"" + path + "\": previous close failed: " + + entry.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -268,7 +263,7 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { - finishCloseWithTestSeam(closing.descriptor); + closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); @@ -419,7 +414,7 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (registeredKey.path == path && !registeredEntry.closeError.empty()) { throw rocksdb_js::DBException( "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry.closeError + registeredEntry.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -428,12 +423,6 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (entryIterator == instance->databases.end()) { entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; } - if (!entryIterator->second.closeError.empty()) { - throw rocksdb_js::DBException( - "Cannot open database \"" + path + "\": previous close failed: " + - entryIterator->second.closeError - ); - } if (!entryIterator->second.descriptor || !entryIterator->second.descriptor->isClosing()) { break; } @@ -607,7 +596,9 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons */ void DBRegistry::PurgeAll() { if (instance) { - std::lock_guard lock(instance->databasesMutex); + std::exception_ptr closeError; + { + std::lock_guard lock(instance->databasesMutex); #ifdef DEBUG size_t initialSize = instance->databases.size(); DEBUG_LOG("%p DBRegistry::PurgeAll Purging %zu databases:\n", instance.get(), instance->databases.size()); @@ -630,7 +621,21 @@ void DBRegistry::PurgeAll() { continue; } DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); - descriptor->close(); + try { + descriptor->close(); + } catch (const std::exception& error) { + it->second.closeError = error.what(); + condition->notify_all(); + if (!closeError) closeError = std::current_exception(); + ++it; + continue; + } catch (...) { + it->second.closeError = "unknown native close failure"; + condition->notify_all(); + if (!closeError) closeError = std::current_exception(); + ++it; + continue; + } } it = instance->databases.erase(it); condition->notify_all(); @@ -647,6 +652,10 @@ void DBRegistry::PurgeAll() { currentSize ); #endif + } + if (closeError) { + std::rethrow_exception(closeError); + } } } @@ -675,6 +684,16 @@ napi_value DBRegistry::RegistryStatus(napi_env env, napi_callback_info info) { napi_value pathValue; NAPI_STATUS_THROWS(::napi_create_string_utf8(env, key.path.c_str(), key.path.size(), &pathValue)); NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "path", pathValue)); + if (!entry.closeError.empty()) { + napi_value closeErrorValue; + NAPI_STATUS_THROWS(::napi_create_string_utf8( + env, + entry.closeError.c_str(), + entry.closeError.size(), + &closeErrorValue + )); + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "closeError", closeErrorValue)); + } napi_value modeValue; std::string mode = entry.descriptor->mode == DBMode::Optimistic ? "optimistic" : "pessimistic"; NAPI_STATUS_THROWS(::napi_create_string_utf8(env, mode.c_str(), mode.size(), &modeValue)); @@ -880,6 +899,7 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { + std::lock_guard shutdownLock(instance->shutdownMutex); std::vector descriptorsToClose; { @@ -911,7 +931,7 @@ void DBRegistry::Shutdown() { for (auto& closing : descriptorsToClose) { DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); try { - finishCloseWithTestSeam(closing.descriptor); + closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index b2cd7b9d1..cf7486d93 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -81,6 +81,7 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; + std::mutex shutdownMutex; // Destruction owns a physical path across every (path, readOnly) entry. // Waiters must re-resolve databases after every wake because the closer can // erase the node while the mutex is released. @@ -93,7 +94,7 @@ class DBRegistry final { static std::unique_ptr instance; public: - static void CloseDB(const std::shared_ptr handle); + static std::string CloseDB(const std::shared_ptr handle); #ifdef DEBUG static void DebugLogDescriptorRefs(); #endif @@ -101,7 +102,7 @@ class DBRegistry final { static void Init(napi_env env, napi_value exports); static std::unique_ptr OpenDB(const std::string& path, const DBOptions& options); static void PurgeAll(); - static void PurgeIfUnreferenced(const std::string& path, bool readOnly); + static std::string PurgeIfUnreferenced(const std::string& path, bool readOnly); static napi_value RegistryStatus(napi_env env, napi_callback_info info); static void CloseTransactionsByEnv(napi_env env); static void RemoveListenersByEnv(napi_env env); diff --git a/src/load-binding.ts b/src/load-binding.ts index 22dfa2d96..e08b2b46d 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -633,6 +633,7 @@ export type RegistryStatusTransaction = { export type RegistryStatusDB = { path: string; + closeError?: string; refCount: number; columnFamilies: string[]; transactions: number; diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts index 765caad7d..c2687e7a7 100644 --- a/test/fixtures/fork-close-failure.mts +++ b/test/fixtures/fork-close-failure.mts @@ -3,7 +3,17 @@ import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; const path = process.argv[2]; const db = RocksDatabase.open(path); db.putSync('key', 'value'); -db.close(); +try { + db.close(); + throw new Error('Expected close to surface the injected native failure'); +} catch (error) { + if (!String(error).includes('Injected database close failure')) throw error; +} +if ( + registryStatus().find((entry) => entry.path === path)?.closeError !== + 'Injected database close failure' +) + throw new Error('Registry status did not expose the quarantined close failure'); const startedAt = Date.now(); try { From f275098d29fb4563eb7e6f190884d95ad5d82c40 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:13:40 -0600 Subject: [PATCH 06/39] Resolve lifecycle review findings --- src/binding/binding.cpp | 11 ++- src/binding/database/database.cpp | 10 ++- src/binding/database/db_descriptor.cpp | 2 +- src/binding/database/db_registry.cpp | 111 ++++++++++++++---------- src/binding/database/db_registry.h | 2 +- test/destroy.test.ts | 9 +- test/fixtures/fork-gc-close-failure.mts | 37 ++++++++ test/fixtures/fork-shutdown-failure.mts | 1 - 8 files changed, 126 insertions(+), 57 deletions(-) create mode 100644 test/fixtures/fork-gc-close-failure.mts diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 19aecb435..674acbb96 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,16 +39,19 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { + std::string shutdownError; try { DBRegistry::Shutdown(); } catch (const std::exception& error) { - ::napi_throw_error(env, nullptr, error.what()); - return nullptr; + shutdownError = error.what(); } catch (...) { - ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); - return nullptr; + shutdownError = "Unknown native database shutdown failure"; } GlobalEvents::Shutdown(); + if (!shutdownError.empty()) { + ::napi_throw_error(env, nullptr, shutdownError.c_str()); + return nullptr; + } napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 8885a6764..842907835 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -16,6 +16,7 @@ #include "core/platform.h" #include "napi/helpers.h" #include "napi/async.h" +#include "napi/global_events.h" #include "core/verification_table.h" #include "core/compression.h" @@ -48,7 +49,14 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) { DEBUG_LOG("Database::Constructor NativeDatabase GC'd dbHandle=%p\n", data); auto* dbHandle = static_cast*>(data); if (*dbHandle) { - DBRegistry::CloseDB(*dbHandle); + std::string path = (*dbHandle)->path; + std::string closeError = DBRegistry::CloseDB(*dbHandle); + if (!closeError.empty() && GlobalEvents::hasListeners()) { + emitGlobalEvent( + "database:closeFailed", + ListenerData::fromStrings({path, closeError}) + ); + } } delete dbHandle; }, diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 15f631814..ac10c18c5 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -491,8 +491,8 @@ void DBDescriptor::finishClose() { // Unregister from transaction log store registry - this will clean up stores // when the last descriptor for this path is closed if (!this->transactionLogsUnregistered) { - TransactionLogStoreRegistry::Unregister(this->path); this->transactionLogsUnregistered = true; + TransactionLogStoreRegistry::Unregister(this->path); } this->transactions.clear(); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index a26fabda4..e560c20a7 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -235,11 +235,12 @@ void DBRegistry::DestroyDB(const std::string& path) { std::lock_guard lock(instance->databasesMutex); claimed.reserve(instance->databases.size()); alreadyClosing.reserve(instance->databases.size()); - for (const auto& [key, entry] : instance->databases) { - if (key.path == path && !entry.closeError.empty()) { + for (bool readOnly : {false, true}) { + auto entry = instance->databases.find(DBKey{path, readOnly}); + if (entry != instance->databases.end() && !entry->second.closeError.empty()) { throw rocksdb_js::DBException( "Cannot destroy database \"" + path + "\": previous close failed: " + - entry.closeError + ". Call shutdown() to retry cleanup" + entry->second.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -410,11 +411,12 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } - for (const auto& [registeredKey, registeredEntry] : instance->databases) { - if (registeredKey.path == path && !registeredEntry.closeError.empty()) { + for (bool readOnly : {false, true}) { + auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); + if (registeredEntry != instance->databases.end() && !registeredEntry->second.closeError.empty()) { throw rocksdb_js::DBException( "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry.closeError + ". Call shutdown() to retry cleanup" + registeredEntry->second.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -596,62 +598,70 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons */ void DBRegistry::PurgeAll() { if (instance) { + std::vector descriptorsToClose; + std::vector> removedConditions; std::exception_ptr closeError; { std::lock_guard lock(instance->databasesMutex); #ifdef DEBUG - size_t initialSize = instance->databases.size(); - DEBUG_LOG("%p DBRegistry::PurgeAll Purging %zu databases:\n", instance.get(), instance->databases.size()); - uint32_t i = 0; + size_t initialSize = instance->databases.size(); + DEBUG_LOG("%p DBRegistry::PurgeAll Purging %zu databases:\n", instance.get(), initialSize); #endif - for (auto it = instance->databases.begin(); it != instance->databases.end();) { - if (instance->destroyingPaths.find(it->first.path) != instance->destroyingPaths.end()) { - ++it; - continue; - } - if (!it->second.closeError.empty()) { - ++it; - continue; - } - auto condition = it->second.condition; - auto descriptor = it->second.descriptor; - if (descriptor) { - if (descriptor->isClosing()) { + descriptorsToClose.reserve(instance->databases.size()); + for (auto it = instance->databases.begin(); it != instance->databases.end();) { + if (instance->destroyingPaths.find(it->first.path) != instance->destroyingPaths.end() || + !it->second.closeError.empty() + ) { ++it; continue; } - DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); - try { - descriptor->close(); - } catch (const std::exception& error) { - it->second.closeError = error.what(); - condition->notify_all(); - if (!closeError) closeError = std::current_exception(); - ++it; - continue; - } catch (...) { - it->second.closeError = "unknown native close failure"; - condition->notify_all(); - if (!closeError) closeError = std::current_exception(); + auto descriptor = it->second.descriptor; + if (descriptor) { + if (!descriptor->beginClose()) { + ++it; + continue; + } + DEBUG_LOG("%p DBRegistry::PurgeAll Claiming \"%s\" (ref count = %ld)\n", + instance.get(), it->first.path.c_str(), descriptor.use_count()); + descriptorsToClose.emplace_back(it->first, descriptor, it->second.condition); ++it; continue; } + removedConditions.push_back(it->second.condition); + it = instance->databases.erase(it); } - it = instance->databases.erase(it); - condition->notify_all(); #ifdef DEBUG - ++i; + DEBUG_LOG("%p DBRegistry::PurgeAll Claimed %zu of %zu descriptors\n", + instance.get(), descriptorsToClose.size(), initialSize); #endif } -#ifdef DEBUG - size_t currentSize = instance->databases.size(); - DEBUG_LOG( - "%p DBRegistry::PurgeAll Purged %zu unused descriptors (size=%zu)\n", - instance.get(), - initialSize - currentSize, - currentSize - ); -#endif + for (const auto& condition : removedConditions) { + condition->notify_all(); + } + + for (auto& closing : descriptorsToClose) { + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (const std::exception& error) { + closing.closeError = error.what(); + if (!closeError) closeError = std::current_exception(); + } catch (...) { + closing.closeError = "unknown native close failure"; + if (!closeError) closeError = std::current_exception(); + } + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { + instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + } + } + } + closing.condition->notify_all(); } if (closeError) { std::rethrow_exception(closeError); @@ -899,7 +909,12 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { - std::lock_guard shutdownLock(instance->shutdownMutex); + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); + std::unique_lock shutdownLock(instance->shutdownMutex, std::defer_lock); + if (!shutdownLock.try_lock_until(deadline)) { + throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); + } std::vector descriptorsToClose; { diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index cf7486d93..cbddbbfe0 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -81,7 +81,7 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; - std::mutex shutdownMutex; + std::timed_mutex shutdownMutex; // Destruction owns a physical path across every (path, readOnly) entry. // Waiters must re-resolve databases after every wake because the closer can // erase the node while the mutex is released. diff --git a/test/destroy.test.ts b/test/destroy.test.ts index abc9607e3..bca814e1e 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest'; const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); +const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); function runDestroyFixture( @@ -16,7 +17,7 @@ function runDestroyFixture( env?: NodeJS.ProcessEnv ): Promise { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [fixture, dbPath], { + const child = spawn(process.execPath, ['--expose-gc', fixture, dbPath], { env: { ...process.env, ...env }, }); let stderr = ''; @@ -121,6 +122,12 @@ describe('Destroy', () => { }, 15_000); it('quarantines a failed automatic last-handle close', async () => { + await runDestroyFixture(gcCloseFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); + + it('surfaces an explicit close failure and permits a shutdown retry', async () => { await runDestroyFixture(closeFailureFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FAILURE: '1', }); diff --git a/test/fixtures/fork-gc-close-failure.mts b/test/fixtures/fork-gc-close-failure.mts new file mode 100644 index 000000000..a028fe978 --- /dev/null +++ b/test/fixtures/fork-gc-close-failure.mts @@ -0,0 +1,37 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; +import { setTimeout as delay } from 'node:timers/promises'; + +const path = process.argv[2]; +let db: RocksDatabase | undefined = RocksDatabase.open(path); +db.putSync('key', 'value'); + +let resolveCloseFailure!: (args: unknown[]) => void; +const closeFailure = new Promise((resolve) => { + resolveCloseFailure = resolve; +}); +RocksDatabase.on('database:closeFailed', (...args) => resolveCloseFailure(args)); + +db = undefined; +for (let attempt = 0; attempt < 40; attempt++) { + global.gc!(); + await delay(25); +} + +const args = await Promise.race([ + closeFailure, + delay(1_000).then(() => { + throw new Error('Automatic close failure did not emit database:closeFailed'); + }), +]); +if (args[0] !== path || args[1] !== 'Injected database close failure') { + throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); +} +if ( + registryStatus().find((entry) => entry.path === path)?.closeError !== + 'Injected database close failure' +) { + throw new Error('Automatic close failure was not quarantined'); +} + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +shutdown(); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 66fc09d23..6dc4f858e 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -10,7 +10,6 @@ try { } catch (error) { if (!String(error).includes('Injected database close failure')) throw error; } - const startedAt = Date.now(); try { RocksDatabase.open(path, { readOnly: true }); From 370da80a90e97fcf22fc8f4fde0abba07b27c2d3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:32:34 -0600 Subject: [PATCH 07/39] Contain close failures to the affected database --- README.md | 6 ++--- src/binding/binding.cpp | 11 ++++----- src/binding/database/database.cpp | 13 +++-------- src/binding/database/db_descriptor.cpp | 14 +++++++---- src/binding/database/db_registry.cpp | 31 ++++++++++++++----------- src/binding/database/db_settings.cpp | 10 ++++++-- src/database.ts | 4 +++- test/destroy.test.ts | 10 ++++++++ test/fixtures/fork-close-failure.mts | 4 ++++ test/fixtures/fork-destroy-failure.mts | 8 ++----- test/fixtures/fork-destroy-open.mts | 12 +++++++++- test/fixtures/fork-gc-close-failure.mts | 4 ++++ test/fixtures/fork-shutdown-failure.mts | 4 ++++ 13 files changed, 84 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 02b8717a3..426065f26 100644 --- a/README.md +++ b/README.md @@ -183,9 +183,9 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. - - `lifecycleWaitSeconds: number` Maximum time a synchronous open or destroy waits for another - lifecycle operation on the same path before throwing a retryable timeout error. Defaults to - `30` seconds and must be a positive integer. + - `lifecycleWaitSeconds: number` Maximum time a synchronous open, destroy, or shutdown waits for + another lifecycle operation before throwing a retryable timeout error. Defaults to `30` seconds + and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global [Verification Table](#verification-table). Each slot is 8 bytes, so the default of `131072` (128K) slots is 1 MB. Set to `0` to disable the verification table. This must be configured diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 674acbb96..19aecb435 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,19 +39,16 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { - std::string shutdownError; try { DBRegistry::Shutdown(); } catch (const std::exception& error) { - shutdownError = error.what(); + ::napi_throw_error(env, nullptr, error.what()); + return nullptr; } catch (...) { - shutdownError = "Unknown native database shutdown failure"; - } - GlobalEvents::Shutdown(); - if (!shutdownError.empty()) { - ::napi_throw_error(env, nullptr, shutdownError.c_str()); + ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); return nullptr; } + GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 842907835..da1bc7d51 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -16,7 +16,6 @@ #include "core/platform.h" #include "napi/helpers.h" #include "napi/async.h" -#include "napi/global_events.h" #include "core/verification_table.h" #include "core/compression.h" @@ -49,14 +48,7 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) { DEBUG_LOG("Database::Constructor NativeDatabase GC'd dbHandle=%p\n", data); auto* dbHandle = static_cast*>(data); if (*dbHandle) { - std::string path = (*dbHandle)->path; - std::string closeError = DBRegistry::CloseDB(*dbHandle); - if (!closeError.empty() && GlobalEvents::hasListeners()) { - emitGlobalEvent( - "database:closeFailed", - ListenerData::fromStrings({path, closeError}) - ); - } + DBRegistry::CloseDB(*dbHandle); } delete dbHandle; }, @@ -200,7 +192,8 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { DEBUG_LOG("%p Database::Close Closing database: \"%s\"\n", dbHandle->get(), (*dbHandle)->path.c_str()); std::string closeError = DBRegistry::CloseDB(*dbHandle); if (!closeError.empty()) { - ::napi_throw_error(env, nullptr, closeError.c_str()); + std::string message = closeError + ". Call destroy() or shutdown() to retry cleanup"; + ::napi_throw_error(env, nullptr, message.c_str()); return nullptr; } DEBUG_LOG("%p Database::Close Closed database\n", dbHandle->get()); diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index ac10c18c5..b27f000c6 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -421,9 +421,12 @@ void DBDescriptor::finishClose() { return; } - // We want to ensure that all in-memory data is written to disk. Keeps the waiting default on - // purpose: flushing immediately here races transaction-log-store teardown (AGENTS invariant 15). - this->flush(); + // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an + // immediate flush races transaction-log-store teardown (AGENTS invariant 15). + rocksdb::Status status = this->flush(); + if (!status.ok()) { + throw rocksdb_js::DBException("Failed to flush database during close: " + status.ToString()); + } // Trigger manual compaction on all column families to reclaim space from // tombstones before closing @@ -451,7 +454,10 @@ void DBDescriptor::finishClose() { // suggestions of the documentation, this method alone does not seem to // trigger a flush rocksdb::WaitForCompactOptions options; - this->db->WaitForCompact(options); + status = this->db->WaitForCompact(options); + if (!status.ok()) { + throw rocksdb_js::DBException("Failed waiting for database compaction during close: " + status.ToString()); + } std::unique_lock txnsLock(this->txnsMutex); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index e560c20a7..da3ecde4d 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -9,6 +9,7 @@ #include "core/compression.h" #include "napi/helpers.h" #include "napi/async.h" +#include "napi/global_events.h" #include "rocksdb/table.h" #include #include @@ -181,6 +182,9 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (condition) { condition->notify_all(); } + if (!closeError.empty() && GlobalEvents::hasListeners()) { + emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, closeError})); + } return closeError; } @@ -235,21 +239,15 @@ void DBRegistry::DestroyDB(const std::string& path) { std::lock_guard lock(instance->databasesMutex); claimed.reserve(instance->databases.size()); alreadyClosing.reserve(instance->databases.size()); - for (bool readOnly : {false, true}) { - auto entry = instance->databases.find(DBKey{path, readOnly}); - if (entry != instance->databases.end() && !entry->second.closeError.empty()) { - throw rocksdb_js::DBException( - "Cannot destroy database \"" + path + "\": previous close failed: " + - entry->second.closeError + ". Call shutdown() to retry cleanup" - ); - } - } for (auto& [key, entry] : instance->databases) { if (key.path != path || !entry.descriptor) { continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; - if (entry.descriptor->beginClose()) { + if (!entry.closeError.empty() && !entry.closeRetrying) { + entry.closeRetrying = true; + claimed.push_back(std::move(closing)); + } else if (entry.descriptor->beginClose()) { claimed.push_back(std::move(closing)); } else { alreadyClosing.push_back(std::move(closing)); @@ -290,6 +288,7 @@ void DBRegistry::DestroyDB(const std::string& path) { instance->databases.erase(entry); } else { entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; } } } @@ -320,7 +319,8 @@ void DBRegistry::DestroyDB(const std::string& path) { if (!closing.condition->wait_until(lock, deadline, [&]() { auto entry = instance->databases.find(closing.key); return entry == instance->databases.end() || - !entry->second.closeError.empty() || entry->second.descriptor != closing.descriptor; + entry->second.descriptor != closing.descriptor || + (!entry->second.closeError.empty() && !entry->second.closeRetrying); })) { throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); } @@ -416,7 +416,7 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (registeredEntry != instance->databases.end() && !registeredEntry->second.closeError.empty()) { throw rocksdb_js::DBException( "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry->second.closeError + ". Call shutdown() to retry cleanup" + registeredEntry->second.closeError + ". Call destroy() or shutdown() to retry cleanup" ); } } @@ -918,7 +918,12 @@ void DBRegistry::Shutdown() { std::vector descriptorsToClose; { - std::lock_guard lock(instance->databasesMutex); + std::unique_lock lock(instance->databasesMutex); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.empty(); + })) { + throw rocksdb_js::DBException("Timed out waiting for database destruction to finish before shutdown"); + } DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); descriptorsToClose.reserve(instance->databases.size()); diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index 2b6099bcc..77a5ee27b 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -217,8 +217,14 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { NAPI_STATUS_THROWS(rocksdb_js::getProperty(env, params, "compactOnClose", settings.compactOnClose, false)); double lifecycleWaitSeconds = 0; - status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); - if (status == napi_ok) { + bool lifecycleWaitProvided = false; + NAPI_STATUS_THROWS(::napi_has_named_property(env, params, "lifecycleWaitSeconds", &lifecycleWaitProvided)); + if (lifecycleWaitProvided) { + status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); + if (status != napi_ok) { + ::napi_throw_type_error(env, nullptr, "Lifecycle wait seconds must be a number"); + return nullptr; + } if (!std::isfinite(lifecycleWaitSeconds) || lifecycleWaitSeconds <= 0 || std::trunc(lifecycleWaitSeconds) != lifecycleWaitSeconds || lifecycleWaitSeconds > std::numeric_limits::max() diff --git a/src/database.ts b/src/database.ts index 8a677e124..f534adf2a 100644 --- a/src/database.ts +++ b/src/database.ts @@ -347,7 +347,9 @@ export class RocksDatabase extends DBI { * native binding use namespaced keys (e.g. `'transactionLog:warning'`). * * Listeners are not tied to any specific database — they fire for every - * matching event emitted in this process. + * matching event emitted in this process. Native lifecycle failures use + * `'database:closeFailed'` with `(path, error)` string arguments; the path + * remains quarantined until `destroy()` or `shutdown()` retries cleanup. * * @example * ```typescript diff --git a/test/destroy.test.ts b/test/destroy.test.ts index bca814e1e..2c558cfe5 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -48,6 +48,9 @@ describe('Destroy', () => { expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 1.5 })).toThrow( 'Lifecycle wait seconds must be a positive integer' ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: '30' as unknown as number })).toThrow( + 'Lifecycle wait seconds must be a number' + ); expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); }); @@ -103,6 +106,13 @@ describe('Destroy', () => { }); }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { + await runDestroyFixture(destroyOpenFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_DELAY_MS: '2000', + ROCKSDB_JS_TEST_SHUTDOWN_DURING_DESTROY: '1', + }); + }, 15_000); + it('releases the path gate when physical destruction fails', async () => { await runDestroyFixture(destroyFailureFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_FAILURE: '1', diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts index c2687e7a7..1a830754d 100644 --- a/test/fixtures/fork-close-failure.mts +++ b/test/fixtures/fork-close-failure.mts @@ -30,3 +30,7 @@ delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined automatic close'); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Shutdown recovery did not preserve the database'); +reopened.destroy(); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index ca3d99eaf..9999a4d72 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -27,12 +27,8 @@ if (closeFailure) { } if (Date.now() - startedAt >= 1_000) throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); - try { - db.destroy(); - throw new Error('Expected repeated destroy to report the previous close failure'); - } catch (error) { - if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; - } + delete process.env.ROCKSDB_JS_CLOSE_FAILURE; + db.destroy(); process.exit(0); } diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index 3bfcfb515..9e15df499 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; import { setTimeout as delay } from 'node:timers/promises'; import { Worker } from 'node:worker_threads'; @@ -34,6 +34,16 @@ while (registryStatus().some((entry) => entry.path === path)) { const destroyResult = nextMessage(); const startedAt = Date.now(); +if (process.env.ROCKSDB_JS_TEST_SHUTDOWN_DURING_DESTROY === '1') { + shutdown(); + const shutdownDuration = Date.now() - startedAt; + const destroyed = await destroyResult; + if (!destroyed.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyed)}`); + if (shutdownDuration < 500) + throw new Error(`Shutdown did not wait for destroy (${shutdownDuration}ms)`); + await worker.terminate(); + process.exit(0); +} const reopened = RocksDatabase.open(path); const openDuration = Date.now() - startedAt; const destroyed = await destroyResult; diff --git a/test/fixtures/fork-gc-close-failure.mts b/test/fixtures/fork-gc-close-failure.mts index a028fe978..cbc73eb36 100644 --- a/test/fixtures/fork-gc-close-failure.mts +++ b/test/fixtures/fork-gc-close-failure.mts @@ -35,3 +35,7 @@ if ( delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Shutdown recovery did not preserve the database'); +reopened.destroy(); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 6dc4f858e..19d868d4e 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -25,3 +25,7 @@ delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined descriptor'); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Shutdown recovery did not preserve the database'); +reopened.destroy(); From a08f35fee55f2fa1dfe61dde32b1110b5225af32 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:45:39 -0600 Subject: [PATCH 08/39] Complete teardown after close status errors --- src/binding/database/db_descriptor.cpp | 20 ++++++++++++++------ src/binding/database/db_descriptor.h | 1 + src/binding/database/db_registry.cpp | 12 +++++++++--- test/checkpoint.test.ts | 13 +++++-------- test/drop.test.ts | 7 ++++++- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index b27f000c6..113fc4010 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -359,9 +359,13 @@ DBDescriptor::DBDescriptor( */ DBDescriptor::~DBDescriptor() { DEBUG_LOG("%p DBDescriptor::~DBDescriptor Closing \"%s\"\n", this, this->path.c_str()); - this->close(); - // Idempotent safety net, matching commitWorker/logWorker's own - // destructor shutdown. + try { + this->close(); + } catch (const std::exception& error) { + DEBUG_LOG("%p DBDescriptor::~DBDescriptor Close failed for \"%s\": %s\n", this, this->path.c_str(), error.what()); + } catch (...) { + DEBUG_LOG("%p DBDescriptor::~DBDescriptor Close failed for \"%s\"\n", this, this->path.c_str()); + } this->parkTimeouts->shutdown(); } @@ -423,9 +427,10 @@ void DBDescriptor::finishClose() { // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an // immediate flush races transaction-log-store teardown (AGENTS invariant 15). + std::string closeError; rocksdb::Status status = this->flush(); if (!status.ok()) { - throw rocksdb_js::DBException("Failed to flush database during close: " + status.ToString()); + closeError = "Failed to flush database during close: " + status.ToString(); } // Trigger manual compaction on all column families to reclaim space from @@ -455,8 +460,8 @@ void DBDescriptor::finishClose() { // trigger a flush rocksdb::WaitForCompactOptions options; status = this->db->WaitForCompact(options); - if (!status.ok()) { - throw rocksdb_js::DBException("Failed waiting for database compaction during close: " + status.ToString()); + if (!status.ok() && closeError.empty()) { + closeError = "Failed waiting for database compaction during close: " + status.ToString(); } std::unique_lock txnsLock(this->txnsMutex); @@ -510,6 +515,9 @@ void DBDescriptor::finishClose() { this->events.releaseAll(); this->db.reset(); + if (!closeError.empty()) { + throw rocksdb_js::DBException(closeError); + } } napi_status DBDescriptor::registerCommitCompletion(napi_env env, napi_threadsafe_function_call_js callJs, bool& closed) { diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 4bf7f2ff2..f13a75dbd 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -439,6 +439,7 @@ struct DBDescriptor final : public std::enable_shared_from_this { void close(); bool isClosing() const { return this->closing.load(); } + bool isClosed() const { return !this->db; } /** * Atomically transitions the descriptor into the closing state. Returns diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index da3ecde4d..eddb57bcb 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -168,7 +168,7 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn // Only erase the entry we claimed. A brand-new descriptor cannot appear // because OpenDB blocks until we notify below. if (eraseIt != instance->databases.end() && eraseIt->second.descriptor == descriptor) { - if (closeError.empty()) { + if (closeError.empty() || descriptor->isClosed()) { instance->databases.erase(eraseIt); } else { eraseIt->second.closeError = closeError; @@ -265,13 +265,15 @@ void DBRegistry::DestroyDB(const std::string& path) { closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = error.what(); - if (!closeError) { + if (!closing.closed && !closeError) { closeError = std::current_exception(); } } catch (...) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = "unknown native close failure"; - if (!closeError) { + if (!closing.closed && !closeError) { closeError = std::current_exception(); } } @@ -644,9 +646,11 @@ void DBRegistry::PurgeAll() { closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = error.what(); if (!closeError) closeError = std::current_exception(); } catch (...) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = "unknown native close failure"; if (!closeError) closeError = std::current_exception(); } @@ -954,11 +958,13 @@ void DBRegistry::Shutdown() { closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = error.what(); if (!closeError) { closeError = std::current_exception(); } } catch (...) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = "unknown native close failure"; if (!closeError) { closeError = std::current_exception(); diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index 6424c34a6..5b8ad48d8 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, shutdown } from '../src/index.ts'; +import { RocksDatabase } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { existsSync, mkdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -26,13 +26,10 @@ async function writeAll(db: RocksDatabase, count: number, prefix = 'value'): Pro describe('Checkpoints', () => { afterEach(() => { - // The close()/destroy()-during-checkpoint tests can leave a descriptor - // pending registry purge (closing before an in-flight, descriptor-pinned - // checkpoint settles defers the purge — see #672), which keeps the source - // database open and its temp dir locked. That fails cleanup on Windows and - // crashes the Bun worker on exit. Purge the registry first so the locks are - // released before we remove the directories. - shutdown(); + // The checkpoint completion path purges a descriptor that became + // unreferenced while the copy was in flight. Avoid process-global shutdown + // here: test files share a process, so it can close another suite's active + // database while that suite is dropping a column family. for (const dir of tempDirs) { rmSync(dir, { force: true, recursive: true, maxRetries: 3, retryDelay: 500 }); } diff --git a/test/drop.test.ts b/test/drop.test.ts index 67d2bba2f..62e64aaec 100644 --- a/test/drop.test.ts +++ b/test/drop.test.ts @@ -213,7 +213,8 @@ describe('Drop', () => { // pre-existing bug tracked as // https://github.com/HarperFast/rocksdb-js/issues/726 (needs the drop // interlocked against in-flight transactions), so this test asserts only - // atomicity and leaves the handles to dbRunner's per-test database.) + // atomicity and verifies that the poisoned environment is reported while + // still completing native teardown.) it('should not partially apply a pessimistic transaction spanning a dropped column family', () => dbRunner( { @@ -235,6 +236,10 @@ describe('Drop', () => { // the live half must NOT have been applied expect(victim.getSync('live')).toBeUndefined(); + + stale.close(); + doomed.close(); + expect(() => victim.close()).toThrow('Failed to flush database during close'); } )); From 33b24e3353a66f286f272c2b1855611cc19c3ff7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:52:48 -0600 Subject: [PATCH 09/39] Make cross-env lifecycle teardown safe --- README.md | 8 ++++++-- src/binding/database/db_descriptor.cpp | 4 ++-- src/binding/database/db_handle.cpp | 16 ++++++++++------ src/binding/database/db_handle.h | 2 ++ src/database.ts | 5 +++-- test/fixtures/fork-destroy-open.mts | 1 + test/lib/util.ts | 16 +++++++++++++++- 7 files changed, 39 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 426065f26..0abe97664 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,9 @@ Creates a new database instance. ### `db.close()` Closes a database. This function can be called multiple times and will only close an opened -database. A database instance can be reopened once its closed. +database. A database instance can be reopened once it is closed. A flush or compaction failure is +reported as an exception after native teardown completes. A failure that prevents teardown emits +`database:closeFailed`; same-path opens then fail until `destroy()` or `shutdown()` retries cleanup. ```typescript const db = RocksDatabase.open('foo'); @@ -1918,7 +1920,9 @@ console.log(registryStatus()); The `shutdown()` will flush all in-memory data to disk and wait for any outstanding compactions to finish, for all open databases. It is highly recommended to call this in a `process` `exit` event -listener (on the main thread), to ensure that all data is flushed to disk before the process exits: +listener (on the main thread), to ensure that all data is flushed to disk before the process exits. +It throws the first close failure after attempting every claimed database; call it again to retry +any descriptor whose teardown did not complete: ```typescript import { shutdown } from '@harperfast/rocksdb-js'; diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 113fc4010..424b4dff2 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -403,6 +403,8 @@ void DBDescriptor::finishClose() { this->logWorker.shutdown(); this->commitWorker.shutdown(); + // An in-flight commit pins this descriptor through its transaction and DB + // handles, so reaching this release pass means only idle per-env TSFNs remain. { std::lock_guard lock(this->commitMutex); for (auto& [env, completion] : this->commitCompletions) { @@ -416,8 +418,6 @@ void DBDescriptor::finishClose() { this->closeWorkersStopped = true; } - // Inject after the one-shot pipeline shutdown so retry coverage exercises - // a genuinely partially-completed close rather than an untouched descriptor. if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { throw rocksdb_js::DBException("Injected database close failure"); } diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 912839911..60ab1ba29 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -67,7 +67,7 @@ void setTxnlogSummaryStatsOnObject( * Creates a new DBHandle. */ DBHandle::DBHandle(napi_env env, napi_ref exportsRef) - : descriptor(nullptr), env(env), exportsRef(exportsRef) {} + : descriptor(nullptr), env(env), ownerThreadId(std::this_thread::get_id()), exportsRef(exportsRef) {} /** * Close the DBHandle and destroy it. @@ -155,12 +155,16 @@ void DBHandle::close() { this->descriptor.reset(); } - // clean up transaction log references - for (auto& [name, ref] : this->logRefs) { - DEBUG_LOG("%p DBHandle::close Releasing transaction log JS reference \"%s\"\n", this, name.c_str()); - ::napi_delete_reference(this->env, ref); + // N-API references are environment-thread-affine. Destroying a shared + // descriptor can close this handle from another worker; retain the refs in + // that case so the owning environment's later close/finalizer releases them. + if (std::this_thread::get_id() == this->ownerThreadId) { + for (auto& [name, ref] : this->logRefs) { + DEBUG_LOG("%p DBHandle::close Releasing transaction log JS reference \"%s\"\n", this, name.c_str()); + ::napi_delete_reference(this->env, ref); + } + this->logRefs.clear(); } - this->logRefs.clear(); DEBUG_LOG("%p DBHandle::close Handle closed\n", this); } diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 1dffc7af2..625f51e77 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "rocksdb/db.h" #include "database/db_descriptor.h" @@ -64,6 +65,7 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro * The node environment. */ napi_env env; + std::thread::id ownerThreadId; /** * A reference to the main `rocksdb_js` exports object. This is needed to diff --git a/src/database.ts b/src/database.ts index f534adf2a..da6f352b5 100644 --- a/src/database.ts +++ b/src/database.ts @@ -348,8 +348,9 @@ export class RocksDatabase extends DBI { * * Listeners are not tied to any specific database — they fire for every * matching event emitted in this process. Native lifecycle failures use - * `'database:closeFailed'` with `(path, error)` string arguments; the path - * remains quarantined until `destroy()` or `shutdown()` retries cleanup. + * `'database:closeFailed'` with `(path, error)` string arguments. A failure + * that prevents teardown quarantines the path until `destroy()` or + * `shutdown()` retries cleanup. * * @example * ```typescript diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index 9e15df499..ee1145cfc 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -6,6 +6,7 @@ import { Worker } from 'node:worker_threads'; const path = process.argv[2]; const original = RocksDatabase.open(path); original.putSync('before-destroy', 'present'); +original.useLog('cross-env-close'); const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { eval: true, diff --git a/test/lib/util.ts b/test/lib/util.ts index 828f2a7e8..898407340 100644 --- a/test/lib/util.ts +++ b/test/lib/util.ts @@ -65,6 +65,10 @@ export async function dbRunner(options: TestOptions | TestFn, test?: TestFn): Pr const dbPath = generateDBPath(); const dbPaths = new Set([dbPath]); const databases: TestDB[] = []; + let testError: unknown; + let closeError: unknown; + let testFailed = false; + let closeFailed = false; try { for (let i = 0; i < testFn.length; i++) { @@ -78,9 +82,17 @@ export async function dbRunner(options: TestOptions | TestFn, test?: TestFn): Pr } await testFn(...databases); + } catch (error) { + testFailed = true; + testError = error; } finally { for (const { db } of databases.reverse()) { - db?.close(); + try { + db?.close(); + } catch (error) { + if (!closeFailed) closeError = error; + closeFailed = true; + } } if (globalThis.gc) { @@ -103,6 +115,8 @@ export async function dbRunner(options: TestOptions | TestFn, test?: TestFn): Pr } } } + if (testFailed) throw testError; + if (closeFailed) throw closeError; } /** From 4d250b93a6033baacac11344e88158e4a6b26538 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:11:05 -0600 Subject: [PATCH 10/39] Close remaining lifecycle race windows --- README.md | 6 +- src/binding/database/database.cpp | 9 +- src/binding/database/db_descriptor.cpp | 7 + src/binding/database/db_registry.cpp | 204 +++++++++++++++---------- src/binding/database/db_registry.h | 9 +- src/database.ts | 6 +- src/load-binding.ts | 4 +- test/checkpoint.test.ts | 6 +- test/destroy.test.ts | 29 +++- test/drop.test.ts | 4 +- test/fixtures/fork-shutdown-retry.mts | 36 +++++ test/workers/shutdown-retry-worker.mts | 11 ++ 12 files changed, 234 insertions(+), 97 deletions(-) create mode 100644 test/fixtures/fork-shutdown-retry.mts create mode 100644 test/workers/shutdown-retry-worker.mts diff --git a/README.md b/README.md index 0abe97664..d5cdc60f8 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,10 @@ Creates a new database instance. Closes a database. This function can be called multiple times and will only close an opened database. A database instance can be reopened once it is closed. A flush or compaction failure is -reported as an exception after native teardown completes. A failure that prevents teardown emits -`database:closeFailed`; same-path opens then fail until `destroy()` or `shutdown()` retries cleanup. +reported as an exception after native teardown completes. All native close errors emit +`database:closeFailed`; when teardown does not complete, same-path opens also fail until +`destroy()` or `shutdown()` retries cleanup. The quarantine applies to both writable and read-only +opens because both modes share the physical path lifecycle. ```typescript const db = RocksDatabase.open('foo'); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index da1bc7d51..e3ca632b2 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -190,9 +190,12 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { if (*dbHandle) { DEBUG_LOG("%p Database::Close Closing database: \"%s\"\n", dbHandle->get(), (*dbHandle)->path.c_str()); - std::string closeError = DBRegistry::CloseDB(*dbHandle); - if (!closeError.empty()) { - std::string message = closeError + ". Call destroy() or shutdown() to retry cleanup"; + CloseResult closeResult = DBRegistry::CloseDB(*dbHandle); + if (!closeResult.error.empty()) { + std::string message = closeResult.error; + if (closeResult.quarantined) { + message += ". Call destroy() or shutdown() to retry cleanup"; + } ::napi_throw_error(env, nullptr, message.c_str()); return nullptr; } diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 424b4dff2..94f45037b 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -387,6 +387,7 @@ void DBDescriptor::finishClose() { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); + const bool retryingClose = this->closeWorkersStopped; if (!this->closeWorkersStopped) { // Wait for all in-flight operations to complete before cleanup. // The closing flag is already set, so new operations will fail with "Database is closing". @@ -417,6 +418,12 @@ void DBDescriptor::finishClose() { } this->closeWorkersStopped = true; } + if (retryingClose) { + const int retryDelayMs = testDelayMs("ROCKSDB_JS_CLOSE_RETRY_DELAY_MS"); + if (retryDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(retryDelayMs)); + } + } if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { throw rocksdb_js::DBException("Injected database close failure"); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index eddb57bcb..feefa8afe 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -65,7 +65,7 @@ std::unique_ptr DBRegistry::instance; /** * Close a RocksDB database handle. */ -std::string DBRegistry::CloseDB(const std::shared_ptr handle) { +CloseResult DBRegistry::CloseDB(const std::shared_ptr handle) { if (!instance) { DEBUG_LOG("%p DBRegistry::CloseDB Registry not initialized\n", instance.get()); return {}; @@ -127,15 +127,16 @@ std::string DBRegistry::CloseDB(const std::shared_ptr handle) { * the duration of finishClose(), so a concurrent OpenDB keeps waiting on * the condition rather than re-opening the path mid-close. */ -std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { +CloseResult DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (!instance) { - return {}; + return CloseResult{}; } DBKey key{path, readOnly}; std::shared_ptr descriptor; std::shared_ptr condition; std::string closeError; + bool quarantined = false; { std::lock_guard lock(instance->databasesMutex); auto entryIterator = instance->databases.find(key); @@ -172,6 +173,7 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn instance->databases.erase(eraseIt); } else { eraseIt->second.closeError = closeError; + quarantined = true; DEBUG_LOG("%p DBRegistry::PurgeIfUnreferenced Quarantined \"%s\": %s\n", instance.get(), path.c_str(), closeError.c_str()); } @@ -185,7 +187,7 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (!closeError.empty() && GlobalEvents::hasListeners()) { emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, closeError})); } - return closeError; + return CloseResult{closeError, quarantined}; } /** @@ -300,19 +302,6 @@ void DBRegistry::DestroyDB(const std::string& path) { if (closeError) { std::rethrow_exception(closeError); } - for (const auto& closing : claimed) { - if (!closing.closed) { - continue; - } - const size_t refCountAfterClose = closing.descriptor.use_count(); - if (refCountAfterClose > 1) { - std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + - " reference(s) still held after closing all handles. This may indicate handles not properly closed or JavaScript objects not yet garbage collected."; - DEBUG_LOG("%p DBRegistry::DestroyDB Error: %s\n", instance.get(), errorMsg.c_str()); - throw rocksdb_js::DBException(errorMsg); - } - } - if (alreadyClosing.empty()) { break; } @@ -351,11 +340,21 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); if (!status.ok()) { - throw rocksdb_js::DBException(status.ToString()); + const std::string error = status.ToString(); + std::lock_guard lock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = error; + throw rocksdb_js::DBException(error); } // remove the database directory including transaction logs - std::filesystem::remove_all(path); + std::error_code cleanupError; + std::filesystem::remove_all(path, cleanupError); + if (cleanupError) { + const std::string error = cleanupError.message(); + std::lock_guard lock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = error; + throw rocksdb_js::DBException("Failed to remove database directory: " + error); + } DEBUG_LOG("%p DBRegistry::DestroyDB Successfully destroyed database at \"%s\"\n", instance.get(), path.c_str()); } @@ -415,13 +414,37 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons for (bool readOnly : {false, true}) { auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); - if (registeredEntry != instance->databases.end() && !registeredEntry->second.closeError.empty()) { + if (registeredEntry != instance->databases.end() && + !registeredEntry->second.closeError.empty() && !registeredEntry->second.closeRetrying + ) { + const bool destroyCleanupFailed = !registeredEntry->second.descriptor; throw rocksdb_js::DBException( - "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry->second.closeError + ". Call destroy() or shutdown() to retry cleanup" + "Cannot open database \"" + path + "\": previous " + + (destroyCleanupFailed ? "destroy cleanup" : "close") + " failed: " + + registeredEntry->second.closeError + + (destroyCleanupFailed ? ". Call destroy() to retry cleanup" : ". Call destroy() or shutdown() to retry cleanup") ); } } + std::shared_ptr retryCondition; + bool retryReadOnly = false; + for (bool readOnly : {false, true}) { + auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); + if (registeredEntry != instance->databases.end() && registeredEntry->second.closeRetrying) { + retryCondition = registeredEntry->second.condition; + retryReadOnly = readOnly; + break; + } + } + if (retryCondition) { + if (!retryCondition->wait_until(lock, deadline, [&]() { + auto registeredEntry = instance->databases.find(DBKey{path, retryReadOnly}); + return registeredEntry == instance->databases.end() || !registeredEntry->second.closeRetrying; + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": close retry is still in progress"); + } + continue; + } entryIterator = instance->databases.find(key); if (entryIterator == instance->databases.end()) { @@ -919,73 +942,94 @@ void DBRegistry::Shutdown() { if (!shutdownLock.try_lock_until(deadline)) { throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); } - std::vector descriptorsToClose; - - { - std::unique_lock lock(instance->databasesMutex); - if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { - return instance->destroyingPaths.empty(); - })) { - throw rocksdb_js::DBException("Timed out waiting for database destruction to finish before shutdown"); + while (true) { + std::vector descriptorsToClose; + std::vector descriptorsToWaitFor; + bool destroysInFlight; + { + std::unique_lock lock(instance->databasesMutex); + destroysInFlight = !instance->destroyingPaths.empty(); + DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); + descriptorsToClose.reserve(instance->databases.size()); + descriptorsToWaitFor.reserve(instance->databases.size()); + + for (auto& [key, entry] : instance->databases) { + if (instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end()) { + continue; + } + if (!entry.descriptor) { + if (!entry.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot complete shutdown: database \"" + key.path + + "\" requires destroy() cleanup: " + entry.closeError + ); + } + continue; + } + ClosingDescriptor closing{key, entry.descriptor, entry.condition}; + if (!entry.closeError.empty() && !entry.closeRetrying) { + entry.closeRetrying = true; + descriptorsToClose.push_back(std::move(closing)); + } else if (entry.closeError.empty() && entry.descriptor->beginClose()) { + entry.closeRetrying = true; + descriptorsToClose.push_back(std::move(closing)); + } else { + descriptorsToWaitFor.push_back(std::move(closing)); + } + } } - DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); - descriptorsToClose.reserve(instance->databases.size()); - // Claim each close while holding the registry lock so DestroyDB can - // safely wait on the matching erase-and-notify below. - for (auto& [key, entry] : instance->databases) { - if (!entry.descriptor || - instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end() - ) { - continue; + std::exception_ptr closeError; + for (auto& closing : descriptorsToClose) { + DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = error.what(); + if (!closeError) closeError = std::current_exception(); + } catch (...) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = "unknown native close failure"; + if (!closeError) closeError = std::current_exception(); } - ClosingDescriptor closing{key, entry.descriptor, entry.condition}; - if (!entry.closeError.empty() && !entry.closeRetrying) { - entry.closeRetrying = true; - descriptorsToClose.push_back(std::move(closing)); - } else if (entry.closeError.empty() && entry.descriptor->beginClose()) { - entry.closeRetrying = true; - descriptorsToClose.push_back(std::move(closing)); + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { + instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; + } + } } + closing.condition->notify_all(); } - } - - // Close all descriptors without holding the lock - std::exception_ptr closeError; - for (auto& closing : descriptorsToClose) { - DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); - try { - closing.descriptor->finishClose(); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closeError) { - closeError = std::current_exception(); - } - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closeError) { - closeError = std::current_exception(); + if (closeError) std::rethrow_exception(closeError); + + for (const auto& closing : descriptorsToWaitFor) { + std::unique_lock lock(instance->databasesMutex); + if (!closing.condition->wait_until(lock, deadline, [&]() { + auto entry = instance->databases.find(closing.key); + return entry == instance->databases.end() || + entry->second.descriptor != closing.descriptor || + (!entry->second.closeError.empty() && !entry->second.closeRetrying); + })) { + throw rocksdb_js::DBException("Timed out waiting for database close to finish during shutdown"); } } - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - entry->second.closeRetrying = false; - } + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) { + if (!destroysInFlight) break; + std::unique_lock lock(instance->databasesMutex); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.empty(); + })) { + throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } } - closing.condition->notify_all(); - } - if (closeError) { - std::rethrow_exception(closeError); } // Purge the registry diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index cbddbbfe0..edd01b8de 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -51,6 +51,11 @@ struct DBRegistryEntry final { : descriptor(std::move(desc)), condition(std::make_shared()) {} }; +struct CloseResult final { + std::string error; + bool quarantined = false; +}; + struct DBHandleParams final { std::shared_ptr descriptor; @@ -94,7 +99,7 @@ class DBRegistry final { static std::unique_ptr instance; public: - static std::string CloseDB(const std::shared_ptr handle); + static CloseResult CloseDB(const std::shared_ptr handle); #ifdef DEBUG static void DebugLogDescriptorRefs(); #endif @@ -102,7 +107,7 @@ class DBRegistry final { static void Init(napi_env env, napi_value exports); static std::unique_ptr OpenDB(const std::string& path, const DBOptions& options); static void PurgeAll(); - static std::string PurgeIfUnreferenced(const std::string& path, bool readOnly); + static CloseResult PurgeIfUnreferenced(const std::string& path, bool readOnly); static napi_value RegistryStatus(napi_env env, napi_callback_info info); static void CloseTransactionsByEnv(napi_env env); static void RemoveListenersByEnv(napi_env env); diff --git a/src/database.ts b/src/database.ts index da6f352b5..73a14200f 100644 --- a/src/database.ts +++ b/src/database.ts @@ -348,9 +348,9 @@ export class RocksDatabase extends DBI { * * Listeners are not tied to any specific database — they fire for every * matching event emitted in this process. Native lifecycle failures use - * `'database:closeFailed'` with `(path, error)` string arguments. A failure - * that prevents teardown quarantines the path until `destroy()` or - * `shutdown()` retries cleanup. + * `'database:closeFailed'` with `(path, error)` string arguments. The event + * reports both completed and incomplete teardowns; only an incomplete + * teardown quarantines the path until `destroy()` or `shutdown()` retries. * * @example * ```typescript diff --git a/src/load-binding.ts b/src/load-binding.ts index e08b2b46d..73584f189 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -511,8 +511,8 @@ export type RocksDatabaseConfig = { verificationTableEntries?: number; compactOnClose?: boolean; /** - * Maximum seconds an open or destroy call waits for another lifecycle - * operation on the same path. Defaults to 30. + * Maximum seconds an open, destroy, or shutdown call waits for another + * lifecycle operation. Defaults to 30. */ lifecycleWaitSeconds?: number; /** diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index 5b8ad48d8..69fac4708 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -220,9 +220,9 @@ describe('Checkpoints', () => { // close()) bypasses the async-work tracker. The checkpoint registers in // operationsInFlight, so finishClose() waits for the copy to finish // before resetting descriptor->db — the worker never touches a freed DB. - // The in-flight op still holds a descriptor reference, so destroy() throws - // rather than tearing the database down mid-copy. - expect(() => db.destroy()).toThrow(); + // The operation reference can outlive registry removal safely because + // finishClose waits for it and resets the native DB before physical destroy. + expect(() => db.destroy()).not.toThrow(); // The checkpoint itself completed (finishClose waited for it), so the // promise settles cleanly and nothing crashes. diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 2c558cfe5..1a1a3aa15 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,7 +1,7 @@ import { RocksDatabase } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; -import { existsSync } from 'node:fs'; +import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -10,6 +10,7 @@ const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure. const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); +const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); function runDestroyFixture( fixture: string, @@ -100,6 +101,26 @@ describe('Destroy', () => { } )); + it.skipIf(process.platform === 'win32')( + 'quarantines a path when post-destroy cleanup fails', + () => + dbRunner(({ db, dbPath }) => { + const lockedDirectory = join(dbPath, 'transaction_logs', 'locked'); + mkdirSync(lockedDirectory, { recursive: true }); + writeFileSync(join(lockedDirectory, 'leftover'), 'data'); + chmodSync(lockedDirectory, 0o000); + try { + expect(() => db.destroy()).toThrow('Failed to remove database directory'); + expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); + } finally { + chmodSync(lockedDirectory, 0o700); + } + db.destroy(); + const reopened = RocksDatabase.open(dbPath); + reopened.close(); + }) + ); + it('waits for physical destruction before reopening the same path', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', @@ -142,4 +163,10 @@ describe('Destroy', () => { ROCKSDB_JS_CLOSE_FAILURE: '1', }); }, 15_000); + + it('waits for an in-progress shutdown retry before reopening', async () => { + await runDestroyFixture(shutdownRetryFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); }); diff --git a/test/drop.test.ts b/test/drop.test.ts index 62e64aaec..d88632a47 100644 --- a/test/drop.test.ts +++ b/test/drop.test.ts @@ -224,7 +224,7 @@ describe('Drop', () => { { name: 'doomed', pessimistic: true }, ], }, - async ({ db: victim }, { db: doomed }, { db: stale }) => { + async ({ db: victim, dbPath }, { db: doomed }, { db: stale }) => { await expect( stale.transaction(async (txn: Transaction) => { await victim.put('live', 'A', { transaction: txn }); @@ -240,6 +240,8 @@ describe('Drop', () => { stale.close(); doomed.close(); expect(() => victim.close()).toThrow('Failed to flush database during close'); + const reopened = RocksDatabase.open(dbPath); + reopened.close(); } )); diff --git a/test/fixtures/fork-shutdown-retry.mts b/test/fixtures/fork-shutdown-retry.mts new file mode 100644 index 000000000..109100dbc --- /dev/null +++ b/test/fixtures/fork-shutdown-retry.mts @@ -0,0 +1,36 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); +try { + db.close(); + throw new Error('Expected the initial close to fail'); +} catch (error) { + if (!String(error).includes('Injected database close failure')) throw error; +} + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS = '1000'; +const worker = new Worker(createWorkerBootstrapScript('./test/workers/shutdown-retry-worker.mts'), { + eval: true, +}); +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} +const started = await nextMessage(); +const shutdownResult = nextMessage(); +const reopened = RocksDatabase.open(path); +const elapsed = Date.now() - started; +if (elapsed < 500) throw new Error(`Open did not wait for the shutdown retry (${elapsed}ms)`); +if (reopened.getSync('key') !== 'value') throw new Error('Shutdown retry did not preserve data'); +const result = await shutdownResult; +if (!result.shutdown) throw new Error(`Shutdown retry failed: ${JSON.stringify(result)}`); +delete process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS; +reopened.destroy(); +await worker.terminate(); diff --git a/test/workers/shutdown-retry-worker.mts b/test/workers/shutdown-retry-worker.mts new file mode 100644 index 000000000..98bcda646 --- /dev/null +++ b/test/workers/shutdown-retry-worker.mts @@ -0,0 +1,11 @@ +import { shutdown } from '../../src/index.ts'; +import { parentPort } from 'node:worker_threads'; + +if (!parentPort) throw new Error('Shutdown retry worker requires a parent port'); +parentPort.postMessage(Date.now()); +try { + shutdown(); + parentPort.postMessage({ shutdown: true }); +} catch (error) { + parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) }); +} From 15b3a32968b05e3f769464c680d701ea3145ea5b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:16:23 -0600 Subject: [PATCH 11/39] Update stream destroy lifecycle coverage --- test/backup-stream.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/backup-stream.test.ts b/test/backup-stream.test.ts index 7d3d90f2a..d09ccfcf7 100644 --- a/test/backup-stream.test.ts +++ b/test/backup-stream.test.ts @@ -188,7 +188,7 @@ describe('Streaming backups', () => { await expect(settled).resolves.toBe('settled'); })); - it('throws on destroy() during a stream and the stream still settles', () => + it('waits for an in-flight stream before destroying', () => dbRunner(async ({ db }) => { await writeAll(db, 200); @@ -203,9 +203,9 @@ describe('Streaming backups', () => { ); await new Promise((r) => setTimeout(r, 50)); - // The in-flight stream pins the descriptor, so destroy() refuses to tear - // the database down mid-stream and throws instead. - expect(() => db.destroy()).toThrow(); + // The stream pins the descriptor and registers an in-flight operation, so + // destroy waits for it before closing the native DB and removing the path. + expect(() => db.destroy()).not.toThrow(); await expect(settled).resolves.toBe('settled'); })); From 63c2f5bd193efce109c3363ed45c9aefbaad1282 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:24:46 -0600 Subject: [PATCH 12/39] Close final lifecycle race windows --- README.md | 5 +- src/binding/database/backup.cpp | 39 ++++++++- src/binding/database/db_registry.cpp | 110 ++++++++++++++++++------ src/load-binding.ts | 1 + test/destroy.test.ts | 15 +++- test/fixtures/fork-backup-destroy.mts | 20 +++++ test/fixtures/fork-destroy-failure.mts | 12 +++ test/fixtures/fork-shutdown-failure.mts | 12 +++ 8 files changed, 179 insertions(+), 35 deletions(-) create mode 100644 test/fixtures/fork-backup-destroy.mts diff --git a/README.md b/README.md index d5cdc60f8..7b1ca39e7 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. - - `lifecycleWaitSeconds: number` Maximum time a synchronous open, destroy, or shutdown waits for + - `lifecycleWaitSeconds: number` Total maximum time a synchronous open, destroy, or shutdown waits for another lifecycle operation before throwing a retryable timeout error. Defaults to `30` seconds and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global @@ -1905,6 +1905,9 @@ console.log(currentThreadId()); Returns an array containing that status of all active RocksDB instances. - `path: string` The database path. +- `closeError?: string` The native lifecycle error retaining this registry entry. +- `destroyCleanupPending?: boolean` The native database is closed, but physical path cleanup must + finish before the next open. Opening the path retries cleanup automatically. - `refCount: number` The number of JavaScript database instances plus the registry's reference. - `columnFamiles: object` A map of column family names and their their info. - `userSharedBuffers: number` The count of active user shared buffers. diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 295bb77e0..3b876fc4e 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -5,15 +5,18 @@ #include "database/db_handle.h" #include "database/db_registry.h" #include "core/file_lock.h" +#include "core/test_seam.h" #include "napi/async.h" #include "napi/helpers.h" #include "napi/macros.h" #include "rocksdb/env.h" #include "rocksdb/status.h" #include "rocksdb/utilities/backup_engine.h" +#include #include #include #include +#include #include namespace rocksdb_js { @@ -67,6 +70,17 @@ struct AsyncBackupState final : BaseAsyncState> { } }; +struct BackupInFlightClaim final { + DBDescriptor* descriptor; + const bool& handedOff; + + ~BackupInFlightClaim() { + if (!handedOff && --descriptor->operationsInFlight == 0 && descriptor->isClosing()) { + descriptor->operationsInFlight.notify_all(); + } + } +}; + /** * State for the `backupRestore` async work. There is no open database during a * restore, so the base handle is null. @@ -135,7 +149,8 @@ static napi_value queueBackupWork( State* state, napi_async_execute_callback execute, napi_async_complete_callback complete, - bool registerWork + bool registerWork, + bool* queued = nullptr ) { NAPI_STATUS_THROWS(::napi_create_reference(env, resolve, 1, &state->resolveRef)); NAPI_STATUS_THROWS(::napi_create_reference(env, reject, 1, &state->rejectRef)); @@ -150,6 +165,7 @@ static napi_value queueBackupWork( } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (queued) *queued = true; NAPI_RETURN_UNDEFINED(); } @@ -180,6 +196,10 @@ static rocksdb::Status runCreateBackup(AsyncBackupState* state) { if (!state->descriptor || !state->handle || state->handle->isCancelled()) { return rocksdb::Status::Aborted("Database closed during backup operation"); } + const int backupDelayMs = testDelayMs("ROCKSDB_JS_BACKUP_DELAY_MS"); + if (backupDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(backupDelayMs)); + } const std::string& backupDir = state->engineOptions.backup_dir; @@ -341,10 +361,19 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { bool checkDiskSpace = true; NAPI_STATUS_THROWS(getProperty(env, options, "checkDiskSpace", checkDiskSpace)); + auto descriptor = (*dbHandle)->descriptor; + ++descriptor->operationsInFlight; + bool handedOff = false; + BackupInFlightClaim claim{descriptor.get(), handedOff}; + if (descriptor->isClosing()) { + ::napi_throw_error(env, nullptr, "Database is closing"); + NAPI_RETURN_UNDEFINED(); + } + auto state = new AsyncBackupState( env, *dbHandle, - (*dbHandle)->descriptor, + descriptor, std::move(engineOptions), std::move(createOptions), std::move(appMetadata) @@ -361,6 +390,9 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { [](napi_env, void* data) { // execute auto state = reinterpret_cast(data); state->status = runCreateBackup(state); + if (--state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing()) { + state->descriptor->operationsInFlight.notify_all(); + } state->signalExecuteCompleted(); }, [](napi_env env, napi_status status, void* data) { // complete @@ -379,7 +411,8 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { } delete state; }, - true // registerWork + true, // registerWork + &handedOff ); } diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index feefa8afe..f89b31c08 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -57,6 +57,28 @@ struct ClosingDescriptor final { ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; +std::string destroyPhysicalPath(const std::string& path) { + rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); + if (!status.ok()) return status.ToString(); + + std::error_code cleanupError; + std::filesystem::remove_all(path, cleanupError); + if (cleanupError) return "Failed to remove database directory: " + cleanupError.message(); + return {}; +} + +void emitCloseFailures(const std::vector& descriptors) { + if (!GlobalEvents::hasListeners()) return; + for (const auto& closing : descriptors) { + if (!closing.closeError.empty()) { + emitGlobalEvent( + "database:closeFailed", + ListenerData::fromStrings({closing.key.path, closing.closeError}) + ); + } + } +} + } // namespace // Initialize the static instance @@ -299,10 +321,9 @@ void DBRegistry::DestroyDB(const std::string& path) { for (const auto& closing : claimed) { closing.condition->notify_all(); } - if (closeError) { - std::rethrow_exception(closeError); - } + emitCloseFailures(claimed); if (alreadyClosing.empty()) { + if (closeError) std::rethrow_exception(closeError); break; } for (const auto& closing : alreadyClosing) { @@ -316,6 +337,7 @@ void DBRegistry::DestroyDB(const std::string& path) { throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); } } + if (closeError) std::rethrow_exception(closeError); } { @@ -338,22 +360,11 @@ void DBRegistry::DestroyDB(const std::string& path) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); - rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); - if (!status.ok()) { - const std::string error = status.ToString(); + const std::string destroyError = destroyPhysicalPath(path); + if (!destroyError.empty()) { std::lock_guard lock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = error; - throw rocksdb_js::DBException(error); - } - - // remove the database directory including transaction logs - std::error_code cleanupError; - std::filesystem::remove_all(path, cleanupError); - if (cleanupError) { - const std::string error = cleanupError.message(); - std::lock_guard lock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = error; - throw rocksdb_js::DBException("Failed to remove database directory: " + error); + instance->databases[DBKey{path, false}].closeError = destroyError; + throw rocksdb_js::DBException(destroyError); } DEBUG_LOG("%p DBRegistry::DestroyDB Successfully destroyed database at \"%s\"\n", instance.get(), path.c_str()); @@ -412,6 +423,32 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } + auto failedDestroy = instance->databases.find(DBKey{path, false}); + if (failedDestroy != instance->databases.end() && + !failedDestroy->second.descriptor && !failedDestroy->second.closeError.empty() + ) { + instance->databases.erase(failedDestroy); + instance->destroyingPaths.insert(path); + lock.unlock(); + std::string destroyError; + { + DestroyPathGuard pathGuard( + instance->databasesMutex, + instance->lifecycleCondition, + instance->destroyingPaths, + path + ); + destroyError = destroyPhysicalPath(path); + if (!destroyError.empty()) { + std::lock_guard cleanupLock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = destroyError; + } + } + if (!destroyError.empty()) throw rocksdb_js::DBException(destroyError); + lock.lock(); + continue; + } + for (bool readOnly : {false, true}) { auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); if (registeredEntry != instance->databases.end() && @@ -690,6 +727,7 @@ void DBRegistry::PurgeAll() { } closing.condition->notify_all(); } + emitCloseFailures(descriptorsToClose); if (closeError) { std::rethrow_exception(closeError); } @@ -713,9 +751,6 @@ napi_value DBRegistry::RegistryStatus(napi_env env, napi_callback_info info) { size_t i = 0; for (auto& [key, entry] : instance->databases) { - if (!entry.descriptor) { - continue; - } napi_value database; NAPI_STATUS_THROWS(::napi_create_object(env, &database)); napi_value pathValue; @@ -731,6 +766,21 @@ napi_value DBRegistry::RegistryStatus(napi_env env, napi_callback_info info) { )); NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "closeError", closeErrorValue)); } + if (!entry.descriptor) { + napi_value pending; + NAPI_STATUS_THROWS(::napi_get_boolean(env, true, &pending)); + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "destroyCleanupPending", pending)); + napi_value zero; + NAPI_STATUS_THROWS(::napi_create_uint32(env, 0, &zero)); + for (const char* property : {"refCount", "transactions", "closables", "locks", "listenerCallbacks"}) { + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, property, zero)); + } + napi_value columnFamilies; + NAPI_STATUS_THROWS(::napi_create_object(env, &columnFamilies)); + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "columnFamilies", columnFamilies)); + NAPI_STATUS_THROWS(::napi_set_element(env, result, i++, database)); + continue; + } napi_value modeValue; std::string mode = entry.descriptor->mode == DBMode::Optimistic ? "optimistic" : "pessimistic"; NAPI_STATUS_THROWS(::napi_create_string_utf8(env, mode.c_str(), mode.size(), &modeValue)); @@ -945,6 +995,7 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; + std::string destroyCleanupError; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -958,11 +1009,10 @@ void DBRegistry::Shutdown() { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty()) { - throw rocksdb_js::DBException( + if (!entry.closeError.empty() && destroyCleanupError.empty()) { + destroyCleanupError = "Cannot complete shutdown: database \"" + key.path + - "\" requires destroy() cleanup: " + entry.closeError - ); + "\" requires destroy cleanup: " + entry.closeError; } continue; } @@ -1008,7 +1058,7 @@ void DBRegistry::Shutdown() { } closing.condition->notify_all(); } - if (closeError) std::rethrow_exception(closeError); + emitCloseFailures(descriptorsToClose); for (const auto& closing : descriptorsToWaitFor) { std::unique_lock lock(instance->databasesMutex); @@ -1021,8 +1071,7 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database close to finish during shutdown"); } } - if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) { - if (!destroysInFlight) break; + if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { return instance->destroyingPaths.empty(); @@ -1030,6 +1079,11 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } } + if (closeError) std::rethrow_exception(closeError); + if (!destroyCleanupError.empty()) { + throw rocksdb_js::DBException(destroyCleanupError); + } + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) break; } // Purge the registry diff --git a/src/load-binding.ts b/src/load-binding.ts index 73584f189..2a3abd5e9 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -634,6 +634,7 @@ export type RegistryStatusTransaction = { export type RegistryStatusDB = { path: string; closeError?: string; + destroyCleanupPending?: boolean; refCount: number; columnFamilies: string[]; transactions: number; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 1a1a3aa15..cb603c82a 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -11,6 +11,7 @@ const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts' const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); +const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); function runDestroyFixture( fixture: string, @@ -75,6 +76,12 @@ describe('Destroy', () => { expect(db.isOpen()).toBe(false); })); + it('waits for an in-flight directory backup before destroying', async () => { + await runDestroyFixture(backupDestroyFixture, generateDBPath(), { + ROCKSDB_JS_BACKUP_DELAY_MS: '500', + }); + }); + it('should destroy all related instances', () => dbRunner( { dbOptions: [{}, { name: 'test' }, { readOnly: true }] }, @@ -111,11 +118,13 @@ describe('Destroy', () => { chmodSync(lockedDirectory, 0o000); try { expect(() => db.destroy()).toThrow('Failed to remove database directory'); - expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); + expect( + registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending + ).toBe(true); + expect(() => RocksDatabase.open(dbPath)).toThrow('Failed to remove database directory'); } finally { chmodSync(lockedDirectory, 0o700); } - db.destroy(); const reopened = RocksDatabase.open(dbPath); reopened.close(); }) diff --git a/test/fixtures/fork-backup-destroy.mts b/test/fixtures/fork-backup-destroy.mts new file mode 100644 index 000000000..27861cb78 --- /dev/null +++ b/test/fixtures/fork-backup-destroy.mts @@ -0,0 +1,20 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { rmSync } from 'node:fs'; + +const path = process.argv[2]; +const backupPath = `${path}-backup`; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); + +try { + const backup = db.backup(backupPath); + const startedAt = Date.now(); + db.destroy(); + const destroyDuration = Date.now() - startedAt; + await backup; + if (destroyDuration < 400) { + throw new Error(`Destroy did not wait for the directory backup (${destroyDuration}ms)`); + } +} finally { + rmSync(backupPath, { force: true, recursive: true }); +} diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 9999a4d72..a090d2dcc 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -4,6 +4,9 @@ const path = process.argv[2]; const closeFailure = process.env.ROCKSDB_JS_CLOSE_FAILURE === '1'; const db = RocksDatabase.open(path); db.putSync('key', 'value'); +let resolveCloseFailure: (args: unknown[]) => void; +const closeFailureEvent = new Promise((resolve) => (resolveCloseFailure = resolve)); +RocksDatabase.on('database:closeFailed', (...args) => resolveCloseFailure(args)); let destroyError: unknown; try { @@ -18,6 +21,15 @@ if (!String(destroyError).includes(expectedError)) throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); if (closeFailure) { + const args = await Promise.race([ + closeFailureEvent, + new Promise((_, reject) => + setTimeout(() => reject(new Error('Destroy did not emit database:closeFailed')), 1_000) + ), + ]); + if (args[0] !== path || args[1] !== expectedError) { + throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); + } const startedAt = Date.now(); try { RocksDatabase.open(path); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 19d868d4e..a834c5671 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -3,6 +3,9 @@ import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; const path = process.argv[2]; const db = RocksDatabase.open(path); db.putSync('key', 'value'); +let resolveCloseFailure: (args: unknown[]) => void; +const closeFailureEvent = new Promise((resolve) => (resolveCloseFailure = resolve)); +RocksDatabase.on('database:closeFailed', (...args) => resolveCloseFailure(args)); try { shutdown(); @@ -10,6 +13,15 @@ try { } catch (error) { if (!String(error).includes('Injected database close failure')) throw error; } +const args = await Promise.race([ + closeFailureEvent, + new Promise((_, reject) => + setTimeout(() => reject(new Error('Shutdown did not emit database:closeFailed')), 1_000) + ), +]); +if (args[0] !== path || args[1] !== 'Injected database close failure') { + throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); +} const startedAt = Date.now(); try { RocksDatabase.open(path, { readOnly: true }); From fe45f683b6de8141566caf67532ce697f0dca467 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:29:28 -0600 Subject: [PATCH 13/39] Gate opens across process shutdown --- src/binding/database/db_descriptor.cpp | 5 ++- src/binding/database/db_registry.cpp | 46 ++++++++++++++++++++++++-- src/binding/database/db_registry.h | 1 + src/binding/database/db_settings.cpp | 9 +++++ test/destroy.test.ts | 1 + 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 94f45037b..5058f99f4 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -406,6 +406,9 @@ void DBDescriptor::finishClose() { // An in-flight commit pins this descriptor through its transaction and DB // handles, so reaching this release pass means only idle per-env TSFNs remain. + // Release rather than abort: queued completions are still delivered before + // finalization, while a later registration observes commitCompletionsClosed + // and falls back to the legacy libuv path. { std::lock_guard lock(this->commitMutex); for (auto& [env, completion] : this->commitCompletions) { @@ -509,8 +512,8 @@ void DBDescriptor::finishClose() { // Unregister from transaction log store registry - this will clean up stores // when the last descriptor for this path is closed if (!this->transactionLogsUnregistered) { - this->transactionLogsUnregistered = true; TransactionLogStoreRegistry::Unregister(this->path); + this->transactionLogsUnregistered = true; } this->transactions.clear(); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index f89b31c08..19f38ed62 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -43,6 +43,28 @@ class DestroyPathGuard final { const std::string& path; }; +class ShutdownGuard final { +public: + ShutdownGuard( + std::mutex& mutex, + std::condition_variable& condition, + bool& shutdownInProgress + ) : mutex(mutex), condition(condition), shutdownInProgress(shutdownInProgress) {} + + ~ShutdownGuard() { + { + std::lock_guard lock(this->mutex); + this->shutdownInProgress = false; + } + this->condition.notify_all(); + } + +private: + std::mutex& mutex; + std::condition_variable& condition; + bool& shutdownInProgress; +}; + struct ClosingDescriptor final { DBKey key; std::shared_ptr descriptor; @@ -242,9 +264,10 @@ void DBRegistry::DestroyDB(const std::string& path) { { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { - return instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); + return !instance->shutdownInProgress && + instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); })) { - throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": another destroy is still in progress"); + throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": another lifecycle operation is still in progress"); } instance->destroyingPaths.insert(path); } @@ -351,7 +374,7 @@ void DBRegistry::DestroyDB(const std::string& path) { } } - // Now the database lock should be released, safe to destroy + // All in-process descriptors are closed; physical destruction can proceed. const int destroyDelayMs = testDelayMs("ROCKSDB_JS_DESTROY_DELAY_MS"); if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); @@ -411,6 +434,14 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons DBKey key{path, options.readOnly}; decltype(instance->databases)::iterator entryIterator; while (true) { + if (instance->shutdownInProgress) { + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return !instance->shutdownInProgress; + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": shutdown is still in progress"); + } + continue; + } if (!instance->destroyingPaths.empty() && instance->destroyingPaths.find(path) != instance->destroyingPaths.end() ) { @@ -992,6 +1023,15 @@ void DBRegistry::Shutdown() { if (!shutdownLock.try_lock_until(deadline)) { throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); } + { + std::lock_guard lock(instance->databasesMutex); + instance->shutdownInProgress = true; + } + ShutdownGuard shutdownGuard( + instance->databasesMutex, + instance->lifecycleCondition, + instance->shutdownInProgress + ); while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index edd01b8de..7091a5ee6 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -87,6 +87,7 @@ class DBRegistry final { */ std::mutex databasesMutex; std::timed_mutex shutdownMutex; + bool shutdownInProgress = false; // Destruction owns a physical path across every (path, readOnly) entry. // Waiters must re-resolve databases after every wake because the closer can // erase the node while the mutex is released. diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index 77a5ee27b..b0cfe00fa 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -219,6 +219,15 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { double lifecycleWaitSeconds = 0; bool lifecycleWaitProvided = false; NAPI_STATUS_THROWS(::napi_has_named_property(env, params, "lifecycleWaitSeconds", &lifecycleWaitProvided)); + if (lifecycleWaitProvided) { + napi_value lifecycleWaitValue; + NAPI_STATUS_THROWS(::napi_get_named_property(env, params, "lifecycleWaitSeconds", &lifecycleWaitValue)); + napi_valuetype lifecycleWaitType; + NAPI_STATUS_THROWS(::napi_typeof(env, lifecycleWaitValue, &lifecycleWaitType)); + if (lifecycleWaitType == napi_undefined || lifecycleWaitType == napi_null) { + lifecycleWaitProvided = false; + } + } if (lifecycleWaitProvided) { status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); if (status != napi_ok) { diff --git a/test/destroy.test.ts b/test/destroy.test.ts index cb603c82a..4cf90974f 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -53,6 +53,7 @@ describe('Destroy', () => { expect(() => RocksDatabase.config({ lifecycleWaitSeconds: '30' as unknown as number })).toThrow( 'Lifecycle wait seconds must be a number' ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: undefined })).not.toThrow(); expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); }); From dad49e398c3814b28e1ce55dc76c4ca83751782e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:32:00 -0600 Subject: [PATCH 14/39] Rescan lifecycle state after destroy --- src/binding/database/db_registry.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 19f38ed62..e3ba02bec 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -1118,6 +1118,11 @@ void DBRegistry::Shutdown() { })) { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } + if (closeError) std::rethrow_exception(closeError); + if (!destroyCleanupError.empty()) { + throw rocksdb_js::DBException(destroyCleanupError); + } + continue; } if (closeError) std::rethrow_exception(closeError); if (!destroyCleanupError.empty()) { From 90d5c3a82b51f40d3f3a799ac91e1597361d6b74 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:37:16 -0600 Subject: [PATCH 15/39] Keep destroy recovery explicit --- README.md | 2 +- src/binding/database/database.cpp | 15 ++++++- src/binding/database/db_registry.cpp | 60 +++++++++++++--------------- src/database.ts | 2 +- src/load-binding.ts | 2 +- test/destroy.test.ts | 19 ++++++++- 6 files changed, 61 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 7b1ca39e7..576f8655c 100644 --- a/README.md +++ b/README.md @@ -1907,7 +1907,7 @@ Returns an array containing that status of all active RocksDB instances. - `path: string` The database path. - `closeError?: string` The native lifecycle error retaining this registry entry. - `destroyCleanupPending?: boolean` The native database is closed, but physical path cleanup must - finish before the next open. Opening the path retries cleanup automatically. + finish before the next open. Call `destroy()` or `shutdown()` to retry cleanup. - `refCount: number` The number of JavaScript database instances plus the registry's reference. - `columnFamiles: object` A map of column family names and their their info. - `userSharedBuffers: number` The count of active user shared buffers. diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index e3ca632b2..13384f7db 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -449,8 +449,21 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { THROW_IF_READONLY((*dbHandle)->descriptor, "Destroy failed: "); if (*dbHandle) { + std::string path = (*dbHandle)->path; + napi_valuetype pathType; + NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &pathType)); + if (pathType == napi_string) { + NAPI_STATUS_THROWS_ERROR(rocksdb_js::getString(env, argv[0], path), "Database path must be a string"); + } else if (pathType != napi_undefined) { + ::napi_throw_type_error(env, nullptr, "Database path must be a string"); + return nullptr; + } + if (path.empty()) { + ::napi_throw_error(env, nullptr, "Database path is required for destroy"); + return nullptr; + } try { - DBRegistry::DestroyDB((*dbHandle)->path); + DBRegistry::DestroyDB(path); } catch (const std::exception& e) { DEBUG_LOG("%p Database::Destroy Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index e3ba02bec..56f0a0235 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -454,32 +454,6 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } - auto failedDestroy = instance->databases.find(DBKey{path, false}); - if (failedDestroy != instance->databases.end() && - !failedDestroy->second.descriptor && !failedDestroy->second.closeError.empty() - ) { - instance->databases.erase(failedDestroy); - instance->destroyingPaths.insert(path); - lock.unlock(); - std::string destroyError; - { - DestroyPathGuard pathGuard( - instance->databasesMutex, - instance->lifecycleCondition, - instance->destroyingPaths, - path - ); - destroyError = destroyPhysicalPath(path); - if (!destroyError.empty()) { - std::lock_guard cleanupLock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = destroyError; - } - } - if (!destroyError.empty()) throw rocksdb_js::DBException(destroyError); - lock.lock(); - continue; - } - for (bool readOnly : {false, true}) { auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); if (registeredEntry != instance->databases.end() && @@ -1035,7 +1009,7 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; - std::string destroyCleanupError; + std::vector destroyCleanupEntries; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -1043,17 +1017,14 @@ void DBRegistry::Shutdown() { DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); descriptorsToClose.reserve(instance->databases.size()); descriptorsToWaitFor.reserve(instance->databases.size()); + destroyCleanupEntries.reserve(instance->databases.size()); for (auto& [key, entry] : instance->databases) { if (instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end()) { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty() && destroyCleanupError.empty()) { - destroyCleanupError = - "Cannot complete shutdown: database \"" + key.path + - "\" requires destroy cleanup: " + entry.closeError; - } + if (!entry.closeError.empty()) destroyCleanupEntries.push_back(key); continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; @@ -1111,6 +1082,27 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database close to finish during shutdown"); } } + + std::string destroyCleanupError; + for (const auto& key : destroyCleanupEntries) { + const std::string cleanupError = destroyPhysicalPath(key.path); + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(key); + if (entry != instance->databases.end() && !entry->second.descriptor) { + if (cleanupError.empty()) { + instance->databases.erase(entry); + } else { + entry->second.closeError = cleanupError; + } + } + } + if (!cleanupError.empty() && destroyCleanupError.empty()) { + destroyCleanupError = + "Cannot complete shutdown: database \"" + key.path + + "\" requires destroy cleanup: " + cleanupError; + } + } if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { @@ -1128,7 +1120,9 @@ void DBRegistry::Shutdown() { if (!destroyCleanupError.empty()) { throw rocksdb_js::DBException(destroyCleanupError); } - if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) break; + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty() && + destroyCleanupEntries.empty() + ) break; } // Purge the registry diff --git a/src/database.ts b/src/database.ts index 73a14200f..4b34d0819 100644 --- a/src/database.ts +++ b/src/database.ts @@ -408,7 +408,7 @@ export class RocksDatabase extends DBI { // committed destroy(): void { - this.store.db.destroy(); + this.store.db.destroy(this.store.path); } async drop(): Promise { diff --git a/src/load-binding.ts b/src/load-binding.ts index 2a3abd5e9..cdc8cf3da 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -428,7 +428,7 @@ export type NativeDatabase = { reject: RejectCallback, targetPath: string ): void; - destroy(): void; + destroy(path?: string): void; drop(resolve: ResolveCallback, reject: RejectCallback): void; dropSync(): void; flush(resolve: ResolveCallback, reject: RejectCallback, options?: FlushOptions): void; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 4cf90974f..d4f4f976f 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus } from '../src/index.ts'; +import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -68,6 +68,13 @@ describe('Destroy', () => { expect(db.isOpen()).toBe(false); })); + it('should destroy a database from an unopened handle', () => + dbRunner(({ db, dbPath }) => { + db.close(); + new RocksDatabase(dbPath).destroy(); + expect(existsSync(dbPath)).toBe(false); + })); + it('should destroy an open database', () => dbRunner(({ db, dbPath }) => { db.putSync('key', 'value'); @@ -113,6 +120,9 @@ describe('Destroy', () => { 'quarantines a path when post-destroy cleanup fails', () => dbRunner(({ db, dbPath }) => { + const healthyPath = generateDBPath(); + const healthy = RocksDatabase.open(healthyPath); + healthy.putSync('key', 'value'); const lockedDirectory = join(dbPath, 'transaction_logs', 'locked'); mkdirSync(lockedDirectory, { recursive: true }); writeFileSync(join(lockedDirectory, 'leftover'), 'data'); @@ -122,10 +132,15 @@ describe('Destroy', () => { expect( registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending ).toBe(true); - expect(() => RocksDatabase.open(dbPath)).toThrow('Failed to remove database directory'); + expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); } finally { chmodSync(lockedDirectory, 0o700); } + shutdown(); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); + const healthyReopened = RocksDatabase.open(healthyPath); + expect(healthyReopened.getSync('key')).toBe('value'); + healthyReopened.destroy(); const reopened = RocksDatabase.open(dbPath); reopened.close(); }) From a9bfc713072a8bfe00161ee0121ba4914fb5128f Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:39:34 -0600 Subject: [PATCH 16/39] Release cancelled lifecycle operations --- src/binding/database/backup.cpp | 7 +++++++ src/binding/database/backup_stream.cpp | 5 +++++ src/binding/database/checkpoint.cpp | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 3b876fc4e..330657e06 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -398,6 +398,13 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { [](napi_env env, napi_status status, void* data) { // complete auto state = reinterpret_cast(data); state->deleteAsyncWork(); + // A cancelled queued work item never ran execute, so complete owns the + // in-flight decrement in that case. + if (status == napi_cancelled && + --state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing() + ) { + state->descriptor->operationsInFlight.notify_all(); + } if (status != napi_cancelled) { if (state->status.ok()) { napi_value result; diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 05d328ed2..5d53d98ea 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -571,6 +571,11 @@ void backupStreamExecute(napi_env, void* data) { void backupStreamComplete(napi_env env, napi_status status, void* data) { auto* state = static_cast(data); state->deleteAsyncWork(); + if (status == napi_cancelled && + --state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing() + ) { + state->descriptor->operationsInFlight.notify_all(); + } // Release the tsfn. On a normal run its queue is already drained; on a // teardown abort a trampoline may still be queued. Either way, tsfnFinalize diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index 930cca065..7a500dde4 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -167,6 +167,11 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { [](napi_env env, napi_status status, void* data) { // complete auto state = reinterpret_cast(data); state->deleteAsyncWork(); + if (status == napi_cancelled && + --state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing() + ) { + state->descriptor->operationsInFlight.notify_all(); + } if (status != napi_cancelled) { if (state->status.ok()) { napi_value undefined; From 74b36cafc0e32c630167d0b87e87e71fc1365c2b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:40:58 -0600 Subject: [PATCH 17/39] Preserve read-only destroy protection --- README.md | 9 ++++++++- src/binding/database/database.cpp | 22 ++++++++++++++++++++-- src/database.ts | 2 +- src/load-binding.ts | 2 +- test/destroy.test.ts | 4 ++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 576f8655c..f8d8a5163 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,14 @@ db.compactSync({ bottommost: true }); ### `db.destroy(): void` Completely removes a database based on the `db` instance's path including all data, column families, -and files on disk. +and files on disk. Destruction owns the physical path for the process: it closes every writable and +read-only handle for that path, waits for registered backups and checkpoints to stop using the +native database, and prevents another handle from reopening the path until removal finishes. Those +waits are synchronous and can outlive `lifecycleWaitSeconds` once destruction has claimed the path, +because releasing the native database beneath an active copy would be unsafe. + +The instance does not need to be open, which allows an explicit `destroy()` retry after failed +physical cleanup. A read-only instance cannot destroy the database. ```typescript db.destroy(); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 13384f7db..4ee4d3f0e 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -444,11 +444,25 @@ napi_value Database::CompactSync(napi_env env, napi_callback_info info) { * ``` */ napi_value Database::Destroy(napi_env env, napi_callback_info info) { - NAPI_METHOD_ARGV(1); + NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE(); - THROW_IF_READONLY((*dbHandle)->descriptor, "Destroy failed: "); if (*dbHandle) { + bool readOnly = (*dbHandle)->descriptor && (*dbHandle)->descriptor->readOnly; + napi_valuetype readOnlyType; + NAPI_STATUS_THROWS(::napi_typeof(env, argv[1], &readOnlyType)); + if (readOnlyType == napi_boolean) { + bool requestedReadOnly = false; + NAPI_STATUS_THROWS_ERROR(rocksdb_js::getValue(env, argv[1], requestedReadOnly), "Read-only flag must be a boolean"); + readOnly = readOnly || requestedReadOnly; + } else if (readOnlyType != napi_undefined) { + ::napi_throw_type_error(env, nullptr, "Read-only flag must be a boolean"); + return nullptr; + } + if (readOnly) { + ::napi_throw_error(env, "ERR_DATABASE_READONLY", "Destroy failed: Unsupported operation in read-only mode"); + return nullptr; + } std::string path = (*dbHandle)->path; napi_valuetype pathType; NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &pathType)); @@ -462,6 +476,10 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { ::napi_throw_error(env, nullptr, "Database path is required for destroy"); return nullptr; } + if (!(*dbHandle)->path.empty() && path != (*dbHandle)->path) { + ::napi_throw_error(env, nullptr, "Destroy path must match the open database"); + return nullptr; + } try { DBRegistry::DestroyDB(path); } catch (const std::exception& e) { diff --git a/src/database.ts b/src/database.ts index 4b34d0819..3faacf3ec 100644 --- a/src/database.ts +++ b/src/database.ts @@ -408,7 +408,7 @@ export class RocksDatabase extends DBI { // committed destroy(): void { - this.store.db.destroy(this.store.path); + this.store.db.destroy(this.store.path, this.store.readOnly); } async drop(): Promise { diff --git a/src/load-binding.ts b/src/load-binding.ts index cdc8cf3da..b2b210309 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -428,7 +428,7 @@ export type NativeDatabase = { reject: RejectCallback, targetPath: string ): void; - destroy(path?: string): void; + destroy(path?: string, readOnly?: boolean): void; drop(resolve: ResolveCallback, reject: RejectCallback): void; dropSync(): void; flush(resolve: ResolveCallback, reject: RejectCallback, options?: FlushOptions): void; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index d4f4f976f..51862608f 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -71,6 +71,10 @@ describe('Destroy', () => { it('should destroy a database from an unopened handle', () => dbRunner(({ db, dbPath }) => { db.close(); + expect(() => new RocksDatabase(dbPath, { readOnly: true }).destroy()).toThrow( + 'Unsupported operation in read-only mode' + ); + expect(existsSync(dbPath)).toBe(true); new RocksDatabase(dbPath).destroy(); expect(existsSync(dbPath)).toBe(false); })); From 4fbc5612e125d4c9e979f85f9aa945a5de130d9b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:02:56 -0600 Subject: [PATCH 18/39] Document destroy during stream backup --- docs/backups.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/backups.md b/docs/backups.md index 1728bfb60..52aba9969 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -202,8 +202,9 @@ The archive contains the standard RocksDB files for the snapshot (`CURRENT`, a ` consumer therefore lets obsolete SST files accumulate (transient disk pressure) until the stream finishes. This is the cost of streaming with no scratch copy; a fast consumer is unaffected. - **The database must stay open for the whole stream.** Closing or destroying the database while a - stream is in flight aborts it: the backup promise rejects, and `destroy()` throws rather than - tearing the database down underneath the copy. + stream is in flight aborts the stream and makes the backup promise reject. `destroy()` waits for + the native producer to release its in-flight claim, then closes every handle for the path and + removes the database; it never tears the native database down underneath the copy. - **A consumer error aborts the backup.** If `stream.write()` rejects (or the stream is aborted), the backup promise rejects and the native producer stops. From 1a57c929a24561d3458606f46458a98bf67db25e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:10:07 -0600 Subject: [PATCH 19/39] Restrict destroy to known database handles --- README.md | 5 +++-- src/binding/database/database.cpp | 23 +++++------------------ src/database.ts | 2 +- src/load-binding.ts | 2 +- test/destroy.test.ts | 7 +++++-- 5 files changed, 15 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index f8d8a5163..9bfcb7e99 100644 --- a/README.md +++ b/README.md @@ -362,8 +362,9 @@ native database, and prevents another handle from reopening the path until remov waits are synchronous and can outlive `lifecycleWaitSeconds` once destruction has claimed the path, because releasing the native database beneath an active copy would be unsafe. -The instance does not need to be open, which allows an explicit `destroy()` retry after failed -physical cleanup. A read-only instance cannot destroy the database. +A previously opened instance does not need to remain open, which allows an explicit `destroy()` +retry after failed physical cleanup. A never-opened or read-only instance cannot destroy the +database; `shutdown()` can recover a tombstone when its original handle is no longer available. ```typescript db.destroy(); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 4ee4d3f0e..43bf06221 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -444,16 +444,16 @@ napi_value Database::CompactSync(napi_env env, napi_callback_info info) { * ``` */ napi_value Database::Destroy(napi_env env, napi_callback_info info) { - NAPI_METHOD_ARGV(2); + NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE(); if (*dbHandle) { bool readOnly = (*dbHandle)->descriptor && (*dbHandle)->descriptor->readOnly; napi_valuetype readOnlyType; - NAPI_STATUS_THROWS(::napi_typeof(env, argv[1], &readOnlyType)); + NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &readOnlyType)); if (readOnlyType == napi_boolean) { bool requestedReadOnly = false; - NAPI_STATUS_THROWS_ERROR(rocksdb_js::getValue(env, argv[1], requestedReadOnly), "Read-only flag must be a boolean"); + NAPI_STATUS_THROWS_ERROR(rocksdb_js::getValue(env, argv[0], requestedReadOnly), "Read-only flag must be a boolean"); readOnly = readOnly || requestedReadOnly; } else if (readOnlyType != napi_undefined) { ::napi_throw_type_error(env, nullptr, "Read-only flag must be a boolean"); @@ -463,25 +463,12 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { ::napi_throw_error(env, "ERR_DATABASE_READONLY", "Destroy failed: Unsupported operation in read-only mode"); return nullptr; } - std::string path = (*dbHandle)->path; - napi_valuetype pathType; - NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &pathType)); - if (pathType == napi_string) { - NAPI_STATUS_THROWS_ERROR(rocksdb_js::getString(env, argv[0], path), "Database path must be a string"); - } else if (pathType != napi_undefined) { - ::napi_throw_type_error(env, nullptr, "Database path must be a string"); - return nullptr; - } - if (path.empty()) { + if ((*dbHandle)->path.empty()) { ::napi_throw_error(env, nullptr, "Database path is required for destroy"); return nullptr; } - if (!(*dbHandle)->path.empty() && path != (*dbHandle)->path) { - ::napi_throw_error(env, nullptr, "Destroy path must match the open database"); - return nullptr; - } try { - DBRegistry::DestroyDB(path); + DBRegistry::DestroyDB((*dbHandle)->path); } catch (const std::exception& e) { DEBUG_LOG("%p Database::Destroy Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); diff --git a/src/database.ts b/src/database.ts index 3faacf3ec..449ffa818 100644 --- a/src/database.ts +++ b/src/database.ts @@ -408,7 +408,7 @@ export class RocksDatabase extends DBI { // committed destroy(): void { - this.store.db.destroy(this.store.path, this.store.readOnly); + this.store.db.destroy(this.store.readOnly); } async drop(): Promise { diff --git a/src/load-binding.ts b/src/load-binding.ts index b2b210309..4c05dcb95 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -428,7 +428,7 @@ export type NativeDatabase = { reject: RejectCallback, targetPath: string ): void; - destroy(path?: string, readOnly?: boolean): void; + destroy(readOnly?: boolean): void; drop(resolve: ResolveCallback, reject: RejectCallback): void; dropSync(): void; flush(resolve: ResolveCallback, reject: RejectCallback, options?: FlushOptions): void; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 51862608f..34eea0f66 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -68,14 +68,17 @@ describe('Destroy', () => { expect(db.isOpen()).toBe(false); })); - it('should destroy a database from an unopened handle', () => + it('should reject destroy from a never-opened handle', () => dbRunner(({ db, dbPath }) => { db.close(); expect(() => new RocksDatabase(dbPath, { readOnly: true }).destroy()).toThrow( 'Unsupported operation in read-only mode' ); + expect(() => new RocksDatabase(dbPath).destroy()).toThrow( + 'Database path is required for destroy' + ); expect(existsSync(dbPath)).toBe(true); - new RocksDatabase(dbPath).destroy(); + db.destroy(); expect(existsSync(dbPath)).toBe(false); })); From deaf306f531423ab3a6b5edf78511369d5d53276 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:32:33 -0600 Subject: [PATCH 20/39] Run lifecycle fault fixtures under Node --- test/destroy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 34eea0f66..862eca06d 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -19,7 +19,7 @@ function runDestroyFixture( env?: NodeJS.ProcessEnv ): Promise { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, ['--expose-gc', fixture, dbPath], { + const child = spawn('node', ['--expose-gc', fixture, dbPath], { env: { ...process.env, ...env }, }); let stderr = ''; From 6236d8cea2537b25fd9c7ce3dae844a7655d589c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:40:19 -0600 Subject: [PATCH 21/39] Resolve Node for lifecycle fixtures --- test/destroy.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 862eca06d..dffaa858a 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -12,6 +12,11 @@ const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); +const nodeExecutable = + process.env.NODE_BINARY ?? + (process.versions.bun || process.versions.deno + ? (process.env.npm_node_execpath ?? 'node') + : process.execPath); function runDestroyFixture( fixture: string, @@ -19,7 +24,8 @@ function runDestroyFixture( env?: NodeJS.ProcessEnv ): Promise { return new Promise((resolve, reject) => { - const child = spawn('node', ['--expose-gc', fixture, dbPath], { + // These fixtures depend on Node's type stripping, GC flag, and worker semantics. + const child = spawn(nodeExecutable, ['--expose-gc', fixture, dbPath], { env: { ...process.env, ...env }, }); let stderr = ''; @@ -30,7 +36,10 @@ function runDestroyFixture( child.kill(); reject(new Error(`Destroy fixture timed out\n${stderr}`)); }, 10_000); - child.on('error', reject); + child.on('error', (error) => { + clearTimeout(timeout); + reject(new Error(`Unable to run lifecycle fixture with Node (${nodeExecutable}): ${error}`)); + }); child.on('close', (code, signal) => { clearTimeout(timeout); if (code === 0 && signal === null) { From db591e0b99d8d1f183a4724f894377bf5220aceb Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:59:13 -0600 Subject: [PATCH 22/39] Consume close-failure seam natively --- src/binding/core/test_seam.h | 15 +++++++++++++++ src/binding/database/db_descriptor.cpp | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index c7e3c15fb..4e0720c35 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -16,6 +16,21 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } +// Consume native fault flags in the same C runtime that reads them. JavaScript +// process.env deletion does not reliably update the MSVC runtime environment. +inline bool testConsumeFlag(const char* envName) { + const char* value = ::getenv(envName); + if (!value || ::atoi(value) <= 0) { + return false; + } +#ifdef _WIN32 + ::_putenv_s(envName, ""); +#else + ::unsetenv(envName); +#endif + return true; +} + // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 5058f99f4..485e0f79e 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -428,7 +428,7 @@ void DBDescriptor::finishClose() { } } - if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { + if (testConsumeFlag("ROCKSDB_JS_CLOSE_FAILURE")) { throw rocksdb_js::DBException("Injected database close failure"); } if (!this->db) { From d143093997a9a4e5011d5946e5f21bdc1482e27d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 01:23:40 -0600 Subject: [PATCH 23/39] Release copy pins before promise settlement --- src/binding/database/backup.cpp | 10 +++++++++- src/binding/database/checkpoint.cpp | 7 ++++++- test/backup.test.ts | 2 +- test/checkpoint.test.ts | 3 ++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 330657e06..a4a663010 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -60,7 +60,7 @@ struct AsyncBackupState final : BaseAsyncState> { // Our descriptor ref can be the reason a concurrent close skipped its // registry purge (use_count() > 1), so on release we must retry the purge or // the registry entry — and the open RocksDB — would linger forever. - ~AsyncBackupState() override { + void releaseDescriptor() { if (this->descriptor) { std::string path = this->descriptor->path; bool readOnly = this->descriptor->readOnly; @@ -68,6 +68,10 @@ struct AsyncBackupState final : BaseAsyncState> { DBRegistry::PurgeIfUnreferenced(path, readOnly); } } + + ~AsyncBackupState() override { + this->releaseDescriptor(); + } }; struct BackupInFlightClaim final { @@ -405,6 +409,10 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { ) { state->descriptor->operationsInFlight.notify_all(); } + // Promise settlement is the public completion boundary. Release the + // descriptor pin and retry any deferred registry purge first so an + // immediate registryStatus() / shutdown() cannot observe stale state. + state->releaseDescriptor(); if (status != napi_cancelled) { if (state->status.ok()) { napi_value result; diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index 7a500dde4..ddf8a1fc2 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -47,7 +47,7 @@ struct AsyncCheckpointState final : BaseAsyncState> { descriptor(std::move(descriptor)), targetPath(std::move(targetPath)) {} - ~AsyncCheckpointState() override { + void releaseDescriptor() { if (this->descriptor) { std::string path = this->descriptor->path; bool readOnly = this->descriptor->readOnly; @@ -55,6 +55,10 @@ struct AsyncCheckpointState final : BaseAsyncState> { DBRegistry::PurgeIfUnreferenced(path, readOnly); } } + + ~AsyncCheckpointState() override { + this->releaseDescriptor(); + } }; /** @@ -172,6 +176,7 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { ) { state->descriptor->operationsInFlight.notify_all(); } + state->releaseDescriptor(); if (status != napi_cancelled) { if (state->status.ok()) { napi_value undefined; diff --git a/test/backup.test.ts b/test/backup.test.ts index 18c5ff8f0..7e5eba489 100644 --- a/test/backup.test.ts +++ b/test/backup.test.ts @@ -475,7 +475,7 @@ describe('Backups', () => { // backup must retry it on release so the entry does not leak (a leaked // entry keeps the RocksDB open forever and shows up in registryStatus() // long after every handle is closed). - expect(registryStatus().length).toBe(0); + expect(registryStatus()).toEqual([]); })); it('should reject listing a non-existent backup directory', async () => { diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index 69fac4708..aed386c2f 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { existsSync, mkdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -204,6 +204,7 @@ describe('Checkpoints', () => { () => 'settled' ) ).resolves.toBe('settled'); + expect(registryStatus()).toEqual([]); })); it('should not free the database under an in-flight checkpoint when destroy() races it', () => From 8a37695ab29757ca432eb4426727a3dc989dc740 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 01:30:14 -0600 Subject: [PATCH 24/39] Align stream backup completion cleanup --- src/binding/database/backup_stream.cpp | 9 ++++++++- test/backup-stream.test.ts | 5 +++-- test/backup.test.ts | 4 ++-- test/checkpoint.test.ts | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 5d53d98ea..6eb8c6803 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -129,7 +129,7 @@ struct AsyncBackupStreamState final : BaseAsyncState> // Our descriptor ref can be the reason a concurrent close skipped its // registry purge (use_count() > 1), so on release we must retry the purge or // the registry entry — and the open RocksDB — would linger forever. - ~AsyncBackupStreamState() override { + void releaseDescriptor() { if (this->descriptor) { std::string path = this->descriptor->path; bool readOnly = this->descriptor->readOnly; @@ -138,6 +138,10 @@ struct AsyncBackupStreamState final : BaseAsyncState> } } + ~AsyncBackupStreamState() override { + this->releaseDescriptor(); + } + /** * Called on the JS thread (promise continuation / failure path) to wake the * blocked worker. `ok == false` requests the worker to abort the stream. @@ -587,6 +591,9 @@ void backupStreamComplete(napi_env env, napi_status status, void* data) { state->tsfn = nullptr; } + // Promise settlement is the public completion boundary. The worker is done, + // so pending JS acknowledgements no longer need the descriptor pin. + state->releaseDescriptor(); if (status != napi_cancelled) { if (state->status.ok()) { napi_value undefined; diff --git a/test/backup-stream.test.ts b/test/backup-stream.test.ts index d09ccfcf7..e655d562a 100644 --- a/test/backup-stream.test.ts +++ b/test/backup-stream.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { gunzipSync } from 'node:zlib'; @@ -164,7 +164,7 @@ describe('Streaming backups', () => { })); it('settles without hanging when the database is closed mid-stream', () => - dbRunner({ skipOpen: true }, async ({ db }) => { + dbRunner({ skipOpen: true }, async ({ db, dbPath }) => { db.open(); await writeAll(db, 200); @@ -186,6 +186,7 @@ describe('Streaming backups', () => { // If the worker deadlocked against close(), this would time out. await expect(settled).resolves.toBe('settled'); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); })); it('waits for an in-flight stream before destroying', () => diff --git a/test/backup.test.ts b/test/backup.test.ts index 7e5eba489..6aba2969e 100644 --- a/test/backup.test.ts +++ b/test/backup.test.ts @@ -452,7 +452,7 @@ describe('Backups', () => { ); it('should not crash when closing during a backup', () => - dbRunner({ skipOpen: true }, async ({ db }) => { + dbRunner({ skipOpen: true }, async ({ db, dbPath }) => { db.open(); await writeAll(db, 200); @@ -475,7 +475,7 @@ describe('Backups', () => { // backup must retry it on release so the entry does not leak (a leaked // entry keeps the RocksDB open forever and shows up in registryStatus() // long after every handle is closed). - expect(registryStatus()).toEqual([]); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); })); it('should reject listing a non-existent backup directory', async () => { diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index aed386c2f..9d958bce2 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -187,7 +187,7 @@ describe('Checkpoints', () => { })); it('should not crash when closing during a checkpoint', () => - dbRunner(async ({ db }) => { + dbRunner(async ({ db, dbPath }) => { await writeAll(db, 200); const checkpointDir = tempDir(); @@ -204,7 +204,7 @@ describe('Checkpoints', () => { () => 'settled' ) ).resolves.toBe('settled'); - expect(registryStatus()).toEqual([]); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); })); it('should not free the database under an in-flight checkpoint when destroy() races it', () => From f4409a5bb2a12ceaa38ae00a72bb751b66e55ed8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 01:41:10 -0600 Subject: [PATCH 25/39] Seed retry delay before fixture startup --- test/destroy.test.ts | 1 + test/fixtures/fork-shutdown-retry.mts | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/destroy.test.ts b/test/destroy.test.ts index dffaa858a..0677fa47b 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -208,6 +208,7 @@ describe('Destroy', () => { it('waits for an in-progress shutdown retry before reopening', async () => { await runDestroyFixture(shutdownRetryFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FAILURE: '1', + ROCKSDB_JS_CLOSE_RETRY_DELAY_MS: '1000', }); }, 15_000); }); diff --git a/test/fixtures/fork-shutdown-retry.mts b/test/fixtures/fork-shutdown-retry.mts index 109100dbc..0bc93e407 100644 --- a/test/fixtures/fork-shutdown-retry.mts +++ b/test/fixtures/fork-shutdown-retry.mts @@ -12,8 +12,6 @@ try { if (!String(error).includes('Injected database close failure')) throw error; } -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; -process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS = '1000'; const worker = new Worker(createWorkerBootstrapScript('./test/workers/shutdown-retry-worker.mts'), { eval: true, }); @@ -31,6 +29,5 @@ if (elapsed < 500) throw new Error(`Open did not wait for the shutdown retry (${ if (reopened.getSync('key') !== 'value') throw new Error('Shutdown retry did not preserve data'); const result = await shutdownResult; if (!result.shutdown) throw new Error(`Shutdown retry failed: ${JSON.stringify(result)}`); -delete process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS; reopened.destroy(); await worker.terminate(); From 0b47d273e09f6ad2154eed7964a930065a53dec1 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 11:27:00 -0600 Subject: [PATCH 26/39] Fix cross-thread destruction cleanup Co-Authored-By: GPT-5 Codex --- src/binding/binding.cpp | 18 +++++++++------ src/binding/core/test_seam.h | 31 +++++++++++++++----------- src/binding/database/db_descriptor.cpp | 2 +- src/binding/database/db_handle.cpp | 12 ++++++---- src/binding/database/db_registry.cpp | 26 +++++++++++---------- test/destroy.test.ts | 15 ++++++++++++- test/fixtures/fork-destroy-open.mts | 13 +++++++++++ 7 files changed, 79 insertions(+), 38 deletions(-) diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 19aecb435..eeae7372d 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,16 +39,19 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { + std::string error; try { DBRegistry::Shutdown(); - } catch (const std::exception& error) { - ::napi_throw_error(env, nullptr, error.what()); - return nullptr; + } catch (const std::exception& exception) { + error = exception.what(); } catch (...) { - ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); - return nullptr; + error = "Unknown native database shutdown failure"; } GlobalEvents::Shutdown(); + if (!error.empty()) { + ::napi_throw_error(env, nullptr, error.c_str()); + return nullptr; + } napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; @@ -162,6 +165,7 @@ napi_value TransactionLogMapCount(napi_env env, napi_callback_info info) { static std::atomic moduleRefCount{0}; NAPI_MODULE_INIT() { + initializeTestSeams(); #ifdef DEBUG // disable buffering for stderr to ensure messages are written immediately ::setvbuf(stderr, nullptr, _IONBF, 0); @@ -233,9 +237,9 @@ NAPI_MODULE_INIT() { ::fprintf(stderr, "rocksdb-js %s cleanup failed: unknown native error\n", name); } }; - cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); - cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); cleanup("database registry", []() { rocksdb_js::DBRegistry::Shutdown(); }); + cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); + cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { DEBUG_LOG("Binding::Init WARNING: Module ref count went negative!\n"); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index 4e0720c35..feaf200a1 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -3,6 +3,7 @@ #include #include +#include // Deterministic test seams that widen a race window are gated on a millisecond // delay read from an environment variable (0 = disabled). They are inert in @@ -16,19 +17,23 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } -// Consume native fault flags in the same C runtime that reads them. JavaScript -// process.env deletion does not reliably update the MSVC runtime environment. -inline bool testConsumeFlag(const char* envName) { - const char* value = ::getenv(envName); - if (!value || ::atoi(value) <= 0) { - return false; - } -#ifdef _WIN32 - ::_putenv_s(envName, ""); -#else - ::unsetenv(envName); -#endif - return true; +// Snapshot native fault flags once; mutating process.env while native workers +// can read it is unsafe, and process.env deletion does not update MSVC's CRT. +inline std::atomic& closeFailureFlag() { + static std::atomic pending{false}; + return pending; +} + +inline void initializeTestSeams() { + static std::once_flag initialized; + std::call_once(initialized, []() { + const char* value = ::getenv("ROCKSDB_JS_CLOSE_FAILURE"); + closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + }); +} + +inline bool testConsumeCloseFailure() { + return closeFailureFlag().exchange(false, std::memory_order_relaxed); } // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 485e0f79e..905b62535 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -428,7 +428,7 @@ void DBDescriptor::finishClose() { } } - if (testConsumeFlag("ROCKSDB_JS_CLOSE_FAILURE")) { + if (testConsumeCloseFailure()) { throw rocksdb_js::DBException("Injected database close failure"); } if (!this->db) { diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 60ab1ba29..13e520039 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -151,13 +151,17 @@ void DBHandle::close() { // DBRegistry::CloseTransactionsByEnv from the env cleanup hook // (HarperFast/rocksdb-js#741). - // release our reference to the descriptor - this->descriptor.reset(); + // A foreign thread can close a handle while its owner is copying this + // shared_ptr for an operation. Keep that member owner-thread-only; the + // descriptor itself is already closed before any foreign close returns. + if (std::this_thread::get_id() == this->ownerThreadId) { + this->descriptor.reset(); + } } // N-API references are environment-thread-affine. Destroying a shared - // descriptor can close this handle from another worker; retain the refs in - // that case so the owning environment's later close/finalizer releases them. + // descriptor can close this handle from another worker; retain the refs and + // descriptor until the owning environment's close or finalizer releases them. if (std::this_thread::get_id() == this->ownerThreadId) { for (auto& [name, ref] : this->logRefs) { DEBUG_LOG("%p DBHandle::close Releasing transaction log JS reference \"%s\"\n", this, name.c_str()); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 56f0a0235..5a01aca83 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -89,15 +89,15 @@ std::string destroyPhysicalPath(const std::string& path) { return {}; } +void emitCloseFailure(const std::string& path, const std::string& error) { + if (!error.empty() && GlobalEvents::hasListeners()) { + emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, error})); + } +} + void emitCloseFailures(const std::vector& descriptors) { - if (!GlobalEvents::hasListeners()) return; for (const auto& closing : descriptors) { - if (!closing.closeError.empty()) { - emitGlobalEvent( - "database:closeFailed", - ListenerData::fromStrings({closing.key.path, closing.closeError}) - ); - } + emitCloseFailure(closing.key.path, closing.closeError); } } @@ -228,9 +228,7 @@ CloseResult DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (condition) { condition->notify_all(); } - if (!closeError.empty() && GlobalEvents::hasListeners()) { - emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, closeError})); - } + emitCloseFailure(path, closeError); return CloseResult{closeError, quarantined}; } @@ -385,8 +383,11 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); const std::string destroyError = destroyPhysicalPath(path); if (!destroyError.empty()) { - std::lock_guard lock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = destroyError; + { + std::lock_guard lock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = destroyError; + } + emitCloseFailure(path, destroyError); throw rocksdb_js::DBException(destroyError); } @@ -1097,6 +1098,7 @@ void DBRegistry::Shutdown() { } } } + emitCloseFailure(key.path, cleanupError); if (!cleanupError.empty() && destroyCleanupError.empty()) { destroyCleanupError = "Cannot complete shutdown: database \"" + key.path + diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 0677fa47b..17108f11d 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -135,9 +135,17 @@ describe('Destroy', () => { it.skipIf(process.platform === 'win32')( 'quarantines a path when post-destroy cleanup fails', () => - dbRunner(({ db, dbPath }) => { + dbRunner(async ({ db, dbPath }) => { const healthyPath = generateDBPath(); const healthy = RocksDatabase.open(healthyPath); + let resolveCloseFailure: (args: unknown[]) => void; + const closeFailure = new Promise((resolve) => { + resolveCloseFailure = resolve; + }); + const listener = (...args: unknown[]) => { + if (args[0] === dbPath) resolveCloseFailure(args); + }; + RocksDatabase.on('database:closeFailed', listener); healthy.putSync('key', 'value'); const lockedDirectory = join(dbPath, 'transaction_logs', 'locked'); mkdirSync(lockedDirectory, { recursive: true }); @@ -149,7 +157,12 @@ describe('Destroy', () => { registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending ).toBe(true); expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); + await expect(closeFailure).resolves.toMatchObject([ + dbPath, + expect.stringContaining('Failed to remove database directory'), + ]); } finally { + RocksDatabase.off('database:closeFailed', listener); chmodSync(lockedDirectory, 0o700); } shutdown(); diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index ee1145cfc..7e6e55574 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -30,6 +30,17 @@ if (!destroying.destroying) const registryDeadline = Date.now() + 5_000; while (registryStatus().some((entry) => entry.path === path)) { if (Date.now() >= registryDeadline) throw new Error('Timed out waiting for the destroy window'); + try { + const value = original.getSync('before-destroy'); + if (value !== 'present') throw new Error(`Destroy raced a read with ${String(value)}`); + } catch (error) { + if ( + !String(error).includes('Database not open') && + !String(error).includes('Database is closing') + ) { + throw error; + } + } await delay(1); } @@ -43,6 +54,7 @@ if (process.env.ROCKSDB_JS_TEST_SHUTDOWN_DURING_DESTROY === '1') { if (shutdownDuration < 500) throw new Error(`Shutdown did not wait for destroy (${shutdownDuration}ms)`); await worker.terminate(); + original.close(); process.exit(0); } const reopened = RocksDatabase.open(path); @@ -57,3 +69,4 @@ if (reopened.getSync('after-destroy') !== 'present') throw new Error('Reopened database is not usable'); reopened.close(); await worker.terminate(); +original.close(); From 76b5b9f85fe016c1f7381309ed1d8d42bfddef7d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 11:42:44 -0600 Subject: [PATCH 27/39] Address lifecycle review feedback Co-Authored-By: GPT-5 Codex --- README.md | 8 +++++++- src/binding/binding.cpp | 2 +- src/binding/database/db_registry.cpp | 6 +++++- test/destroy.test.ts | 8 ++++---- test/fixtures/fork-close-failure.mts | 1 - test/fixtures/fork-destroy-failure.mts | 1 - test/fixtures/fork-gc-close-failure.mts | 1 - test/fixtures/fork-shutdown-failure.mts | 1 - 8 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9bfcb7e99..18c288b6b 100644 --- a/README.md +++ b/README.md @@ -1939,7 +1939,13 @@ any descriptor whose teardown did not complete: ```typescript import { shutdown } from '@harperfast/rocksdb-js'; -process.on('exit', shutdown); +process.on('exit', () => { + try { + shutdown(); + } catch (error) { + console.error('rocksdb-js shutdown failed', error); + } +}); ``` ### `versions: { 'rocksdb': string; 'rocksdb-js': string }` diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index eeae7372d..9dbc365c1 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -47,11 +47,11 @@ napi_value Shutdown(napi_env env, napi_callback_info info) { } catch (...) { error = "Unknown native database shutdown failure"; } - GlobalEvents::Shutdown(); if (!error.empty()) { ::napi_throw_error(env, nullptr, error.c_str()); return nullptr; } + GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 5a01aca83..5c8eda7b3 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -361,18 +361,22 @@ void DBRegistry::DestroyDB(const std::string& path) { if (closeError) std::rethrow_exception(closeError); } + std::vector> conditions; { std::lock_guard lock(instance->databasesMutex); for (auto it = instance->databases.begin(); it != instance->databases.end();) { if (it->first.path == path) { + conditions.push_back(it->second.condition); it = instance->databases.erase(it); } else { ++it; } } } + for (const auto& condition : conditions) { + condition->notify_all(); + } - // All in-process descriptors are closed; physical destruction can proceed. const int destroyDelayMs = testDelayMs("ROCKSDB_JS_DESTROY_DELAY_MS"); if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 17108f11d..b9f9d7bff 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -132,7 +132,7 @@ describe('Destroy', () => { } )); - it.skipIf(process.platform === 'win32')( + it.skipIf(process.platform === 'win32' || (process.getuid?.() ?? 0) === 0)( 'quarantines a path when post-destroy cleanup fails', () => dbRunner(async ({ db, dbPath }) => { @@ -163,9 +163,9 @@ describe('Destroy', () => { ]); } finally { RocksDatabase.off('database:closeFailed', listener); - chmodSync(lockedDirectory, 0o700); + if (existsSync(lockedDirectory)) chmodSync(lockedDirectory, 0o700); } - shutdown(); + db.destroy(); expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); const healthyReopened = RocksDatabase.open(healthyPath); expect(healthyReopened.getSync('key')).toBe('value'); diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts index 1a830754d..a4eb74236 100644 --- a/test/fixtures/fork-close-failure.mts +++ b/test/fixtures/fork-close-failure.mts @@ -26,7 +26,6 @@ try { if (Date.now() - startedAt >= 1_000) throw new Error('Open waited instead of reporting the failed automatic close immediately'); -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined automatic close'); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index a090d2dcc..73846ce12 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -39,7 +39,6 @@ if (closeFailure) { } if (Date.now() - startedAt >= 1_000) throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); - delete process.env.ROCKSDB_JS_CLOSE_FAILURE; db.destroy(); process.exit(0); } diff --git a/test/fixtures/fork-gc-close-failure.mts b/test/fixtures/fork-gc-close-failure.mts index cbc73eb36..6c1e9a759 100644 --- a/test/fixtures/fork-gc-close-failure.mts +++ b/test/fixtures/fork-gc-close-failure.mts @@ -33,7 +33,6 @@ if ( throw new Error('Automatic close failure was not quarantined'); } -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); const reopened = RocksDatabase.open(path); if (reopened.getSync('key') !== 'value') diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index a834c5671..592532b42 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -33,7 +33,6 @@ try { if (Date.now() - startedAt >= 1_000) throw new Error('Cross-mode open waited instead of reporting the quarantined path immediately'); -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined descriptor'); From b1c271346693c10960cf2e3a517410da1342cf8c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 23:09:43 -0600 Subject: [PATCH 28/39] fix(lifecycle): close teardown review gaps Co-Authored-By: GPT-5 Codex --- AGENTS.md | 8 +++++- README.md | 10 +++++--- src/binding/database/database.cpp | 33 ++++++++++++++++++++++--- src/binding/database/database.h | 6 ++++- src/binding/database/db_handle.cpp | 5 +--- src/binding/database/db_registry.cpp | 32 ++++++------------------ src/database.ts | 2 +- src/load-binding.ts | 1 + src/store.ts | 4 +-- test/destroy.test.ts | 8 +++++- test/fixtures/fork-shutdown-failure.mts | 1 + 11 files changed, 67 insertions(+), 43 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2939e9058..6932474cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -308,7 +308,13 @@ sufficient (env teardown does not honor tsfn acquire counts); see their own `shared_ptr` for the duration of a copy (backup, backup stream, checkpoint) make a racing close skip the purge (`use_count > 1`), so their state destructors re-run `PurgeIfUnreferenced` after releasing the ref — without that retry the skipped purge is permanent - and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). + and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). Once `beginClose()` wins, + `DBHandle::opened()` must report false even while the native DB still exists. Any synchronous N-API + path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` + immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer + from another env after the in-flight count drains. A failed physical destroy leaves a registry + tombstone, but `shutdown()` is deliberately non-destructive: it reports the tombstone and only an + explicit `destroy()` retries path deletion. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/README.md b/README.md index 18c288b6b..63f45407c 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Closes a database. This function can be called multiple times and will only clos database. A database instance can be reopened once it is closed. A flush or compaction failure is reported as an exception after native teardown completes. All native close errors emit `database:closeFailed`; when teardown does not complete, same-path opens also fail until -`destroy()` or `shutdown()` retries cleanup. The quarantine applies to both writable and read-only +`destroy()` retries cleanup. The quarantine applies to both writable and read-only opens because both modes share the physical path lifecycle. ```typescript @@ -364,7 +364,8 @@ because releasing the native database beneath an active copy would be unsafe. A previously opened instance does not need to remain open, which allows an explicit `destroy()` retry after failed physical cleanup. A never-opened or read-only instance cannot destroy the -database; `shutdown()` can recover a tombstone when its original handle is no longer available. +database. `shutdown()` reports a pending cleanup tombstone but never retries deletion; only an +explicit `destroy()` can remove the path. ```typescript db.destroy(); @@ -1915,7 +1916,7 @@ Returns an array containing that status of all active RocksDB instances. - `path: string` The database path. - `closeError?: string` The native lifecycle error retaining this registry entry. - `destroyCleanupPending?: boolean` The native database is closed, but physical path cleanup must - finish before the next open. Call `destroy()` or `shutdown()` to retry cleanup. + finish before the next open. Call `destroy()` to retry cleanup. - `refCount: number` The number of JavaScript database instances plus the registry's reference. - `columnFamiles: object` A map of column family names and their their info. - `userSharedBuffers: number` The count of active user shared buffers. @@ -1935,7 +1936,8 @@ The `shutdown()` will flush all in-memory data to disk and wait for any outstand finish, for all open databases. It is highly recommended to call this in a `process` `exit` event listener (on the main thread), to ensure that all data is flushed to disk before the process exits. It throws the first close failure after attempting every claimed database; call it again to retry -any descriptor whose teardown did not complete: +any descriptor whose native teardown did not complete. It reports pending destroy-cleanup +tombstones without deleting their paths; retry those with an explicit `destroy()`: ```typescript import { shutdown } from '@harperfast/rocksdb-js'; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 43bf06221..d7c5979cc 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -389,6 +389,7 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { napi_value Database::CompactSync(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(3); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice startSlice; rocksdb::Slice* startPtr = nullptr; @@ -508,6 +509,7 @@ static bool isColumnFamilyAlreadyDropped(const rocksdb::Status& status) { napi_value Database::Drop(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); if ((*dbHandle)->getColumnFamilyName() == "default") { return doClear(env, info, "Drop failed"); @@ -568,12 +570,12 @@ napi_value Database::Drop(napi_env env, napi_callback_info info) { napi_value Database::DropSync(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); if ((*dbHandle)->getColumnFamilyName() == "default") { return doClearSync(env, info, "Drop failed"); } - ACQUIRE_OPERATIONS_LOCK(); DEBUG_LOG("%p Database::DropSync dropping database: %s\n", dbHandle->get(), (*dbHandle)->path.c_str()); rocksdb::Status status = (*dbHandle)->descriptor->db->DropColumnFamily((*dbHandle)->getColumnFamilyHandle()); if (!status.ok() && !isColumnFamilyAlreadyDropped(status)) { @@ -1001,6 +1003,7 @@ napi_value Database::Resume(napi_env env, napi_callback_info info) { napi_value Database::GetCompression(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Options opts = (*dbHandle)->descriptor->db->GetOptions((*dbHandle)->getColumnFamilyHandle()); std::string name = compressionNameFromType(opts.compression); @@ -1075,6 +1078,7 @@ napi_value Database::GetLogOptions(napi_env env, napi_callback_info info) { napi_value Database::GetCount(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); DBIteratorOptions itOptions; itOptions.initFromNapiObject(env, argv[0]); @@ -1332,6 +1336,7 @@ napi_value Database::GetMonotonicTimestamp(napi_env env, napi_callback_info info napi_value Database::GetOldestSnapshotTimestamp(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); uint64_t timestamp = 0; bool success = (*dbHandle)->descriptor->db->GetIntProperty( @@ -1362,6 +1367,7 @@ napi_value Database::GetOldestSnapshotTimestamp(napi_env env, napi_callback_info napi_value Database::GetDBProperty(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); NAPI_GET_STRING(argv[0], propertyName, "Property name is required"); @@ -1398,6 +1404,7 @@ napi_value Database::GetDBProperty(napi_env env, napi_callback_info info) { napi_value Database::GetDBIntProperty(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); NAPI_GET_STRING(argv[0], propertyName, "Property name is required"); @@ -1428,6 +1435,7 @@ napi_value Database::GetDBIntProperty(napi_env env, napi_callback_info info) { napi_value Database::GetStat(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); NAPI_GET_STRING(argv[0], statName, "Stat name is required"); return (*dbHandle)->getStat(env, statName); } @@ -1444,6 +1452,7 @@ napi_value Database::GetStat(napi_env env, napi_callback_info info) { napi_value Database::GetStats(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); bool all = false; NAPI_STATUS_THROWS(::napi_get_value_bool(env, argv[0], &all)); @@ -1793,6 +1802,7 @@ napi_value Database::GetUserSharedBuffer(napi_env env, napi_callback_info info) NAPI_METHOD_ARGV(3); NAPI_GET_BUFFER(argv[0], key, "Key is required"); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); std::string keyStr(key + keyStart, keyEnd - keyStart); // if we have a callback, add it as a listener @@ -1839,6 +1849,20 @@ napi_value Database::HasLock(napi_env env, napi_callback_info info) { return result; } +/** + * Checks if the RocksDB database is closing or quarantined. + */ +napi_value Database::IsClosing(napi_env env, napi_callback_info info) { + NAPI_METHOD(); + UNWRAP_DB_HANDLE(); + + const bool closing = dbHandle != nullptr && *dbHandle && (*dbHandle)->descriptor && + (*dbHandle)->descriptor->isClosing(); + napi_value result; + NAPI_STATUS_THROWS(::napi_get_boolean(env, closing, &result)); + return result; +} + /** * Checks if the RocksDB database is open. */ @@ -2105,10 +2129,10 @@ napi_value Database::PurgeLogs(napi_env env, napi_callback_info info) { */ napi_value Database::PutSync(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(3); - NAPI_GET_BUFFER(argv[0], key, "Key is required"); - NAPI_GET_BUFFER(argv[1], value, nullptr); UNWRAP_DB_HANDLE_AND_OPEN(); ACQUIRE_OPERATIONS_LOCK(); + NAPI_GET_BUFFER(argv[0], key, "Key is required"); + NAPI_GET_BUFFER(argv[1], value, nullptr); // THROW_IF_READONLY((*dbHandle)->descriptor, "Put failed: "); rocksdb::Status status; @@ -2191,9 +2215,9 @@ napi_value Database::PutSync(napi_env env, napi_callback_info info) { */ napi_value Database::RemoveSync(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); - NAPI_GET_BUFFER(argv[0], key, "Key is required"); UNWRAP_DB_HANDLE_AND_OPEN(); ACQUIRE_OPERATIONS_LOCK(); + NAPI_GET_BUFFER(argv[0], key, "Key is required"); // THROW_IF_READONLY((*dbHandle)->descriptor, "Remove failed: "); rocksdb::Status status; @@ -2407,6 +2431,7 @@ void Database::Init(napi_env env, napi_value exports) { { "getSync", nullptr, GetSync, nullptr, nullptr, nullptr, napi_default, nullptr }, { "getUserSharedBuffer", nullptr, GetUserSharedBuffer, nullptr, nullptr, nullptr, napi_default, nullptr }, { "hasLock", nullptr, HasLock, nullptr, nullptr, nullptr, napi_default, nullptr }, + { "closing", nullptr, nullptr, IsClosing, nullptr, nullptr, napi_default, nullptr }, { "listeners", nullptr, Listeners, nullptr, nullptr, nullptr, napi_default, nullptr }, { "listLogs", nullptr, ListLogs, nullptr, nullptr, nullptr, napi_default, nullptr }, { "notify", nullptr, Notify, nullptr, nullptr, nullptr, napi_default, nullptr }, diff --git a/src/binding/database/database.h b/src/binding/database/database.h index 7ff785fd8..11ee429c2 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -227,7 +227,10 @@ inline void vtPopulateIfSettled( UNWRAP_DB_HANDLE(); \ do { \ if (dbHandle == nullptr || !(*dbHandle)->opened()) { \ - ::napi_throw_error(env, nullptr, "Database not open"); \ + const char* message = dbHandle != nullptr && *dbHandle && (*dbHandle)->descriptor && (*dbHandle)->descriptor->isClosing() \ + ? "Database is closing" \ + : "Database not open"; \ + ::napi_throw_error(env, nullptr, message); \ NAPI_RETURN_UNDEFINED(); \ } \ } while (0) @@ -327,6 +330,7 @@ struct Database final { static napi_value GetSync(napi_env env, napi_callback_info info); static napi_value GetUserSharedBuffer(napi_env env, napi_callback_info info); static napi_value HasLock(napi_env env, napi_callback_info info); + static napi_value IsClosing(napi_env env, napi_callback_info info); static napi_value IsOpen(napi_env env, napi_callback_info info); static napi_value Listeners(napi_env env, napi_callback_info info); static napi_value ListLogs(napi_env env, napi_callback_info info); diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 13e520039..a699e4e65 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -362,10 +362,7 @@ void DBHandle::open(const std::string& path, const DBOptions& options) { * Checks if the referenced database is opened. */ bool DBHandle::opened() const { - if (this->descriptor && this->descriptor->db) { - return true; - } - return false; + return this->descriptor && !this->descriptor->isClosing(); } /** diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 5c8eda7b3..5e579c015 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -1014,7 +1014,7 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; - std::vector destroyCleanupEntries; + std::string destroyCleanupError; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -1022,14 +1022,17 @@ void DBRegistry::Shutdown() { DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); descriptorsToClose.reserve(instance->databases.size()); descriptorsToWaitFor.reserve(instance->databases.size()); - destroyCleanupEntries.reserve(instance->databases.size()); for (auto& [key, entry] : instance->databases) { if (instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end()) { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty()) destroyCleanupEntries.push_back(key); + if (!entry.closeError.empty() && destroyCleanupError.empty()) { + destroyCleanupError = + "Cannot complete shutdown: database \"" + key.path + + "\" requires explicit destroy() cleanup: " + entry.closeError; + } continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; @@ -1088,27 +1091,6 @@ void DBRegistry::Shutdown() { } } - std::string destroyCleanupError; - for (const auto& key : destroyCleanupEntries) { - const std::string cleanupError = destroyPhysicalPath(key.path); - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(key); - if (entry != instance->databases.end() && !entry->second.descriptor) { - if (cleanupError.empty()) { - instance->databases.erase(entry); - } else { - entry->second.closeError = cleanupError; - } - } - } - emitCloseFailure(key.path, cleanupError); - if (!cleanupError.empty() && destroyCleanupError.empty()) { - destroyCleanupError = - "Cannot complete shutdown: database \"" + key.path + - "\" requires destroy cleanup: " + cleanupError; - } - } if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { @@ -1127,7 +1109,7 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException(destroyCleanupError); } if (descriptorsToClose.empty() && descriptorsToWaitFor.empty() && - destroyCleanupEntries.empty() + destroyCleanupError.empty() ) break; } diff --git a/src/database.ts b/src/database.ts index 449ffa818..81a8244df 100644 --- a/src/database.ts +++ b/src/database.ts @@ -350,7 +350,7 @@ export class RocksDatabase extends DBI { * matching event emitted in this process. Native lifecycle failures use * `'database:closeFailed'` with `(path, error)` string arguments. The event * reports both completed and incomplete teardowns; only an incomplete - * teardown quarantines the path until `destroy()` or `shutdown()` retries. + * teardown quarantines the path until `destroy()` retries. * * @example * ```typescript diff --git a/src/load-binding.ts b/src/load-binding.ts index 4c05dcb95..210a5f961 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -469,6 +469,7 @@ export type NativeDatabase = { hasLock(key: BufferWithDataView): boolean; listeners(event: string | BufferWithDataView): number; listLogs(): string[]; + closing: boolean; opened: boolean; open(path: string, options?: NativeDatabaseOptions): void; populateVersion(keyLengthOrKeyBuffer: number | Buffer, version: number): void; diff --git a/src/store.ts b/src/store.ts index eeb6be55a..1a75613f1 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1172,7 +1172,7 @@ export class Store { putSync(context: StoreContext, key: Key, value: any, options?: StorePutOptions): void { if (!this.db.opened) { - throw new Error('Database not open'); + throw new Error(this.db.closing ? 'Database is closing' : 'Database not open'); } // IMPORTANT! @@ -1187,7 +1187,7 @@ export class Store { removeSync(context: StoreContext, key: Key, options?: StoreRemoveOptions): void { if (!this.db.opened) { - throw new Error('Database not open'); + throw new Error(this.db.closing ? 'Database is closing' : 'Database not open'); } context.removeSync(this.encodeKey(key), this.getTxnId(options)); diff --git a/test/destroy.test.ts b/test/destroy.test.ts index b9f9d7bff..e02197916 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus } from '../src/index.ts'; +import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -161,6 +161,12 @@ describe('Destroy', () => { dbPath, expect.stringContaining('Failed to remove database directory'), ]); + chmodSync(lockedDirectory, 0o700); + expect(() => shutdown()).toThrow('requires explicit destroy() cleanup'); + expect(existsSync(dbPath)).toBe(true); + expect( + registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending + ).toBe(true); } finally { RocksDatabase.off('database:closeFailed', listener); if (existsSync(lockedDirectory)) chmodSync(lockedDirectory, 0o700); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 592532b42..ef7778f12 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -22,6 +22,7 @@ const args = await Promise.race([ if (args[0] !== path || args[1] !== 'Injected database close failure') { throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); } +if (db.isOpen()) throw new Error('A quarantined database still reports itself open'); const startedAt = Date.now(); try { RocksDatabase.open(path, { readOnly: true }); From 39b74dbb7d96f6da1427798b70d7557da9b1c8ae Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 23:35:42 -0600 Subject: [PATCH 29/39] fix(lifecycle): quarantine unsafe close failures Co-Authored-By: GPT-5 Codex --- AGENTS.md | 5 ++++- README.md | 10 +++++----- src/binding/core/test_seam.h | 11 +++++++++++ src/binding/database/database.cpp | 4 +++- src/binding/database/db_descriptor.cpp | 9 +++++++-- src/binding/database/db_descriptor.h | 2 +- src/binding/database/db_handle.cpp | 1 + src/binding/database/db_handle.h | 2 ++ src/binding/database/db_registry.cpp | 6 ++++-- src/database.ts | 4 ++-- test/destroy.test.ts | 7 +++++++ test/drop.test.ts | 2 ++ test/fixtures/fork-flush-failure.mts | 27 ++++++++++++++++++++++++++ 13 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 test/fixtures/fork-flush-failure.mts diff --git a/AGENTS.md b/AGENTS.md index 6932474cd..2847610e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -312,7 +312,10 @@ sufficient (env teardown does not honor tsfn acquire counts); see `DBHandle::opened()` must report false even while the native DB still exists. Any synchronous N-API path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer - from another env after the in-flight count drains. A failed physical destroy leaves a registry + from another env after the in-flight count drains. `DBHandle::close()` itself is cross-env and must + serialize mutation of its `shared_ptr` members. A close-time flush failure keeps the native DB + quarantined so `shutdown()` can retry without losing `disableWAL` writes; an explicit destroy may + force teardown because the caller requested deletion. A failed physical destroy leaves a registry tombstone, but `shutdown()` is deliberately non-destructive: it reports the tombstone and only an explicit `destroy()` retries path deletion. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its diff --git a/README.md b/README.md index 63f45407c..2cd7214f8 100644 --- a/README.md +++ b/README.md @@ -141,11 +141,11 @@ Creates a new database instance. ### `db.close()` Closes a database. This function can be called multiple times and will only close an opened -database. A database instance can be reopened once it is closed. A flush or compaction failure is -reported as an exception after native teardown completes. All native close errors emit -`database:closeFailed`; when teardown does not complete, same-path opens also fail until -`destroy()` retries cleanup. The quarantine applies to both writable and read-only -opens because both modes share the physical path lifecycle. +database. A database instance can be reopened once it is closed. A flush failure leaves the native +database quarantined so `shutdown()` can retry without losing unflushed data; an explicit +`destroy()` can instead delete it. A compaction failure is reported after native teardown +completes. All native close errors emit `database:closeFailed`. The quarantine applies to both +writable and read-only opens because both modes share the physical path lifecycle. ```typescript const db = RocksDatabase.open('foo'); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index feaf200a1..effdcd7ee 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -24,11 +24,18 @@ inline std::atomic& closeFailureFlag() { return pending; } +inline std::atomic& closeFlushFailureFlag() { + static std::atomic pending{false}; + return pending; +} + inline void initializeTestSeams() { static std::once_flag initialized; std::call_once(initialized, []() { const char* value = ::getenv("ROCKSDB_JS_CLOSE_FAILURE"); closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + value = ::getenv("ROCKSDB_JS_CLOSE_FLUSH_FAILURE"); + closeFlushFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); }); } @@ -36,6 +43,10 @@ inline bool testConsumeCloseFailure() { return closeFailureFlag().exchange(false, std::memory_order_relaxed); } +inline bool testConsumeCloseFlushFailure() { + return closeFlushFailureFlag().exchange(false, std::memory_order_relaxed); +} + // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index d7c5979cc..5e9e0f149 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -194,7 +194,7 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { if (!closeResult.error.empty()) { std::string message = closeResult.error; if (closeResult.quarantined) { - message += ". Call destroy() or shutdown() to retry cleanup"; + message += ". Call shutdown() to retry close, or destroy() to delete the database"; } ::napi_throw_error(env, nullptr, message.c_str()); return nullptr; @@ -1663,6 +1663,7 @@ napi_value Database::GetSync(napi_env env, napi_callback_info info) { napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { @@ -1699,6 +1700,7 @@ napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { napi_value Database::PopulateVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 905b62535..0971c98ed 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -383,7 +383,7 @@ void DBDescriptor::close() { this->finishClose(); } -void DBDescriptor::finishClose() { +void DBDescriptor::finishClose(bool destroying) { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); @@ -438,9 +438,14 @@ void DBDescriptor::finishClose() { // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an // immediate flush races transaction-log-store teardown (AGENTS invariant 15). std::string closeError; - rocksdb::Status status = this->flush(); + rocksdb::Status status = testConsumeCloseFlushFailure() + ? rocksdb::Status::IOError("Injected database close flush failure") + : this->flush(); if (!status.ok()) { closeError = "Failed to flush database during close: " + status.ToString(); + if (!destroying) { + throw rocksdb_js::DBException(closeError); + } } // Trigger manual compaction on all column families to reclaim space from diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index f13a75dbd..e88dc2cad 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -458,7 +458,7 @@ struct DBDescriptor final : public std::enable_shared_from_this { * Only valid after `beginClose()` returned true; `close()` is the all-in-one * entry point that claims and then runs this. */ - void finishClose(); + void finishClose(bool destroying = false); void attach(std::shared_ptr closable); void detach(std::shared_ptr closable); diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index a699e4e65..88a61816f 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -126,6 +126,7 @@ rocksdb::Status DBHandle::clear() { * Closes the DBHandle. */ void DBHandle::close() { + std::lock_guard closeLock(this->closeMutex); DEBUG_LOG("%p DBHandle::close dbDescriptor=%p (ref count = %ld)\n", this, this->descriptor.get(), this->descriptor.use_count()); // cancel all active async work before closing diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 625f51e77..60ffbf067 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "rocksdb/db.h" #include "database/db_descriptor.h" @@ -72,6 +73,7 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro * get the `TransactionLog` class. */ napi_ref exportsRef; + std::mutex closeMutex; /** * The default transaction log store. diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 5e579c015..bc13de015 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -307,7 +307,7 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { - closing.descriptor->finishClose(); + closing.descriptor->finishClose(true); closing.closed = true; } catch (const std::exception& error) { closing.closed = closing.descriptor->isClosed(); @@ -469,7 +469,9 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons "Cannot open database \"" + path + "\": previous " + (destroyCleanupFailed ? "destroy cleanup" : "close") + " failed: " + registeredEntry->second.closeError + - (destroyCleanupFailed ? ". Call destroy() to retry cleanup" : ". Call destroy() or shutdown() to retry cleanup") + (destroyCleanupFailed + ? ". Call destroy() to retry cleanup" + : ". Call shutdown() to retry close, or destroy() to delete the database") ); } } diff --git a/src/database.ts b/src/database.ts index 81a8244df..05d487328 100644 --- a/src/database.ts +++ b/src/database.ts @@ -349,8 +349,8 @@ export class RocksDatabase extends DBI { * Listeners are not tied to any specific database — they fire for every * matching event emitted in this process. Native lifecycle failures use * `'database:closeFailed'` with `(path, error)` string arguments. The event - * reports both completed and incomplete teardowns; only an incomplete - * teardown quarantines the path until `destroy()` retries. + * reports both completed and incomplete teardowns; an incomplete close can + * be retried by `shutdown()` or explicitly deleted by `destroy()`. * * @example * ```typescript diff --git a/test/destroy.test.ts b/test/destroy.test.ts index e02197916..c1211ae08 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -11,6 +11,7 @@ const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts' const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); +const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); const nodeExecutable = process.env.NODE_BINARY ?? @@ -224,6 +225,12 @@ describe('Destroy', () => { }); }, 15_000); + it('quarantines a flush failure until shutdown preserves the unflushed data', async () => { + await runDestroyFixture(flushFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FLUSH_FAILURE: '1', + }); + }, 15_000); + it('waits for an in-progress shutdown retry before reopening', async () => { await runDestroyFixture(shutdownRetryFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FAILURE: '1', diff --git a/test/drop.test.ts b/test/drop.test.ts index d88632a47..c19987d16 100644 --- a/test/drop.test.ts +++ b/test/drop.test.ts @@ -240,6 +240,8 @@ describe('Drop', () => { stale.close(); doomed.close(); expect(() => victim.close()).toThrow('Failed to flush database during close'); + expect(() => RocksDatabase.open(dbPath)).toThrow('previous close failed'); + victim.destroy(); const reopened = RocksDatabase.open(dbPath); reopened.close(); } diff --git a/test/fixtures/fork-flush-failure.mts b/test/fixtures/fork-flush-failure.mts new file mode 100644 index 000000000..7c1dca4de --- /dev/null +++ b/test/fixtures/fork-flush-failure.mts @@ -0,0 +1,27 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path, { disableWAL: true }); +db.putSync('key', 'unflushed'); + +try { + db.close(); + throw new Error('Expected close to surface the injected flush failure'); +} catch (error) { + if (!String(error).includes('Injected database close flush failure')) throw error; +} +if (db.isOpen()) throw new Error('A flush-failed database still reports itself open'); +if (!registryStatus().some((entry) => entry.path === path && entry.closeError)) + throw new Error('Flush failure did not quarantine the descriptor'); +try { + RocksDatabase.open(path); + throw new Error('Expected the flush failure to block reopen'); +} catch (error) { + if (!String(error).includes('previous close failed')) throw error; +} + +shutdown(); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'unflushed') + throw new Error('Shutdown retry did not preserve the unflushed write'); +reopened.destroy(); From 9bc9d3f6b17b91429754538d8eac7726ee2650ec Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:09:36 -0600 Subject: [PATCH 30/39] Fix worker teardown ordering in CI gates Co-Authored-By: GPT-5 Codex --- benchmark/setup.ts | 22 ++++++++----------- stress-test/db-instances.stress.test.ts | 14 ++++++++---- .../workers/stress-db-instances-worker.mts | 1 + .../workers/stress-transaction-put-worker.mts | 4 ++++ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/benchmark/setup.ts b/benchmark/setup.ts index 89c93e79e..820c39f46 100644 --- a/benchmark/setup.ts +++ b/benchmark/setup.ts @@ -381,6 +381,7 @@ export function workerBenchmark(type: string, options: any): void { } const workerState: WorkerState[] = []; + let dbPath: string; const workerPayload = { suites: workerCurrentSuites.map((suite) => suite.name), benchmark: benchmarkName, @@ -408,11 +409,7 @@ export function workerBenchmark(type: string, options: any): void { if (mode === 'run') { return; } - const path = join( - 'benchmark', - 'data', - `rocksdb-benchmark-${randomBytes(8).toString('hex')}` - ); + dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`); let teardownTimeoutId: NodeJS.Timeout; await Promise.race([ @@ -436,7 +433,7 @@ export function workerBenchmark(type: string, options: any): void { benchmarkFile: pathToFileURL(benchmarkFile).toString(), benchmarkWorkerId: i + 1, mode, - path, + path: dbPath, }); // important! these promises need to be referenced as // properties of `state` because they are reset by reference @@ -492,6 +489,11 @@ export function workerBenchmark(type: string, options: any): void { return workerState[i].exitPromise.promise; }) ); + try { + rmSync(dbPath, { force: true, recursive: true, maxRetries: 3 }); + } catch (err) { + console.warn(`Benchmark teardown failed to delete db path: ${err}`); + } resolve(); }, @@ -529,13 +531,7 @@ export async function workerInit(): Promise { await teardown(ctx); } if (ctx.db) { - // console.log('workerTeardown', workerData.benchmarkWorkerId, workerData.mode, type, path); - ctx.db.close(); - try { - rmSync(path, { force: true, recursive: true, maxRetries: 3 }); - } catch (err) { - console.warn(`Benchmark teardown failed to delete db path: ${err}`); - } + await ctx.db.close(); } parentPort!.postMessage({ teardownDone: true, benchmarkWorkerId }); process.exit(0); diff --git a/stress-test/db-instances.stress.test.ts b/stress-test/db-instances.stress.test.ts index 895946aa9..967e974a2 100644 --- a/stress-test/db-instances.stress.test.ts +++ b/stress-test/db-instances.stress.test.ts @@ -39,8 +39,6 @@ describe('Stress DB Instances', () => { worker.on('message', (event) => { if (event.done) { resolve(); - } else if (event.closed) { - resolve(); } }); }) @@ -48,14 +46,22 @@ describe('Stress DB Instances', () => { } await Promise.all(promises); - promises.length = 0; const [before] = registryStatus(); + const closePromises = workers.map( + (worker) => + new Promise((resolve, reject) => { + worker.on('error', reject); + worker.on('message', (event) => { + if (event.closed) resolve(); + }); + }) + ); for (const worker of workers) { worker.postMessage({ close: true }); } - await Promise.all(promises); + await Promise.all(closePromises); if (globalThis.gc) { globalThis.gc(); diff --git a/stress-test/workers/stress-db-instances-worker.mts b/stress-test/workers/stress-db-instances-worker.mts index 9908e23c0..857dba3ef 100644 --- a/stress-test/workers/stress-db-instances-worker.mts +++ b/stress-test/workers/stress-db-instances-worker.mts @@ -21,5 +21,6 @@ parentPort?.on('message', (event) => { db.close(); } parentPort?.postMessage({ closed: true }); + parentPort?.close(); } }); diff --git a/stress-test/workers/stress-transaction-put-worker.mts b/stress-test/workers/stress-transaction-put-worker.mts index e82439a41..e14cda5f0 100644 --- a/stress-test/workers/stress-transaction-put-worker.mts +++ b/stress-test/workers/stress-transaction-put-worker.mts @@ -27,7 +27,9 @@ async function runTransactions10k() { await last; } + db.close(); parentPort?.postMessage({ done: true }); + parentPort?.close(); } async function runTransactions10kWithLogs() { @@ -44,5 +46,7 @@ async function runTransactions10kWithLogs() { }); } + db.close(); parentPort?.postMessage({ done: true }); + parentPort?.close(); } From ecf1e37e9e55ee1e2c2d75cabf9b9d6015b9a02a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:26:02 -0600 Subject: [PATCH 31/39] Surface worker benchmark teardown failures Co-Authored-By: GPT-5 Codex --- benchmark/setup.ts | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/benchmark/setup.ts b/benchmark/setup.ts index 820c39f46..b501a28ee 100644 --- a/benchmark/setup.ts +++ b/benchmark/setup.ts @@ -235,6 +235,7 @@ interface WorkerState { benchPromise: ReturnType>; exitPromise: ReturnType>; teardownPromise: ReturnType>; + teardownError?: Error; } interface WorkerBenchmarkOptions extends BenchmarkOptions { @@ -381,7 +382,7 @@ export function workerBenchmark(type: string, options: any): void { } const workerState: WorkerState[] = []; - let dbPath: string; + const dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`); const workerPayload = { suites: workerCurrentSuites.map((suite) => suite.name), benchmark: benchmarkName, @@ -409,8 +410,6 @@ export function workerBenchmark(type: string, options: any): void { if (mode === 'run') { return; } - dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`); - let teardownTimeoutId: NodeJS.Timeout; await Promise.race([ activeBenchmark, @@ -437,7 +436,7 @@ export function workerBenchmark(type: string, options: any): void { }); // important! these promises need to be referenced as // properties of `state` because they are reset by reference - const state = { + const state: WorkerState = { worker, benchPromise: withResolvers(), exitPromise: withResolvers(), @@ -445,7 +444,12 @@ export function workerBenchmark(type: string, options: any): void { }; workerState[i] = state; worker.on('error', reject); - worker.on('exit', () => { + worker.on('exit', (code) => { + if (code !== 0 && !state.teardownError) { + state.teardownError = new Error( + `Benchmark worker ${i + 1} exited with code ${code}` + ); + } state.benchPromise.resolve(); state.teardownPromise.resolve(); state.exitPromise.resolve(); @@ -458,6 +462,9 @@ export function workerBenchmark(type: string, options: any): void { state.benchPromise.resolve(); } else if (event.teardownDone) { state.teardownPromise.resolve(); + } else if (event.teardownError) { + state.teardownError = new Error(event.teardownError); + state.teardownPromise.resolve(); } else if (event.timeout) { state.teardownPromise.resolve(); state.benchPromise.reject(new Error('Benchmark timed out')); @@ -489,13 +496,17 @@ export function workerBenchmark(type: string, options: any): void { return workerState[i].exitPromise.promise; }) ); - try { - rmSync(dbPath, { force: true, recursive: true, maxRetries: 3 }); - } catch (err) { - console.warn(`Benchmark teardown failed to delete db path: ${err}`); + const teardownError = workerState.find((state) => state.teardownError)?.teardownError; + if (!teardownError) { + try { + rmSync(dbPath, { force: true, recursive: true, maxRetries: 3 }); + } catch (err) { + console.warn(`Benchmark teardown failed to delete db path: ${err}`); + } } resolve(); + if (teardownError) throw teardownError; }, } ); @@ -531,7 +542,15 @@ export async function workerInit(): Promise { await teardown(ctx); } if (ctx.db) { - await ctx.db.close(); + try { + await ctx.db.close(); + } catch (error) { + parentPort!.postMessage({ + teardownError: error instanceof Error ? error.message : String(error), + benchmarkWorkerId, + }); + process.exit(1); + } } parentPort!.postMessage({ teardownDone: true, benchmarkWorkerId }); process.exit(0); From 03702869ee5c8d2767a0d1ce698a8fc5e77a07ff Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:26:04 -0600 Subject: [PATCH 32/39] Keep VT fast paths teardown-independent Co-Authored-By: GPT-5 Codex --- AGENTS.md | 5 ++++- src/binding/database/database.cpp | 2 -- src/binding/database/database.h | 10 +++++----- src/binding/database/db_handle.cpp | 2 ++ src/binding/database/db_handle.h | 8 ++++++++ 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2847610e8..7d9ace718 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -312,7 +312,10 @@ sufficient (env teardown does not honor tsfn acquire counts); see `DBHandle::opened()` must report false even while the native DB still exists. Any synchronous N-API path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer - from another env after the in-flight count drains. `DBHandle::close()` itself is cross-env and must + from another env after the in-flight count drains. The VT-only `verifyVersion` / `populateVersion` + fast paths are the exception: `DBHandle::open()` snapshots their immutable per-open VT epoch and + column-family ID so they do not touch teardown-owned native state or register an in-flight operation. + `DBHandle::close()` itself is cross-env and must serialize mutation of its `shared_ptr` members. A close-time flush failure keeps the native DB quarantined so `shutdown()` can retry without losing `disableWAL` writes; an explicit destroy may force teardown because the caller requested deletion. A failed physical destroy leaves a registry diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5e9e0f149..9730d1802 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -1663,7 +1663,6 @@ napi_value Database::GetSync(napi_env env, napi_callback_info info) { napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); - ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { @@ -1700,7 +1699,6 @@ napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { napi_value Database::PopulateVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); - ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { diff --git a/src/binding/database/database.h b/src/binding/database/database.h index 11ee429c2..1d4c4f34a 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -42,11 +42,11 @@ inline std::atomic* vtSlotFor( const rocksdb::Slice& key ) { if (!vt) return nullptr; - // Per-open epoch, not the descriptor pointer: the pointer is reused across a - // close/reopen of the same path while cfId stays stable (HarperFast/harper#1864). - uint64_t dbId = dbHandle->descriptor->vtEpoch; - uint32_t cfId = dbHandle->getColumnFamilyHandle()->GetID(); - return vt->slotFor(dbId, cfId, key); + return vt->slotFor( + dbHandle->verificationTableDbId, + dbHandle->verificationTableColumnFamilyId, + key + ); } /** diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 88a61816f..cc124d244 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -349,6 +349,8 @@ void DBHandle::open(const std::string& path, const DBOptions& options) { auto handleParams = DBRegistry::OpenDB(path, options); this->columnDescriptor = std::move(handleParams->columnDescriptor); this->descriptor = std::move(handleParams->descriptor); + this->verificationTableDbId = this->descriptor->vtEpoch; + this->verificationTableColumnFamilyId = this->columnDescriptor->column->GetID(); this->disableWAL = options.disableWAL; this->enableVerificationTable = options.verificationTable; diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 60ffbf067..7fa31cc91 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -62,6 +62,14 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro */ bool enableVerificationTable = false; + /** + * Immutable VerificationTable address components for this open lifecycle. + * These let VT-only fast paths avoid dereferencing teardown-owned native + * descriptors or registering as in-flight database operations. + */ + uint64_t verificationTableDbId = 0; + uint32_t verificationTableColumnFamilyId = 0; + /** * The node environment. */ From 6251c0023ce7aa9144d0953cf48eda5d866ecb78 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:59:51 -0600 Subject: [PATCH 33/39] Serialize iterators with forced teardown Co-Authored-By: GPT-5 Codex --- AGENTS.md | 4 ++- src/binding/database/database.cpp | 4 +++ src/binding/database/database.h | 29 ------------------- src/binding/database/db_descriptor.h | 25 ++++++++++++++++ src/binding/iterator/db_iterator.cpp | 32 +++++++++++++++++++-- src/binding/iterator/db_iterator_handle.cpp | 13 ++++++--- src/binding/iterator/db_iterator_handle.h | 3 ++ test/destroy.test.ts | 8 ++++++ test/fixtures/fork-destroy-open.mts | 19 +++++++++++- test/workers/destroy-open-worker.mts | 4 ++- 10 files changed, 102 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7d9ace718..1ba82411e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -315,7 +315,9 @@ sufficient (env teardown does not honor tsfn acquire counts); see from another env after the in-flight count drains. The VT-only `verifyVersion` / `populateVersion` fast paths are the exception: `DBHandle::open()` snapshots their immutable per-open VT epoch and column-family ID so they do not touch teardown-owned native state or register an in-flight operation. - `DBHandle::close()` itself is cross-env and must + Async N-API setup must hold the guard until it hands off to `DBHandle::registerAsyncWork()`. Iterators + take the guard through construction/descriptor attachment, then serialize each native iterator call + against foreign forced close with their per-iterator mutex. `DBHandle::close()` itself is cross-env and must serialize mutation of its `shared_ptr` members. A close-time flush failure keeps the native DB quarantined so `shutdown()` can retry without losing `disableWAL` writes; an explicit destroy may force teardown because the caller requested deletion. A failed physical destroy leaves a registry diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 9730d1802..8f02d9a88 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -67,6 +67,7 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) { static napi_value doClear(napi_env env, napi_callback_info info, const char* failureMsg) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); napi_value resolve = argv[0]; napi_value reject = argv[1]; @@ -257,6 +258,7 @@ napi_value Database::Columns(napi_env env, napi_callback_info info) { napi_value Database::Compact(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(5); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); napi_value resolve = argv[0]; napi_value reject = argv[1]; @@ -655,6 +657,7 @@ napi_value Database::FlushSync(napi_env env, napi_callback_info info) { napi_value Database::Flush(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(3); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); napi_value resolve = argv[0]; napi_value reject = argv[1]; @@ -758,6 +761,7 @@ napi_value Database::Get(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(5); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { return nullptr; diff --git a/src/binding/database/database.h b/src/binding/database/database.h index 1d4c4f34a..61449c5eb 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -235,35 +235,6 @@ inline void vtPopulateIfSettled( } \ } while (0) -/** - * RAII guard that tracks in-flight operations on a DBDescriptor. - * Increments counter on construction, decrements on destruction. - * Notifies waiters via atomic::notify_all() when count reaches zero. - */ -struct OperationGuard { - std::shared_ptr descriptor; - - explicit OperationGuard(std::shared_ptr desc) : descriptor(std::move(desc)) { - if (descriptor) { - ++descriptor->operationsInFlight; - } - } - - ~OperationGuard() { - if (descriptor) { - if (--descriptor->operationsInFlight == 0 && descriptor->isClosing()) { - descriptor->operationsInFlight.notify_all(); - } - } - } - - // Non-copyable, non-movable - OperationGuard(const OperationGuard&) = delete; - OperationGuard& operator=(const OperationGuard&) = delete; - OperationGuard(OperationGuard&&) = delete; - OperationGuard& operator=(OperationGuard&&) = delete; -}; - /** * Registers an in-flight operation to prevent use-after-free during shutdown. * Also checks if the database is closing and throws an error if so. diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index e88dc2cad..1bb6faec5 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -591,6 +591,31 @@ struct DBDescriptor final : public std::enable_shared_from_this { ); }; +/** + * Pins a descriptor operation across cross-environment teardown. Callers must + * check `isClosing()` after construction and before touching native DB state. + */ +struct OperationGuard final { + std::shared_ptr descriptor; + + explicit OperationGuard(std::shared_ptr desc) : descriptor(std::move(desc)) { + if (descriptor) { + ++descriptor->operationsInFlight; + } + } + + ~OperationGuard() { + if (descriptor && --descriptor->operationsInFlight == 0 && descriptor->isClosing()) { + descriptor->operationsInFlight.notify_all(); + } + } + + OperationGuard(const OperationGuard&) = delete; + OperationGuard& operator=(const OperationGuard&) = delete; + OperationGuard(OperationGuard&&) = delete; + OperationGuard& operator=(OperationGuard&&) = delete; +}; + /** * State to pass into `napi_call_threadsafe_function()` for a lock callback. */ diff --git a/src/binding/iterator/db_iterator.cpp b/src/binding/iterator/db_iterator.cpp index d2ac626e3..e6576762d 100644 --- a/src/binding/iterator/db_iterator.cpp +++ b/src/binding/iterator/db_iterator.cpp @@ -5,6 +5,7 @@ #include "napi/macros.h" #include "transaction/transaction.h" #include "core/platform.h" +#include "core/test_seam.h" #include "napi/helpers.h" #include "napi/async.h" @@ -140,6 +141,20 @@ napi_value DBIterator::Constructor(napi_env env, napi_callback_info info) { } DEBUG_LOG("DBIterator::Constructor Initializing iterator handle with Database instance (dbHandle=%p)\n", (*dbHandle).get()); } + const int setupDelayMs = testDelayMs("ROCKSDB_JS_ITERATOR_SETUP_DELAY_MS"); + if (setupDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(setupDelayMs)); + } + auto descriptor = (*dbHandle)->descriptor; + if (!descriptor) { + ::napi_throw_error(env, nullptr, "Database not open"); + return nullptr; + } + OperationGuard operationGuard(descriptor); + if (descriptor->isClosing()) { + ::napi_throw_error(env, nullptr, "Database is closing"); + return nullptr; + } // Resolve start/end key pointers from the shared default key buffer char* keyBufferPtr = (*dbHandle)->defaultKeyBufferPtr; @@ -212,7 +227,7 @@ napi_value DBIterator::Constructor(napi_env env, napi_callback_info info) { std::shared_ptr* itHandle = nullptr; \ do { \ NAPI_STATUS_THROWS(::napi_unwrap(env, jsThis, reinterpret_cast(&itHandle))); \ - if (!itHandle || (*itHandle)->iterator == nullptr) { \ + if (!itHandle || !*itHandle) { \ ::napi_throw_error(env, nullptr, fnName " failed: Iterator not initialized"); \ return nullptr; \ } \ @@ -264,6 +279,11 @@ napi_value DBIterator::Next(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Next"); auto& it = *itHandle; + std::lock_guard iteratorLock(it->iteratorMutex); + if (!it->iterator) { + ::napi_throw_error(env, nullptr, "Next failed: Iterator not initialized"); + return nullptr; + } napi_value result; if (!it->iterator->Valid()) { @@ -350,7 +370,10 @@ napi_value DBIterator::Return(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Return"); DEBUG_LOG("%p DBIterator::Return Closing iterator handle\n", (*itHandle).get()); - (*itHandle)->close(); + if (!(*itHandle)->closeIfOpen()) { + ::napi_throw_error(env, nullptr, "Return failed: Iterator not initialized"); + return nullptr; + } NAPI_RETURN_UNDEFINED(); } @@ -364,7 +387,10 @@ napi_value DBIterator::Throw(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Throw"); DEBUG_LOG("%p DBIterator::Throw Closing iterator handle\n", (*itHandle).get()); - (*itHandle)->close(); + if (!(*itHandle)->closeIfOpen()) { + ::napi_throw_error(env, nullptr, "Throw failed: Iterator not initialized"); + return nullptr; + } NAPI_RETURN_UNDEFINED(); } diff --git a/src/binding/iterator/db_iterator_handle.cpp b/src/binding/iterator/db_iterator_handle.cpp index 9ccfc589e..98b646862 100644 --- a/src/binding/iterator/db_iterator_handle.cpp +++ b/src/binding/iterator/db_iterator_handle.cpp @@ -60,15 +60,20 @@ DBIteratorHandle::~DBIteratorHandle() { } void DBIteratorHandle::close() { + this->closeIfOpen(); +} + +bool DBIteratorHandle::closeIfOpen() { + std::lock_guard lock(this->iteratorMutex); DEBUG_LOG("%p DBIteratorHandle::close dbHandle=%p dbDescriptor=%p\n", this, this->dbHandle.get(), this->dbHandle->descriptor.get()); - if (this->iterator) { - this->iterator->Reset(); - this->iterator.reset(); - } + if (!this->iterator) return false; + this->iterator->Reset(); + this->iterator.reset(); if (this->txnHandle) { auto txnHandle = std::move(this->txnHandle); txnHandle->unregisterIterator(); } + return true; } void DBIteratorHandle::init(DBIteratorOptions& options) { diff --git a/src/binding/iterator/db_iterator_handle.h b/src/binding/iterator/db_iterator_handle.h index 06e237275..be0172d1f 100644 --- a/src/binding/iterator/db_iterator_handle.h +++ b/src/binding/iterator/db_iterator_handle.h @@ -1,6 +1,7 @@ #ifndef __DB_ITERATOR_HANDLE_H__ #define __DB_ITERATOR_HANDLE_H__ +#include #include "database/db_handle.h" #include "iterator/db_iterator.h" #include "transaction/transaction_handle.h" @@ -45,6 +46,7 @@ struct DBIteratorHandle final : Closable, public std::enable_shared_from_this { }); }, 15_000); + it('closes an iterator safely when destroy races its construction', async () => { + await runDestroyFixture(destroyOpenFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_DELAY_MS: '2000', + ROCKSDB_JS_ITERATOR_SETUP_DELAY_MS: '250', + ROCKSDB_JS_TEST_ITERATOR_DESTROY_RACE: '1', + }); + }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index 7e6e55574..718dcdd8e 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -10,7 +10,10 @@ original.useLog('cross-env-close'); const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { eval: true, - workerData: { path }, + workerData: { + path, + destroyStartDelayMs: process.env.ROCKSDB_JS_TEST_ITERATOR_DESTROY_RACE === '1' ? 50 : 0, + }, }); function nextMessage(): Promise { @@ -27,6 +30,20 @@ const destroying = await nextMessage(); if (!destroying.destroying) throw new Error(`Destroy worker did not start: ${JSON.stringify(destroying)}`); +if (process.env.ROCKSDB_JS_TEST_ITERATOR_DESTROY_RACE === '1') { + try { + const rows = original.getRange({ limit: 1 }).asArray; + if (rows.length !== 1) throw new Error(`Iterator returned ${rows.length} rows before destroy`); + } catch (error) { + if ( + !String(error).includes('Database not open') && + !String(error).includes('Database is closing') + ) { + throw error; + } + } +} + const registryDeadline = Date.now() + 5_000; while (registryStatus().some((entry) => entry.path === path)) { if (Date.now() >= registryDeadline) throw new Error('Timed out waiting for the destroy window'); diff --git a/test/workers/destroy-open-worker.mts b/test/workers/destroy-open-worker.mts index 7fe2c0585..58d331047 100644 --- a/test/workers/destroy-open-worker.mts +++ b/test/workers/destroy-open-worker.mts @@ -1,4 +1,5 @@ import { RocksDatabase } from '../../src/index.ts'; +import { setTimeout as delay } from 'node:timers/promises'; import { parentPort, workerData } from 'node:worker_threads'; const db = RocksDatabase.open(workerData.path); @@ -6,8 +7,9 @@ if (!parentPort) throw new Error('Destroy/open worker requires a parent port'); const port = parentPort; port.postMessage({ ready: true }); -port.once('message', () => { +port.once('message', async () => { port.postMessage({ destroying: true }); + if (workerData.destroyStartDelayMs > 0) await delay(workerData.destroyStartDelayMs); try { db.destroy(); port.postMessage({ destroyed: true }); From 9a4b6a534a764b5e1de641d1030c2ea200b99491 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 22:28:12 -0600 Subject: [PATCH 34/39] Address remaining lifecycle review findings - shutdown() no longer permanently throws once a destroy-cleanup tombstone exists; it stays non-destructive (per AGENTS.md) and skips the entry instead of poisoning every later call - binding.cpp always releases global listener threadsafe functions, even when DBRegistry::Shutdown() throws - compactSync() cancels its manual compaction when finishClose() is draining in-flight operations, instead of blocking the untimed drain (and cascading OpenDB timeouts) for the compaction's full duration - Iterator Return()/Throw() are idempotent again on an already-closed iterator, matching close() elsewhere, instead of throwing over a clean loop exit or the caller's real error - narrow the AGENTS.md VT fast-path claim to what's actually true - log the retained path on a benchmark teardown failure instead of leaking it silently - add a deterministic test for iteratorMutex serializing Next() against a foreign forced close, plus a return()/throw() idempotency unit test --- AGENTS.md | 10 +++- benchmark/setup.ts | 2 + src/binding/binding.cpp | 4 +- src/binding/database/db_descriptor.cpp | 10 ++++ src/binding/database/db_descriptor.h | 10 ++++ src/binding/database/db_registry.cpp | 23 ++++----- src/binding/iterator/db_iterator.cpp | 26 ++++++---- test/destroy.test.ts | 17 ++++++- test/fixtures/fork-iterator-next-race.mts | 58 +++++++++++++++++++++++ test/ranges.test.ts | 23 +++++++++ 10 files changed, 156 insertions(+), 27 deletions(-) create mode 100644 test/fixtures/fork-iterator-next-race.mts diff --git a/AGENTS.md b/AGENTS.md index 1ba82411e..d3757bf4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -313,8 +313,14 @@ sufficient (env teardown does not honor tsfn acquire counts); see path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer from another env after the in-flight count drains. The VT-only `verifyVersion` / `populateVersion` - fast paths are the exception: `DBHandle::open()` snapshots their immutable per-open VT epoch and - column-family ID so they do not touch teardown-owned native state or register an in-flight operation. + fast paths narrow, but do not remove, that requirement: both still start with + `UNWRAP_DB_HANDLE_AND_OPEN()`, so they still gate on `descriptor`/`isClosing()`. What + `DBHandle::open()`'s snapshotted `verificationTableDbId` / `verificationTableColumnFamilyId` actually + avoids is the `getColumnFamilyHandle()` dereference and the `OperationGuard`'s in-flight + registration — the two things that are unsafe to skip everywhere else. `PutSync`/`RemoveSync`/ + `TransactionHandle` still compute the VT address as `descriptor->vtEpoch` + + `getColumnFamilyHandle()->GetID()` rather than reading the cached fields, so there are two + spellings of the same address computation that must stay in agreement. Async N-API setup must hold the guard until it hands off to `DBHandle::registerAsyncWork()`. Iterators take the guard through construction/descriptor attachment, then serialize each native iterator call against foreign forced close with their per-iterator mutex. `DBHandle::close()` itself is cross-env and must diff --git a/benchmark/setup.ts b/benchmark/setup.ts index b501a28ee..e586ff79b 100644 --- a/benchmark/setup.ts +++ b/benchmark/setup.ts @@ -503,6 +503,8 @@ export function workerBenchmark(type: string, options: any): void { } catch (err) { console.warn(`Benchmark teardown failed to delete db path: ${err}`); } + } else { + console.warn(`Benchmark teardown failed; retaining ${dbPath} for inspection`); } resolve(); diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 9dbc365c1..d5e39f401 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -47,11 +47,13 @@ napi_value Shutdown(napi_env env, napi_callback_info info) { } catch (...) { error = "Unknown native database shutdown failure"; } + // Release global listener threadsafe functions on every path, including a + // failed shutdown -- otherwise they outlive this N-API environment. + GlobalEvents::Shutdown(); if (!error.empty()) { ::napi_throw_error(env, nullptr, error.c_str()); return nullptr; } - GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 0971c98ed..bfe81c3e8 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -392,11 +392,18 @@ void DBDescriptor::finishClose(bool destroying) { // Wait for all in-flight operations to complete before cleanup. // The closing flag is already set, so new operations will fail with "Database is closing". // Existing operations will decrement operationsInFlight and notify us when done. + // A long-running compactRange() holding an OperationGuard is the one + // in-flight op that can run unboundedly, so ask it to cancel rather + // than blocking this untimed wait for its full duration. + this->compactCancelRequested.store(true); DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { this->operationsInFlight.wait(current); } + // Clear it before any further compaction runs below (compact-on-close), + // which must not be cancelled -- nothing external is waiting on it. + this->compactCancelRequested.store(false); DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); // Drain the commit pipeline before flushing so its data is included in @@ -2147,6 +2154,9 @@ rocksdb::Status DBDescriptor::compactRange( std::lock_guard lock(this->compactMutex); DEBUG_LOG("%p DBDescriptor::compactRange Compacting range (bottommost=%d)\n", this, bottommost); rocksdb::CompactRangeOptions options; + // Let a concurrent finishClose() interrupt this compaction rather than + // wait out its full, unbounded duration; see compactCancelRequested. + options.canceled = &this->compactCancelRequested; if (bottommost) { // RocksDB defaults this to kIfHaveCompactionFilter, so with no compaction filter installed // the bottommost level is skipped — and that is where the bulk of the data sits. Rewriting diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 1bb6faec5..968071591 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -292,6 +292,16 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool closeWorkersStopped = false; bool transactionLogsUnregistered = false; + /** + * Set by finishClose() only while it is draining operationsInFlight, so an + * OperationGuard-holding compactRange() in progress on another thread can + * cancel its manual compaction and release the guard promptly instead of + * blocking the untimed drain wait for the compaction's full duration. + * Cleared once the drain completes so the close-time "compact on close" + * pass below always runs to completion. + */ + std::atomic compactCancelRequested{false}; + /** * Counter tracking in-flight database operations. close() uses * atomic::wait() to block until this reaches zero. diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index bc13de015..2d284b030 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -1016,7 +1016,6 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; - std::string destroyCleanupError; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -1030,11 +1029,13 @@ void DBRegistry::Shutdown() { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty() && destroyCleanupError.empty()) { - destroyCleanupError = - "Cannot complete shutdown: database \"" + key.path + - "\" requires explicit destroy() cleanup: " + entry.closeError; - } + // A prior destroy() left a tombstone (descriptor cleared, + // closeError set) after its physical cleanup failed. That + // failure was already surfaced via database:closeFailed and + // stays visible in registryStatus().destroyCleanupPending. + // shutdown() is deliberately non-destructive -- only an + // explicit destroy() retries path deletion -- so skip it + // here rather than re-throwing the same error forever. continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; @@ -1101,18 +1102,10 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } if (closeError) std::rethrow_exception(closeError); - if (!destroyCleanupError.empty()) { - throw rocksdb_js::DBException(destroyCleanupError); - } continue; } if (closeError) std::rethrow_exception(closeError); - if (!destroyCleanupError.empty()) { - throw rocksdb_js::DBException(destroyCleanupError); - } - if (descriptorsToClose.empty() && descriptorsToWaitFor.empty() && - destroyCleanupError.empty() - ) break; + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) break; } // Purge the registry diff --git a/src/binding/iterator/db_iterator.cpp b/src/binding/iterator/db_iterator.cpp index e6576762d..b800dc6c5 100644 --- a/src/binding/iterator/db_iterator.cpp +++ b/src/binding/iterator/db_iterator.cpp @@ -8,6 +8,8 @@ #include "core/test_seam.h" #include "napi/helpers.h" #include "napi/async.h" +#include +#include namespace rocksdb_js { @@ -280,6 +282,14 @@ napi_value DBIterator::Next(napi_env env, napi_callback_info info) { auto& it = *itHandle; std::lock_guard iteratorLock(it->iteratorMutex); + // Test-only: widen the window where a foreign finishClose()'s closables + // sweep is blocked on iteratorMutex behind this call, so a fixture can + // reliably position a forced close mid-Next() rather than only ever + // between calls. + const int nextDelayMs = testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"); + if (nextDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(nextDelayMs)); + } if (!it->iterator) { ::napi_throw_error(env, nullptr, "Next failed: Iterator not initialized"); return nullptr; @@ -370,10 +380,11 @@ napi_value DBIterator::Return(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Return"); DEBUG_LOG("%p DBIterator::Return Closing iterator handle\n", (*itHandle).get()); - if (!(*itHandle)->closeIfOpen()) { - ::napi_throw_error(env, nullptr, "Return failed: Iterator not initialized"); - return nullptr; - } + // Idempotent by design: a foreign forced close (finishClose()'s closables + // sweep) or an earlier limit-triggered return() may have already closed + // this iterator, and a clean loop exit / a second explicit return() must + // not turn into a thrown error. + (*itHandle)->close(); NAPI_RETURN_UNDEFINED(); } @@ -387,10 +398,9 @@ napi_value DBIterator::Throw(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Throw"); DEBUG_LOG("%p DBIterator::Throw Closing iterator handle\n", (*itHandle).get()); - if (!(*itHandle)->closeIfOpen()) { - ::napi_throw_error(env, nullptr, "Throw failed: Iterator not initialized"); - return nullptr; - } + // Idempotent for the same reason as Return above -- must not replace the + // caller's real thrown error with a spurious native one. + (*itHandle)->close(); NAPI_RETURN_UNDEFINED(); } diff --git a/test/destroy.test.ts b/test/destroy.test.ts index aa8815bc8..4a04ab830 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -13,6 +13,7 @@ const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failur const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); +const iteratorNextRaceFixture = join(__dirname, 'fixtures', 'fork-iterator-next-race.mts'); const nodeExecutable = process.env.NODE_BINARY ?? (process.versions.bun || process.versions.deno @@ -163,7 +164,11 @@ describe('Destroy', () => { expect.stringContaining('Failed to remove database directory'), ]); chmodSync(lockedDirectory, 0o700); - expect(() => shutdown()).toThrow('requires explicit destroy() cleanup'); + // shutdown() is deliberately non-destructive: it must not retry + // path deletion (only an explicit destroy() call may), so a + // pending tombstone does not make it throw, and it does not + // clear the tombstone even though the underlying cause is fixed. + shutdown(); expect(existsSync(dbPath)).toBe(true); expect( registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending @@ -196,6 +201,16 @@ describe('Destroy', () => { }); }, 15_000); + it('serializes an in-progress Next() against a foreign forced close', async () => { + // Unlike the constructor race above, this positions the destroy while + // a Next() call already holds iteratorMutex, so it must block on the + // mutex rather than racing it -- the actual case iteratorMutex exists + // for. See fork-iterator-next-race.mts. + await runDestroyFixture(iteratorNextRaceFixture, generateDBPath(), { + ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS: '250', + }); + }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-iterator-next-race.mts b/test/fixtures/fork-iterator-next-race.mts new file mode 100644 index 000000000..8422e9b46 --- /dev/null +++ b/test/fixtures/fork-iterator-next-race.mts @@ -0,0 +1,58 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const original = RocksDatabase.open(path); +original.putSync('a', '1'); +original.putSync('b', '2'); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 50 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +const iterator = original.getRange({})[Symbol.iterator](); +const first = iterator.next(); +if (first.done) throw new Error('Expected a first row before the destroy race'); + +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +// Holds iteratorMutex for the ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS test seam +// while the worker's destroy() -- ticking on its own 50ms delay -- reaches +// finishClose()'s closables sweep and blocks on the same mutex behind this +// call. This is the case iteratorMutex exists for: a foreign forced close +// racing an in-progress Next(), not just one racing the constructor. +const second = iterator.next(); +if (second.done) throw new Error('Expected a second row before destroy claimed the iterator'); + +const destroyResult = await nextMessage(); +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); + +if (registryStatus().some((entry) => entry.path === path)) { + throw new Error('Expected destroy to fully clear the registry entry'); +} + +// The mutex handoff must leave the iterator cleanly (not torn/crashing) +// closed once finishClose() gets its turn. +try { + iterator.next(); + throw new Error('Expected Next to fail once destroy closed the iterator'); +} catch (error) { + if (!String(error).includes('Iterator not initialized')) throw error; +} + +await worker.terminate(); diff --git a/test/ranges.test.ts b/test/ranges.test.ts index 1abde0857..b3a384f5d 100644 --- a/test/ranges.test.ts +++ b/test/ranges.test.ts @@ -165,6 +165,29 @@ describe('Ranges', () => { } })); + it('is idempotent when return()/throw() are called on an already-closed iterator', () => + dbRunner(async ({ db }) => { + for (const key of ['a', 'b']) { + await db.put(key, `value ${key}`); + } + + const iter = db.getRange()[Symbol.iterator](); + iter.next(); + iter.return!(); + // A second return() (e.g. a `finally` block after an earlier + // `break`) must stay a no-op, not throw over the already-closed + // native iterator. + expect(() => iter.return!()).not.toThrow(); + + const limited = db.getRange({ limit: 1 })[Symbol.iterator](); + limited.next(); // yields the one row within the limit + limited.next(); // over the limit: auto-closes the native iterator + // The native iterator is already closed at this point; the + // caller's own thrown error must survive, not get replaced by a + // native "Iterator not initialized" error. + expect(() => limited.throw!(new Error('caller error'))).toThrow('caller error'); + })); + it('should get iterate in reverse', () => dbRunner(async ({ db }) => { for (const key of ['a', 'b', 'c', 'd', 'e']) { From ba405dc2997b765405db956a1a2916d03e9d44d5 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 24 Aug 2026 15:49:48 -0600 Subject: [PATCH 35/39] Address remaining lifecycle review threads - DBIterator::Next() no longer pays a getenv() scan per row for the ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS seam; it is snapshotted once in initializeTestSeams() alongside the close-failure flags. Next() returns one row per call, so this was ~a quarter of the per-row getRange cost for a seam that is unset in production. - Extract closeClaimedDescriptors() in db_registry.cpp: the finishClose() -> erase-or-quarantine -> notify -> emit tail was copied four times (PurgeIfUnreferenced, DestroyDB, PurgeAll, Shutdown), each handling closeError/closeRetrying slightly differently. Only the claim predicate genuinely differs per caller, so that is all that is left at the call sites. The completed-but-errored policy that had drifted is now one named option: fatal for shutdown()/PurgeAll() because dropping a failed close-time flush would hide possible data loss, non-fatal for destroy(), whose caller asked for the data to be deleted anyway. - getKeysCount() was the remaining unbounded OperationGuard holder that finishClose()'s untimed drain could not cancel. The scan now polls isClosing() per row and reports the abort instead of a partial count, on both the database and transaction paths, so a foreign destroy() is no longer blocked for the length of the range (and concurrent OpenDB() calls for that path no longer time out behind it). The comment claiming compaction was the only unbounded in-flight op is corrected. Co-Authored-By: Claude Opus --- AGENTS.md | 20 ++ src/binding/core/test_seam.h | 19 ++ src/binding/database/database.cpp | 11 +- src/binding/database/db_descriptor.cpp | 7 +- src/binding/database/db_registry.cpp | 219 ++++++++---------- src/binding/iterator/db_iterator.cpp | 2 +- src/binding/iterator/db_iterator_handle.cpp | 21 ++ src/binding/iterator/db_iterator_handle.h | 10 + src/binding/transaction/transaction.cpp | 5 +- .../transaction/transaction_handle.cpp | 6 +- src/binding/transaction/transaction_handle.h | 4 +- test/destroy.test.ts | 9 + test/fixtures/fork-count-destroy-race.mts | 59 +++++ 13 files changed, 251 insertions(+), 141 deletions(-) create mode 100644 test/fixtures/fork-count-destroy-race.mts diff --git a/AGENTS.md b/AGENTS.md index d3757bf4c..b04b83a22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -212,6 +212,11 @@ sufficient (env teardown does not honor tsfn acquire counts); see legitimately slow holder raises the value instead - `ROCKSDB_JS_DESTROY_DELAY_MS` - Test-only: delay after descriptor teardown and before physical database destruction (widens same-path reopen races) +- `ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS` / `ROCKSDB_JS_COUNT_DELAY_MS` - Test-only: per-row delays in + `DBIterator::Next()` and `DBIteratorHandle::countRemaining()`. Both are read **once** in + `initializeTestSeams()` rather than per row: these are the two per-row native loops, and a + `getenv()` scan per row is a measurable share of their cost for a seam unset in production. Add + new per-row seams the same way. ## Test Structure @@ -329,6 +334,21 @@ sufficient (env teardown does not honor tsfn acquire counts); see force teardown because the caller requested deletion. A failed physical destroy leaves a registry tombstone, but `shutdown()` is deliberately non-destructive: it reports the tombstone and only an explicit `destroy()` retries path deletion. + Because `finishClose()` drains `operationsInFlight` with an **untimed** wait, any operation that can + run unboundedly while holding an `OperationGuard` must abort itself once `closing` is published, or + it blocks teardown — and, since the blocked closer holds the path gate, times out every concurrent + `OpenDB()` for that path. There are two such operations and they cancel differently: the whole-range + count scan (`DBIteratorHandle::countRemaining`, behind `getKeysCount()` on both the database and + transaction paths) polls `isClosing()` per row and reports the abort to its caller rather than a + partial count; a manual `compactRange()` cannot poll from inside RocksDB, so it gets an explicit + cancel token (`DBDescriptor::compactCancelRequested` → `CompactRangeOptions::canceled`). + Everything the four registry teardown paths do _after_ claiming a descriptor — + `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, + `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its + `failOnCompletedWithError` option is the one deliberate asymmetry: a close that finished native + teardown but reported an error (a failed close-time flush) is fatal for `shutdown()`/`PurgeAll()` + because dropping it silently would hide possible data loss, and non-fatal for `destroy()`, whose + caller asked for the data to be deleted anyway. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index effdcd7ee..d2c922097 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -29,6 +29,21 @@ inline std::atomic& closeFlushFailureFlag() { return pending; } +// DBIterator::Next() returns one row per call, so its seam is snapshotted here +// rather than read per call: a getenv() scan per row is a measurable share of +// the per-row cost for a seam that is unset in production. +inline std::atomic& iteratorNextDelayMsFlag() { + static std::atomic delayMs{0}; + return delayMs; +} + +// Per-row delay for DBIteratorHandle::countRemaining(), snapshotted for the +// same reason. +inline std::atomic& countScanDelayMsFlag() { + static std::atomic delayMs{0}; + return delayMs; +} + inline void initializeTestSeams() { static std::once_flag initialized; std::call_once(initialized, []() { @@ -36,6 +51,10 @@ inline void initializeTestSeams() { closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); value = ::getenv("ROCKSDB_JS_CLOSE_FLUSH_FAILURE"); closeFlushFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + iteratorNextDelayMsFlag().store( + testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"), std::memory_order_relaxed); + countScanDelayMsFlag().store( + testDelayMs("ROCKSDB_JS_COUNT_DELAY_MS"), std::memory_order_relaxed); }); } diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 8f02d9a88..19c9f7fbe 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -1103,12 +1103,15 @@ napi_value Database::GetCount(napi_env env, napi_callback_info info) { ::napi_throw_error(env, nullptr, errorMsg.c_str()); NAPI_RETURN_UNDEFINED(); } - txnHandle->getCount(itOptions, count, *dbHandle); + if (!txnHandle->getCount(itOptions, count, *dbHandle)) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); + } } else { std::unique_ptr itHandle = std::make_unique(*dbHandle, itOptions); - while (itHandle->iterator->Valid()) { - ++count; - itHandle->iterator->Next(); + if (!itHandle->countRemaining(count)) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); } } diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index bfe81c3e8..02e559deb 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -392,9 +392,10 @@ void DBDescriptor::finishClose(bool destroying) { // Wait for all in-flight operations to complete before cleanup. // The closing flag is already set, so new operations will fail with "Database is closing". // Existing operations will decrement operationsInFlight and notify us when done. - // A long-running compactRange() holding an OperationGuard is the one - // in-flight op that can run unboundedly, so ask it to cancel rather - // than blocking this untimed wait for its full duration. + // Unbounded in-flight operations must abort once `closing` is published + // rather than block this untimed wait for their full duration. A count + // scan polls isClosing() itself; a manual compactRange() cannot, so it + // gets an explicit cancel token. this->compactCancelRequested.store(true); DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 2d284b030..4a69d355f 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -101,6 +101,76 @@ void emitCloseFailures(const std::vector& descriptors) { } } +struct ClaimedCloseOptions final { + // destroy() is deleting the data, so it forces teardown and does not treat a + // close that finished but reported an error (a failed close-time flush) as + // fatal. Every other caller does: dropping that error silently would hide + // possible data loss. The entry is erased either way, so the failure is + // reported once rather than wedging the path. + bool destroying = false; + bool failOnCompletedWithError = true; +}; + +/** + * Runs finishClose() over descriptors already claimed by the caller, then + * erases each entry or quarantines it with its close error, notifies that + * path's waiters, and emits `database:closeFailed`. + * + * Returns the first exception the caller should rethrow, or null. Claiming + * differs per caller (one path, every path, or a single unreferenced + * descriptor); everything after the claim is this one policy. + */ +std::exception_ptr closeClaimedDescriptors( + std::vector& claimed, + const ClaimedCloseOptions& options, + std::unordered_map& databases, + std::mutex& databasesMutex +) { + std::exception_ptr closeError; + + for (auto& closing : claimed) { + DEBUG_LOG("DBRegistry::closeClaimedDescriptors Closing descriptor %p for \"%s\" (ref count = %ld)\n", + closing.descriptor.get(), closing.key.path.c_str(), closing.descriptor.use_count()); + + std::exception_ptr thrown; + try { + closing.descriptor->finishClose(options.destroying); + closing.closed = true; + } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = error.what(); + thrown = std::current_exception(); + } catch (...) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = "unknown native close failure"; + thrown = std::current_exception(); + } + + if (thrown && !closeError && (!closing.closed || options.failOnCompletedWithError)) { + closeError = thrown; + } + + { + std::lock_guard lock(databasesMutex); + auto entry = databases.find(closing.key); + if (entry != databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { + databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; + DEBUG_LOG("DBRegistry::closeClaimedDescriptors Quarantined \"%s\": %s\n", + closing.key.path.c_str(), closing.closeError.c_str()); + } + } + } + closing.condition->notify_all(); + } + + emitCloseFailures(claimed); + return closeError; +} + } // namespace // Initialize the static instance @@ -200,35 +270,17 @@ CloseResult DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (descriptor) { // We claimed the close under the lock via beginClose(); run the actual // teardown now. The local copy keeps the descriptor alive throughout. - try { - descriptor->finishClose(); - } catch (const std::exception& error) { - closeError = error.what(); - } catch (...) { - closeError = "unknown native close failure"; - } - - std::lock_guard lock(instance->databasesMutex); - auto eraseIt = instance->databases.find(key); - // Only erase the entry we claimed. A brand-new descriptor cannot appear - // because OpenDB blocks until we notify below. - if (eraseIt != instance->databases.end() && eraseIt->second.descriptor == descriptor) { - if (closeError.empty() || descriptor->isClosed()) { - instance->databases.erase(eraseIt); - } else { - eraseIt->second.closeError = closeError; - quarantined = true; - DEBUG_LOG("%p DBRegistry::PurgeIfUnreferenced Quarantined \"%s\": %s\n", - instance.get(), path.c_str(), closeError.c_str()); - } - } + // Only the entry we claimed is erased -- a brand-new descriptor cannot + // appear because OpenDB blocks until the helper notifies. + std::vector claimed; + claimed.emplace_back(key, descriptor, condition); + // The close error is reported through CloseResult, not thrown. + closeClaimedDescriptors( + claimed, ClaimedCloseOptions{}, instance->databases, instance->databasesMutex); + closeError = claimed.front().closeError; + quarantined = !closeError.empty() && !claimed.front().closed; } - // notify only waiters for this specific path - if (condition) { - condition->notify_all(); - } - emitCloseFailure(path, closeError); return CloseResult{closeError, quarantined}; } @@ -300,49 +352,15 @@ void DBRegistry::DestroyDB(const std::string& path) { } } - // Keep entries discoverable while finishClose runs: env cleanup uses the - // registry to remove callbacks owned by a worker that exits mid-close. - std::exception_ptr closeError; - for (auto& closing : claimed) { - DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", - instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); - try { - closing.descriptor->finishClose(true); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closing.closed && !closeError) { - closeError = std::current_exception(); - } - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closing.closed && !closeError) { - closeError = std::current_exception(); - } - } - } - - { - std::lock_guard lock(instance->databasesMutex); - for (const auto& closing : claimed) { - auto entry = instance->databases.find(closing.key); - if (entry == instance->databases.end() || entry->second.descriptor != closing.descriptor) { - continue; - } - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - entry->second.closeRetrying = false; - } - } - } - for (const auto& closing : claimed) { - closing.condition->notify_all(); - } - emitCloseFailures(claimed); + // Each entry stays discoverable until its own close finishes: env cleanup + // uses the registry to remove callbacks owned by a worker that exits + // mid-close. + std::exception_ptr closeError = closeClaimedDescriptors( + claimed, + ClaimedCloseOptions{.destroying = true, .failOnCompletedWithError = false}, + instance->databases, + instance->databasesMutex + ); if (alreadyClosing.empty()) { if (closeError) std::rethrow_exception(closeError); break; @@ -713,33 +731,8 @@ void DBRegistry::PurgeAll() { condition->notify_all(); } - for (auto& closing : descriptorsToClose) { - try { - closing.descriptor->finishClose(); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closeError) closeError = std::current_exception(); - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closeError) closeError = std::current_exception(); - } - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - } - } - } - closing.condition->notify_all(); - } - emitCloseFailures(descriptorsToClose); + closeError = closeClaimedDescriptors( + descriptorsToClose, ClaimedCloseOptions{}, instance->databases, instance->databasesMutex); if (closeError) { std::rethrow_exception(closeError); } @@ -1051,36 +1044,8 @@ void DBRegistry::Shutdown() { } } - std::exception_ptr closeError; - for (auto& closing : descriptorsToClose) { - DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); - try { - closing.descriptor->finishClose(); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closeError) closeError = std::current_exception(); - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closeError) closeError = std::current_exception(); - } - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - entry->second.closeRetrying = false; - } - } - } - closing.condition->notify_all(); - } - emitCloseFailures(descriptorsToClose); + std::exception_ptr closeError = closeClaimedDescriptors( + descriptorsToClose, ClaimedCloseOptions{}, instance->databases, instance->databasesMutex); for (const auto& closing : descriptorsToWaitFor) { std::unique_lock lock(instance->databasesMutex); diff --git a/src/binding/iterator/db_iterator.cpp b/src/binding/iterator/db_iterator.cpp index b800dc6c5..1728298b0 100644 --- a/src/binding/iterator/db_iterator.cpp +++ b/src/binding/iterator/db_iterator.cpp @@ -286,7 +286,7 @@ napi_value DBIterator::Next(napi_env env, napi_callback_info info) { // sweep is blocked on iteratorMutex behind this call, so a fixture can // reliably position a forced close mid-Next() rather than only ever // between calls. - const int nextDelayMs = testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"); + const int nextDelayMs = iteratorNextDelayMsFlag().load(std::memory_order_relaxed); if (nextDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(nextDelayMs)); } diff --git a/src/binding/iterator/db_iterator_handle.cpp b/src/binding/iterator/db_iterator_handle.cpp index 98b646862..9daecee2b 100644 --- a/src/binding/iterator/db_iterator_handle.cpp +++ b/src/binding/iterator/db_iterator_handle.cpp @@ -1,5 +1,7 @@ #include "iterator/db_iterator_handle.h" #include "database/db_descriptor.h" +#include "core/test_seam.h" +#include #include namespace rocksdb_js { @@ -103,6 +105,25 @@ void DBIteratorHandle::init(DBIteratorOptions& options) { } } +bool DBIteratorHandle::countRemaining(uint64_t& count) { + const DBDescriptor* descriptor = this->dbHandle->descriptor.get(); + // Test-only: stretch the scan so a fixture can land a foreign destroy() + // inside it rather than only before or after. + const int rowDelayMs = countScanDelayMsFlag().load(std::memory_order_relaxed); + count = 0; + while (this->iterator->Valid()) { + if (descriptor->isClosing()) { + return false; + } + if (rowDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(rowDelayMs)); + } + ++count; + this->iterator->Next(); + } + return true; +} + void DBIteratorHandle::seek(DBIteratorOptions& options) { if (options.reverse) { this->iterator->SeekToLast(); diff --git a/src/binding/iterator/db_iterator_handle.h b/src/binding/iterator/db_iterator_handle.h index be0172d1f..45a4e8a20 100644 --- a/src/binding/iterator/db_iterator_handle.h +++ b/src/binding/iterator/db_iterator_handle.h @@ -54,6 +54,16 @@ struct DBIteratorHandle final : Closable, public std::enable_shared_from_this dbHandle; std::shared_ptr txnHandle; bool exclusiveStart; diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index 3b2d6e158..ef98174ab 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -1055,7 +1055,10 @@ napi_value Transaction::GetCount(napi_env env, napi_callback_info info) { itOptions.values = false; uint64_t count = 0; - (*txnHandle)->getCount(itOptions, count); + if (!(*txnHandle)->getCount(itOptions, count)) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); + } napi_value result; NAPI_STATUS_THROWS(::napi_create_int64(env, count, &result)); diff --git a/src/binding/transaction/transaction_handle.cpp b/src/binding/transaction/transaction_handle.cpp index a0e346130..84637a739 100644 --- a/src/binding/transaction/transaction_handle.cpp +++ b/src/binding/transaction/transaction_handle.cpp @@ -590,7 +590,7 @@ napi_value TransactionHandle::get( return returnStatus; } -void TransactionHandle::getCount( +bool TransactionHandle::getCount( DBIteratorOptions& itOptions, uint64_t& count, std::shared_ptr dbHandleOverride @@ -602,9 +602,7 @@ void TransactionHandle::getCount( std::unique_ptr itHandle = std::make_unique(this->shared_from_this(), itOptions, dbHandleOverride); - for (count = 0; itHandle->iterator->Valid(); ++count) { - itHandle->iterator->Next(); - } + return itHandle->countRemaining(count); } /** diff --git a/src/binding/transaction/transaction_handle.h b/src/binding/transaction/transaction_handle.h index 85c01e33a..9ac8b0dd8 100644 --- a/src/binding/transaction/transaction_handle.h +++ b/src/binding/transaction/transaction_handle.h @@ -232,8 +232,10 @@ struct TransactionHandle final : Closable, AsyncWorkHandle, std::enable_shared_f * @param dbHandleOverride - Database handle override to use instead of the * transaction's database handle when called via the `NativeDatabase` with * the `transaction` property set. + * @returns False when the descriptor began closing mid-scan; see + * `DBIteratorHandle::countRemaining()`. */ - void getCount( + [[nodiscard]] bool getCount( DBIteratorOptions& itOptions, uint64_t& count, std::shared_ptr dbHandleOverride = nullptr diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 4a04ab830..0a22e1107 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -14,6 +14,7 @@ const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mt const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); const iteratorNextRaceFixture = join(__dirname, 'fixtures', 'fork-iterator-next-race.mts'); +const countDestroyRaceFixture = join(__dirname, 'fixtures', 'fork-count-destroy-race.mts'); const nodeExecutable = process.env.NODE_BINARY ?? (process.versions.bun || process.versions.deno @@ -211,6 +212,14 @@ describe('Destroy', () => { }); }, 15_000); + it('aborts an in-flight getCount() when a foreign destroy begins', async () => { + // getCount() scans the whole range under one OperationGuard, which + // finishClose() drains with an untimed wait. See fork-count-destroy-race.mts. + await runDestroyFixture(countDestroyRaceFixture, generateDBPath(), { + ROCKSDB_JS_COUNT_DELAY_MS: '50', + }); + }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-count-destroy-race.mts b/test/fixtures/fork-count-destroy-race.mts new file mode 100644 index 000000000..c6c36d471 --- /dev/null +++ b/test/fixtures/fork-count-destroy-race.mts @@ -0,0 +1,59 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const original = RocksDatabase.open(path); +for (let i = 0; i < 20; i++) { + original.putSync(`key-${i}`, i); +} + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 50 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +// 20 rows x ROCKSDB_JS_COUNT_DELAY_MS holds an OperationGuard well past the +// worker's 50ms destroy delay. finishClose() drains in-flight operations with +// an untimed wait, so without the isClosing() poll in countRemaining() this +// scan would run to completion, block the destroy for its full duration, and +// report a count of a database that is being deleted. +const started = Date.now(); +let countError: unknown; +try { + original.getKeysCount({}); +} catch (error) { + countError = error; +} +const elapsed = Date.now() - started; + +if (!countError) + throw new Error(`Expected getKeysCount to abort, but it returned after ${elapsed}ms`); +if (!String(countError).includes('Database is closing')) throw countError; +if (elapsed >= 1000) { + throw new Error(`getKeysCount ran ${elapsed}ms; it should abort rather than finish the scan`); +} + +const destroyResult = await nextMessage(); +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); + +if (registryStatus().some((entry) => entry.path === path)) { + throw new Error('Expected destroy to fully clear the registry entry'); +} + +await worker.terminate(); From 352f9ee9b63aae367da91e1ac9e2eaa07937e2fe Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 05:38:45 -0600 Subject: [PATCH 36/39] Keep close-time compaction cancellable through the full drain, guard GetCount against concurrent close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit finishClose() cleared compactCancelRequested right after the operationsInFlight drain, but an async compact() releases its OperationGuard at setup handoff and is not awaited until the closables sweep — so it can still be running after the drain returns, and clearing the token there left it able to stall teardown (and every concurrent open on the path) indefinitely. Keep the token armed for finishClose()'s whole duration instead, and have the close-time compact-on-close pass opt out via a new compactRange() `cancellable` param rather than relying on the shared flag being cleared. Transaction::GetCount now takes an OperationGuard and checks isClosing() before scanning: without it, finishClose()'s drain can return immediately and the closables sweep can roll back the transaction while the count scan is parked between rows, reading freed memory. Carries the in-progress PR #787 lifecycle repair plan describing the fuller atomic-admission fix these two changes are a first slice of. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013CvCKcGKG7vyhRts1Mv4Wh --- .pr787-lifecycle-repair-plan.md | 31 +++++++++++++++++++++++++ src/binding/database/db_descriptor.cpp | 16 +++++++------ src/binding/database/db_descriptor.h | 20 ++++++++++------ src/binding/transaction/transaction.cpp | 14 +++++++++++ 4 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 .pr787-lifecycle-repair-plan.md diff --git a/.pr787-lifecycle-repair-plan.md b/.pr787-lifecycle-repair-plan.md new file mode 100644 index 000000000..58ed001fe --- /dev/null +++ b/.pr787-lifecycle-repair-plan.md @@ -0,0 +1,31 @@ +# PR 787 lifecycle repair plan + +## Root cause and invariant + +Invariant: once descriptor close or handle cancellation begins, no new native operation may be admitted; teardown must not release native state until every operation admitted before that transition has drained. + +Current admission violates this invariant twice. `OperationGuard` increments `operationsInFlight` without serializing against `DBDescriptor::beginClose()`, so close can observe zero before a late increment. `AsyncWorkHandle::registerAsyncWork()` likewise increments without serializing against cancellation, and `waitForAsyncWorkCompletion()` continues after five seconds even if work remains. + +## Chosen repair + +1. Add a descriptor operation-admission mutex. `beginClose()` takes it while publishing `closing`; `OperationGuard` takes it while checking `closing` and incrementing `operationsInFlight`, and exposes whether admission succeeded. Existing callers stop touching native state when admission fails. +2. Make async-work admission atomic with cancellation under `AsyncWorkHandle::waitMutex`. Admission returns false after cancellation. Cancellation publishes under that mutex. Waiting becomes predicate-based and does not continue teardown while work remains. +3. Use a shared scoped async registration helper at N-API setup boundaries. It refuses admission after cancellation and transfers an admitted registration to queued state where needed. +4. Register transaction and iterator setup against their transaction handle, database handle, and descriptor before dereferencing teardown-owned state. Keep descriptor-wide operations pinned only for the duration in which native state is directly used; queued async state remains protected by the handle close drain. +5. Add deterministic setup/next race seams and subprocess tests covering database and transaction iterators, repeated idempotent iterator return, and forced destroy during `Next()`. +6. Address the remaining review feedback mechanically: retry destroy tombstone cleanup during shutdown, narrow the VT invariant text/use cached VT coordinates consistently, and diagnose retained benchmark data. + +## Approaches considered + +- Different layer — serialize only JavaScript `destroy()`/`open()` calls. Rejected: worker environments have separate JavaScript heaps, while the violated registry and native handles are process-global; a JS mutex cannot own the invariant across envs. +- Deeper cause — remove foreign forced close and require every environment to cooperatively close. Rejected: `destroy()` must invalidate all handles to the physical path, including handles in environments that may no longer service messages; cooperative teardown cannot guarantee physical deletion safety. +- Do less — retain the current per-call-site checks and increase/remove the five-second timeout. Rejected: timeout changes do not close either admission race, and a late registration can still begin after the waiter observed zero. +- Chosen — enforce atomic admission in the shared descriptor and handle primitives, then keep call sites responsible only for transferring valid claims. This is the only option that makes the close transition and every admission mutually ordered across worker threads without changing the public API. + +## Verification + +- Build the native debug binding first. +- Run focused destroy/iterator/transaction lifecycle tests repeatedly, including parallel subprocess batches. +- Run `pnpm check`, `pnpm test`, and `pnpm test:native` as the repository full gates. +- End-to-end route: the Node subprocess integration fixture uses real worker threads, a shared native descriptor, real RocksDB iterators/transactions, and physical destroy/reopen. +- Bug-proof route: the new next-race test must fail when the per-iterator mutex is removed; the admission-race tests must fail when admission/cancellation serialization is removed. diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 02e559deb..ed2e4acb5 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -395,16 +395,15 @@ void DBDescriptor::finishClose(bool destroying) { // Unbounded in-flight operations must abort once `closing` is published // rather than block this untimed wait for their full duration. A count // scan polls isClosing() itself; a manual compactRange() cannot, so it - // gets an explicit cancel token. + // gets an explicit cancel token. The token stays armed past this drain: + // an async compact() released its OperationGuard at setup handoff, so it + // is still running here and is not awaited until the closables sweep. this->compactCancelRequested.store(true); DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { this->operationsInFlight.wait(current); } - // Clear it before any further compaction runs below (compact-on-close), - // which must not be cancelled -- nothing external is waiting on it. - this->compactCancelRequested.store(false); DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); // Drain the commit pipeline before flushing so its data is included in @@ -471,7 +470,7 @@ void DBDescriptor::finishClose(bool destroying) { } for (const auto& columnDesc : pinnedColumns) { if (columnDesc && columnDesc->column) { - this->compactRange(columnDesc->column.get(), nullptr, nullptr); + this->compactRange(columnDesc->column.get(), nullptr, nullptr, false, false); } } } @@ -2150,14 +2149,17 @@ rocksdb::Status DBDescriptor::compactRange( rocksdb::ColumnFamilyHandle* column, const rocksdb::Slice* start, const rocksdb::Slice* end, - bool bottommost + bool bottommost, + bool cancellable ) { std::lock_guard lock(this->compactMutex); DEBUG_LOG("%p DBDescriptor::compactRange Compacting range (bottommost=%d)\n", this, bottommost); rocksdb::CompactRangeOptions options; // Let a concurrent finishClose() interrupt this compaction rather than // wait out its full, unbounded duration; see compactCancelRequested. - options.canceled = &this->compactCancelRequested; + if (cancellable) { + options.canceled = &this->compactCancelRequested; + } if (bottommost) { // RocksDB defaults this to kIfHaveCompactionFilter, so with no compaction filter installed // the bottommost level is skipped — and that is where the bulk of the data sits. Rewriting diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 968071591..61d2ff9d8 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -293,12 +293,15 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool transactionLogsUnregistered = false; /** - * Set by finishClose() only while it is draining operationsInFlight, so an - * OperationGuard-holding compactRange() in progress on another thread can - * cancel its manual compaction and release the guard promptly instead of - * blocking the untimed drain wait for the compaction's full duration. - * Cleared once the drain completes so the close-time "compact on close" - * pass below always runs to completion. + * Armed by finishClose() for its whole duration so an in-progress manual + * compactRange() on another thread aborts instead of making teardown wait + * out its full, unbounded duration. It covers both shapes: compactSync() + * holds an OperationGuard and so blocks the drain, while an async compact() + * released its guard at setup handoff and is not awaited until the closables + * sweep -- clearing the token after the drain left that second one able to + * stall teardown (and every concurrent open on the path) indefinitely. + * Close-initiated compaction passes `cancellable = false` rather than + * clearing this, since nothing external is waiting on it. */ std::atomic compactCancelRequested{false}; @@ -597,7 +600,10 @@ struct DBDescriptor final : public std::enable_shared_from_this { rocksdb::ColumnFamilyHandle* column, const rocksdb::Slice* start, const rocksdb::Slice* end, - bool bottommost = false + bool bottommost = false, + // Close-initiated compaction opts out: nothing external is waiting on it, + // and `compactCancelRequested` stays armed for the whole of finishClose(). + bool cancellable = true ); }; diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index ef98174ab..af9948bb4 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -1050,6 +1050,20 @@ napi_value Transaction::GetCount(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_TRANSACTION_HANDLE("GetCount"); + // Without this claim finishClose()'s drain returns immediately and its + // closables sweep rolls back the transaction while the scan is parked + // between rows, leaving the iterator reading freed memory. + auto& txnDbHandle = (*txnHandle)->dbHandle; + if (!txnDbHandle || !txnDbHandle->descriptor) { + ::napi_throw_error(env, nullptr, "Get count failed: Database not open"); + NAPI_RETURN_UNDEFINED(); + } + OperationGuard operationGuard(txnDbHandle->descriptor); + if (txnDbHandle->descriptor->isClosing()) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); + } + DBIteratorOptions itOptions; itOptions.initFromNapiObject(env, argv[0]); itOptions.values = false; From 3abf8002a1a4b6260a5557245991add655745e89 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 26 Aug 2026 07:35:56 -0600 Subject: [PATCH 37/39] Make async-work admission and cancellation mutually exclusive; wait unbounded for drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AsyncWorkHandle::registerAsyncWork() unconditionally incremented its counter with no serialization against cancelAllAsyncWork(), and waitForAsyncWorkCompletion() gave up after a hardcoded 5s even if work remained. Since Flush/Compact/Clear/async Get in database.cpp only hold the descriptor's operationsInFlight guard through synchronous setup (not through the queued execute callback), the 5s bound let DBDescriptor::finishClose() reach this->db.reset() while a slow flush() (legitimately waiting out a write stall per AGENTS.md invariant 16, which has no bound) was still executing against it — a genuine use-after-free, not a theoretical one. registerAsyncWork()/cancelAllAsyncWork() now share waitMutex so admission and cancellation can never interleave: a registration either fully lands before cancellation publishes, or is refused. waitForAsyncWorkCompletion() is now unbounded, matching the existing unbounded operationsInFlight wait pattern elsewhere in db_descriptor.cpp. Every registerAsyncWork() call site (database.cpp Clear/Compact/Flush/Get, backup.cpp's shared queueBackupWork, checkpoint.cpp, transaction.cpp's two Commit() paths) is wired through a new admitAsyncWorkOrReject() helper that rejects the already-constructed promise and tears down cleanly on refusal instead of proceeding into a closing handle. ScopedAsyncWorkRegistration (transaction_handle.cpp, used for cross-column-family transactional reads) now tracks admission via ok() so its destructor can't underflow the count on a refused registration, and both of its call sites in TransactionHandle::get() check ok() explicitly rather than relying on the (currently-true but unenforced) correlation with isCancelled(). backup_stream.cpp's registration is left unchecked, with an explanatory comment: its operationsInFlight claim is held through the whole async execution already, so it can't hit refusal in practice. Corrected the README's lifecycleWaitSeconds doc: it said "Total maximum time," which contradicted the existing note that destroy()/shutdown()'s wait for in-flight backups/checkpoints is intentionally unbounded — reworded to clarify it only bounds the wait for a conflicting lifecycle op on the same path. Deleted .pr787-lifecycle-repair-plan.md (superseded by this commit and AGENTS.md invariant 17, which documents the fix). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019nt6hrpR4pnotKkZ8AUVv5 --- .pr787-lifecycle-repair-plan.md | 31 ----- AGENTS.md | 21 +++ README.md | 9 +- src/binding/database/backup.cpp | 4 +- src/binding/database/backup_stream.cpp | 9 +- src/binding/database/checkpoint.cpp | 4 +- src/binding/database/database.cpp | 18 ++- src/binding/napi/async.h | 123 ++++++++++++------ src/binding/transaction/transaction.cpp | 24 +++- .../transaction/transaction_handle.cpp | 45 ++++--- 10 files changed, 183 insertions(+), 105 deletions(-) delete mode 100644 .pr787-lifecycle-repair-plan.md diff --git a/.pr787-lifecycle-repair-plan.md b/.pr787-lifecycle-repair-plan.md deleted file mode 100644 index 58ed001fe..000000000 --- a/.pr787-lifecycle-repair-plan.md +++ /dev/null @@ -1,31 +0,0 @@ -# PR 787 lifecycle repair plan - -## Root cause and invariant - -Invariant: once descriptor close or handle cancellation begins, no new native operation may be admitted; teardown must not release native state until every operation admitted before that transition has drained. - -Current admission violates this invariant twice. `OperationGuard` increments `operationsInFlight` without serializing against `DBDescriptor::beginClose()`, so close can observe zero before a late increment. `AsyncWorkHandle::registerAsyncWork()` likewise increments without serializing against cancellation, and `waitForAsyncWorkCompletion()` continues after five seconds even if work remains. - -## Chosen repair - -1. Add a descriptor operation-admission mutex. `beginClose()` takes it while publishing `closing`; `OperationGuard` takes it while checking `closing` and incrementing `operationsInFlight`, and exposes whether admission succeeded. Existing callers stop touching native state when admission fails. -2. Make async-work admission atomic with cancellation under `AsyncWorkHandle::waitMutex`. Admission returns false after cancellation. Cancellation publishes under that mutex. Waiting becomes predicate-based and does not continue teardown while work remains. -3. Use a shared scoped async registration helper at N-API setup boundaries. It refuses admission after cancellation and transfers an admitted registration to queued state where needed. -4. Register transaction and iterator setup against their transaction handle, database handle, and descriptor before dereferencing teardown-owned state. Keep descriptor-wide operations pinned only for the duration in which native state is directly used; queued async state remains protected by the handle close drain. -5. Add deterministic setup/next race seams and subprocess tests covering database and transaction iterators, repeated idempotent iterator return, and forced destroy during `Next()`. -6. Address the remaining review feedback mechanically: retry destroy tombstone cleanup during shutdown, narrow the VT invariant text/use cached VT coordinates consistently, and diagnose retained benchmark data. - -## Approaches considered - -- Different layer — serialize only JavaScript `destroy()`/`open()` calls. Rejected: worker environments have separate JavaScript heaps, while the violated registry and native handles are process-global; a JS mutex cannot own the invariant across envs. -- Deeper cause — remove foreign forced close and require every environment to cooperatively close. Rejected: `destroy()` must invalidate all handles to the physical path, including handles in environments that may no longer service messages; cooperative teardown cannot guarantee physical deletion safety. -- Do less — retain the current per-call-site checks and increase/remove the five-second timeout. Rejected: timeout changes do not close either admission race, and a late registration can still begin after the waiter observed zero. -- Chosen — enforce atomic admission in the shared descriptor and handle primitives, then keep call sites responsible only for transferring valid claims. This is the only option that makes the close transition and every admission mutually ordered across worker threads without changing the public API. - -## Verification - -- Build the native debug binding first. -- Run focused destroy/iterator/transaction lifecycle tests repeatedly, including parallel subprocess batches. -- Run `pnpm check`, `pnpm test`, and `pnpm test:native` as the repository full gates. -- End-to-end route: the Node subprocess integration fixture uses real worker threads, a shared native descriptor, real RocksDB iterators/transactions, and physical destroy/reopen. -- Bug-proof route: the new next-race test must fail when the per-iterator mutex is removed; the admission-race tests must fail when admission/cancellation serialization is removed. diff --git a/AGENTS.md b/AGENTS.md index b04b83a22..296394293 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -629,6 +629,27 @@ sufficient (env teardown does not honor tsfn acquire counts); see is the descriptor's single `CommitWorker` thread (see "Commit execution" above), which dispatches every `Transaction.commit()` in order — so opting a flush into a stall queues up every commit behind it, including ones from callers that never touched flush. +17. **Async-work admission and cancellation share one mutex; the drain that follows must never time + out**: `AsyncWorkHandle` (`napi/async.h`) tracks in-flight async work per `DBHandle`/ + `TransactionHandle`. `registerAsyncWork()` and `cancelAllAsyncWork()` both take `waitMutex`, so a + registration that races a close either lands (and is counted) before cancellation publishes, or + is refused — there is no window where it is admitted after `waitForAsyncWorkCompletion()` has + already observed the count at zero. Refusal returns `false`; every call site (the shared + `admitAsyncWorkOrReject()` helper, or `ScopedAsyncWorkRegistration::ok()` for the RAII + cross-handle case in `transaction_handle.cpp`) must fail the operation — reject the + already-constructed promise and touch no native state — rather than proceed with work nothing is + tracking anymore. `waitForAsyncWorkCompletion()` itself has no timeout: `DBHandle::close()` / + `TransactionHandle::close()` call it immediately before releasing the `rocksdb::DB`, column + family, or transaction that admitted work may still be using, and a flush legitimately waiting + out a write stall (invariant 16) can run far longer than any fixed bound. A bounded wait that + gives up anyway — the previous 5-second default — let `finishClose()` reach `this->db.reset()` + while a flush was still executing against it, a genuine use-after-free. `database.cpp`'s + `Flush`/`Compact`/`Clear`/async `Get` rely entirely on this drain for safety: their + `OperationGuard` from `ACQUIRE_OPERATIONS_LOCK()` covers only the synchronous setup, not the + queued execute callback. Backup/checkpoint/backup-stream additionally hold the descriptor's + `operationsInFlight` claim through their whole async execution (the pinning pattern from + invariant 9), so for those this drain is defense in depth rather than the only thing preventing + a use-after-free. 17. **An env's pending transactions are reaped by its cleanup hook — never by `DBHandle::close()`**: `transactionAdd` stores a strong `shared_ptr` in the process-global diff --git a/README.md b/README.md index 2cd7214f8..1f6afd5a3 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,12 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. - - `lifecycleWaitSeconds: number` Total maximum time a synchronous open, destroy, or shutdown waits for - another lifecycle operation before throwing a retryable timeout error. Defaults to `30` seconds - and must be a positive integer. + - `lifecycleWaitSeconds: number` How long a synchronous open, destroy, or shutdown waits for a + _conflicting_ lifecycle operation already in progress on the same path (e.g. another open or + close) before throwing a retryable timeout error. It does not bound the separate, intentionally + unbounded wait that `destroy()`/`shutdown()` make for in-flight backups, checkpoints, or other + async work still using the database — see [`db.destroy()`](#dbdestroy-void). Defaults to `30` + seconds and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global [Verification Table](#verification-table). Each slot is 8 bytes, so the default of `131072` (128K) slots is 1 MB. Set to `0` to disable the verification table. This must be configured diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index a4a663010..6c7b14b05 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -165,7 +165,9 @@ static napi_value queueBackupWork( NAPI_STATUS_THROWS(::napi_create_async_work(env, nullptr, name, execute, complete, state, &state->asyncWork)); if (registerWork && state->handle) { - state->handle->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, state->handle.get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 6eb8c6803..054f7e46e 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -718,7 +718,14 @@ napi_value Database::BackupStream(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + // The operationsInFlight claim taken above is held for the whole stream + // (released at the end of backupStreamExecute, not here), so it already + // rules out a concurrent cancelAllAsyncWork() racing this registration — + // finishClose()'s first (unbounded) wait cannot reach the closables sweep + // that would call it until this claim releases. Admission is therefore + // guaranteed to succeed; discard the result rather than threading the + // tsfn's ownership through the same reject path as the simpler async ops. + (void)(*dbHandle)->registerAsyncWork(); // On a queue failure the claim rolls the counter back (execute never runs). // The state/tsfn leak on this rare N-API failure path matches the existing diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index ddf8a1fc2..588fc2dc7 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -194,7 +194,9 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } // On a queue failure the claim above rolls the counter back (execute never // runs); the state leak on this rare N-API failure path matches the existing diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 19c9f7fbe..2bfd4ba26 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -125,7 +125,9 @@ static napi_value doClear(napi_env env, napi_callback_info info, const char* fai )); // Register the async work with the database handle - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); @@ -366,7 +368,9 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); @@ -725,7 +729,9 @@ napi_value Database::Flush(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); @@ -868,7 +874,11 @@ napi_value Database::Get(napi_env env, napi_callback_info info) { // performs at the end of the execute handler. Without it the count goes negative, // so close() does not wait for this read and the worker dereferences a descriptor // that close() has already reset. - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + napi_value returnStatus; + NAPI_STATUS_THROWS(::napi_create_uint32(env, 1, &returnStatus)); + return returnStatus; + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); diff --git a/src/binding/napi/async.h b/src/binding/napi/async.h index bae24ccf7..b949711aa 100644 --- a/src/binding/napi/async.h +++ b/src/binding/napi/async.h @@ -141,67 +141,69 @@ struct AsyncWorkHandle { std::mutex waitMutex; std::condition_variable asyncWorkComplete; - void registerAsyncWork() { + /** + * Admits one unit of async work. Returns false, without incrementing the + * count, once cancellation has been published — the caller must not queue + * the async work or otherwise touch native state in that case. Admission + * and `cancelAllAsyncWork()` share `waitMutex` so the two can never + * interleave: either this call is fully visible to the wait below before + * cancellation publishes, or it is refused. Without that, a registration + * racing a close's cancel+wait could land after the wait already observed + * zero, letting new work run concurrently with (or after) the native + * state that close() goes on to release. + */ + [[nodiscard]] bool registerAsyncWork() { + std::lock_guard lock(this->waitMutex); + if (this->cancelled.load()) { + return false; + } ++this->activeAsyncWorkCount; + return true; } void unregisterAsyncWork() { + std::lock_guard lock(this->waitMutex); auto activeAsyncWorkCount = --this->activeAsyncWorkCount; if (activeAsyncWorkCount > 0) { DEBUG_LOG("%p AsyncWorkHandle::unregisterAsyncWork Still have %u active async work tasks\n", this, activeAsyncWorkCount); } else if (activeAsyncWorkCount == 0) { DEBUG_LOG("%p AsyncWorkHandle::unregisterAsyncWork All async work has completed, notifying\n", this); - this->asyncWorkComplete.notify_one(); + this->asyncWorkComplete.notify_all(); } } void cancelAllAsyncWork() { + std::lock_guard lock(this->waitMutex); this->cancelled.store(true); } /** - * Waits for in-flight async work to finish. Returns true when the count - * actually reached zero, false when the timeout expired with work still - * running — callers that destroy state the work is using MUST check it. + * Blocks until every admitted unit of async work has completed. This must + * not time out: the caller is about to release native state (a + * `rocksdb::DB`, a column family, a transaction) that admitted work may + * still be using. A flush legitimately waiting out a write stall (see + * AGENTS.md invariant 16) can run far longer than any fixed bound, and a + * bounded wait that gives up anyway turns into a use-after-free once the + * caller proceeds to tear down that state. + * + * Supersedes the bounded, bool-returning version this replaced (a 5s + * timeout with a "leak instead of free" fallback in + * `TransactionHandle::close()`) — that was a deliberate stopgap tracked as + * HarperFast/rocksdb-js#784, this unbounded wait *is* #784. Every caller + * can now assume the drain always completes; there is no timed-out case + * left to handle. */ - bool waitForAsyncWorkCompletion( - std::chrono::milliseconds timeout = std::chrono::milliseconds(5000) - ) { - auto start = std::chrono::steady_clock::now(); - const auto pollInterval = std::chrono::milliseconds(10); + void waitForAsyncWorkCompletion() { std::unique_lock lock(this->waitMutex); - auto activeAsyncWorkCount = this->activeAsyncWorkCount.load(); - - if (activeAsyncWorkCount == 0) { + if (this->activeAsyncWorkCount.load() == 0) { DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion no async work to wait for\n", this); - return true; - } - - while (activeAsyncWorkCount > 0) { - auto elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - start); - if (elapsed >= timeout) { - DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion timeout waiting for async work completion, %u items remaining\n", this, activeAsyncWorkCount); - return false; - } - - auto remainingTime = timeout - elapsed; - auto waitTime = std::min(pollInterval, remainingTime); - - DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion waiting for %u active work items\n", this, activeAsyncWorkCount); - - bool completed = this->asyncWorkComplete.wait_for(lock, waitTime, [this] { - return this->activeAsyncWorkCount.load() == 0; - }); - - if (completed) { - DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion all async work execution completed\n", this); - return true; - } - - activeAsyncWorkCount = this->activeAsyncWorkCount.load(); + return; } - - return true; + DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion waiting for active work items\n", this); + this->asyncWorkComplete.wait(lock, [this] { + return this->activeAsyncWorkCount.load() == 0; + }); + DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion all async work execution completed\n", this); } bool isCancelled() const { @@ -209,10 +211,51 @@ struct AsyncWorkHandle { } void resetCancelled() { + std::lock_guard lock(this->waitMutex); this->cancelled.store(false); } }; +/** + * Admits `state`'s async work onto `handle`. On success, returns true and the + * caller proceeds to `napi_queue_async_work()` as usual. On refusal + * (cancellation already published by a concurrent close), tears down the + * async work object created for `state` (if any) and the promise references + * already captured on it, rejects the promise with `message`, deletes + * `state`, and returns false — the caller must return to JS immediately + * without dereferencing any native state `state` was set up to use. + */ +template +bool admitAsyncWorkOrReject(napi_env env, AsyncWorkHandle* handle, State* state, const char* message) { + if (handle->registerAsyncWork()) { + return true; + } + + // Nothing was incremented, so mark completed before the destructor's + // signalExecuteCompleted() runs — otherwise it would call + // unregisterAsyncWork() for a registration that never succeeded. + state->completed.store(true); + + if (state->asyncWork) { + if (::napi_delete_async_work(env, state->asyncWork) == napi_ok) { + state->asyncWork = nullptr; + } + } + + napi_value error = nullptr; + napi_value messageValue; + if (::napi_create_string_utf8(env, message, NAPI_AUTO_LENGTH, &messageValue) == napi_ok) { + ::napi_create_error(env, nullptr, messageValue, &error); + } + if (error == nullptr) { + ::napi_get_undefined(env, &error); + } + state->callReject(error); + + delete state; + return false; +} + } // namespace rocksdb_js #endif diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index af9948bb4..9e8dc246b 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -789,8 +789,24 @@ napi_value Transaction::Commit(napi_env env, napi_callback_info info) { // below rather than re-creating a tsfn the close will never release. NAPI_STATUS_THROWS(descriptor->registerCommitCompletion(env, commitCompletionCallJs, completionsClosed)); if (!completionsClosed) { - // register the commit with the transaction handle so close() can wait - (*txnHandle)->registerAsyncWork(); + // Register the commit with the transaction handle so close() can wait. + // Refusal means a concurrent close() already published cancellation (and, + // by the same happens-before edge, already forced state to Aborted) — + // undo the completion registration just above and reject rather than + // dispatching into a transaction that is being torn down. + if (!(*txnHandle)->registerAsyncWork()) { + descriptor->finishCommitCompletion(env); + // Nothing was incremented on the handle, so mark completed before + // the destructor's signalExecuteCompleted() runs — otherwise it + // would call unregisterAsyncWork() for a registration that never + // succeeded. + state->completed.store(true); + napi_value error; + rocksdb_js::createJSError(env, "ERR_TRANSACTION_CLOSING", "Transaction is closing", error); + state->callReject(error); + delete state; + NAPI_RETURN_UNDEFINED(); + } // Commit-lane stage: RocksDB commit, then marshal the completion // back to the originating env. @@ -871,7 +887,9 @@ napi_value Transaction::Commit(napi_env env, napi_callback_info info) { )); // register the async work with the transaction handle - (*txnHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*txnHandle).get(), state, "Transaction is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); diff --git a/src/binding/transaction/transaction_handle.cpp b/src/binding/transaction/transaction_handle.cpp index 84637a739..9003b3d73 100644 --- a/src/binding/transaction/transaction_handle.cpp +++ b/src/binding/transaction/transaction_handle.cpp @@ -19,19 +19,28 @@ namespace { * handle's column descriptor. This bridges the gap between the caller's open * check and the transaction's async-work registration without making the * worker access a concurrently closing DBHandle. + * + * Admission can be refused if `handle`'s cancellation was already published + * (a concurrent close); `admitted` tracks that so the destructor only + * unregisters a claim it actually holds — unregistering an admission that + * never succeeded would underflow the target's active-work count and could + * make a later close() stop waiting while real work is still outstanding. + * Callers must check `ok()` after construction and treat a false result the + * same as the target already being closed. */ struct ScopedAsyncWorkRegistration { AsyncWorkHandle* handle; + bool admitted; explicit ScopedAsyncWorkRegistration(AsyncWorkHandle* handle) - : handle(handle) { + : handle(handle), admitted(false) { if (this->handle) { - this->handle->registerAsyncWork(); + this->admitted = this->handle->registerAsyncWork(); } } ~ScopedAsyncWorkRegistration() { - if (this->handle) { + if (this->handle && this->admitted) { this->handle->unregisterAsyncWork(); } } @@ -39,6 +48,10 @@ struct ScopedAsyncWorkRegistration { ScopedAsyncWorkRegistration(const ScopedAsyncWorkRegistration&) = delete; ScopedAsyncWorkRegistration& operator=(const ScopedAsyncWorkRegistration&) = delete; + bool ok() const { + return !this->handle || this->admitted; + } + void release() { this->handle = nullptr; } @@ -317,8 +330,12 @@ void TransactionHandle::close() { // Drain BEFORE touching anything the in-flight work owns. Nothing below is // safe while a commit is still executing: `state` feeds the commitAborted() // decision, releaseIntent() mutates VT state the commit is using, and - // `delete txn` hands RocksDB a dangling transaction. - const bool drained = this->waitForAsyncWorkCompletion(); + // `delete txn` hands RocksDB a dangling transaction. waitForAsyncWorkCompletion() + // is unbounded (HarperFast/rocksdb-js#784) and cancelAllAsyncWork() above shares + // a mutex with registerAsyncWork(), so this always drains to zero — there is no + // timed-out case left to handle here; an earlier version of this close() leaked + // the transaction on a 5s drain timeout as a stopgap for exactly this window. + this->waitForAsyncWorkCompletion(); // Test seam: widen the PATH A vs PATH B race window (see txnCloseTestDelayMs). // This window is real in production (PATH B fires after waitForAsyncWorkCompletion @@ -329,20 +346,6 @@ void TransactionHandle::close() { std::this_thread::sleep_for(std::chrono::milliseconds(closeDelayMs)); } - if (!drained) { - // The drain timed out with work still executing against `txn` (e.g. a - // worker env torn down during a slow commit). Destroying now would free - // a transaction RocksDB is still using and could mark the log aborted - // while the data commit goes on to succeed — so deliberately leak - // instead. The in-flight commit owns its own cleanup (it releases VT - // intents and resolves the log position on completion); this close must - // not steal it. A leaked transaction is recoverable; a use-after-free - // and a log/data disagreement are not. The complete admission-and-drain - // contract that removes this window is HarperFast/rocksdb-js#784. - DEBUG_LOG("%p TransactionHandle::close async work still in flight after drain timeout; leaking txn rather than freeing it\n", this); - return; - } - // Only now that no native work can be running: settle the final state. if (this->state == TransactionState::Pending || this->state == TransactionState::Committing) { this->state = TransactionState::Aborted; @@ -418,7 +421,7 @@ napi_value TransactionHandle::get( // the middle of this setup. Async fallback transfers this registration to its // state; synchronous and failed setup paths release it on return. ScopedAsyncWorkRegistration transactionRegistration(this); - if (this->isCancelled() || !this->txn) { + if (!transactionRegistration.ok() || this->isCancelled() || !this->txn) { ::napi_throw_error(env, nullptr, "Transaction is closed"); return nullptr; } @@ -441,7 +444,7 @@ napi_value TransactionHandle::get( // that handle while copying its descriptor so a concurrent close cannot reset // columnDescriptor in the gap between the caller's open check and this read. ScopedAsyncWorkRegistration targetHandleRegistration(dbHandleOverride.get()); - if (dbHandleOverride && dbHandle->isCancelled()) { + if (dbHandleOverride && (!targetHandleRegistration.ok() || dbHandle->isCancelled())) { ::napi_throw_error(env, nullptr, "Database closed during transaction get operation"); return nullptr; } From ab5cd9696c47e9e753a0c0e281b390b8c9a76486 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 26 Aug 2026 08:09:10 -0600 Subject: [PATCH 38/39] fix(async): release admitted async-work claims on a queue failure Gemini + Harper-domain pre-push review (round 1) found that every admitAsyncWorkOrReject() call site followed a successful admission with a bare NAPI_STATUS_THROWS(::napi_queue_async_work(...)). On the rare napi_queue_async_work() failure that macro throws and returns immediately, leaking `state` with its AsyncWorkHandle registration still counted. Since waitForAsyncWorkCompletion() is now unbounded (this branch's whole point), that stuck count blocks the handle's close forever, which blocks every later OpenDB() for its path -- worse than the leak it fixed. Added queueAsyncWorkOrReject() alongside admitAsyncWorkOrReject() in async.h: releases the admitted claim via signalExecuteCompleted(), deletes the async work object, rejects the promise, deletes state. Takes an `admitted` flag for backup.cpp's queueBackupWork(), whose registration is conditional on its registerWork/state->handle parameters -- unregistering an admission that never happened would underflow the count the same way. backup_stream.cpp's AsyncBackupStreamState is refcounted (acquire()/ release(), shared with an N-API tsfn) rather than a plain heap object, so the generic helper's `delete state` would double-free against tsfnFinalize()'s later release(). Wrote the queue-failure cleanup by hand there instead, mirroring backupStreamComplete()'s existing teardown sequence (delete async work, release the tsfn, drop the descriptor pin, reject, release the constructor's own ref). Also asserted the invariant its "(void)registerAsyncWork()" comment already claimed (registration there is guaranteed by the operationsInFlight claim held for the whole stream) -- the review's other nit, that the claim was load-bearing but unenforced. transaction_handle.cpp's async Get fallback was checked and left alone: its queue call runs before a still-in-scope PendingAsyncState RAII guard is released, so a queue failure already unwinds and deletes state correctly. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019nt6hrpR4pnotKkZ8AUVv5 --- src/binding/database/backup.cpp | 6 ++- src/binding/database/backup_stream.cpp | 58 +++++++++++++++++++++---- src/binding/database/checkpoint.cpp | 7 ++- src/binding/database/database.cpp | 18 ++++++-- src/binding/napi/async.h | 53 ++++++++++++++++++++++ src/binding/transaction/transaction.cpp | 4 +- 6 files changed, 128 insertions(+), 18 deletions(-) diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 6c7b14b05..3229ce5df 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -164,13 +164,17 @@ static napi_value queueBackupWork( NAPI_STATUS_THROWS(::napi_create_async_work(env, nullptr, name, execute, complete, state, &state->asyncWork)); + bool admitted = false; if (registerWork && state->handle) { if (!admitAsyncWorkOrReject(env, state->handle.get(), state, "Database is closing")) { NAPI_RETURN_UNDEFINED(); } + admitted = true; } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue backup work", admitted)) { + NAPI_RETURN_UNDEFINED(); + } if (queued) *queued = true; NAPI_RETURN_UNDEFINED(); diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 054f7e46e..17042e84e 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -718,19 +718,61 @@ napi_value Database::BackupStream(napi_env env, napi_callback_info info) { &state->asyncWork )); + // This state is refcounted (not a plain `delete`), shared with the tsfn + // above, so queueAsyncWorkOrReject()'s generic `delete state` would + // double-free against tsfnFinalize()'s later release(). Mirror + // backupStreamComplete()'s cleanup by hand instead for both failure exits + // below, since execute/complete never run to do it themselves: delete the + // async work object, release the tsfn (tsfnFinalize drops its ref), drop + // the descriptor pin, reject, and release the constructor's own ref. + // `admitted` selects how the AsyncWorkHandle claim is released: a queue + // failure must decrement a real registration (signalExecuteCompleted()), + // while a refused registration never incremented anything and must only + // be marked completed -- decrementing there would underflow the count. + auto rejectAndCleanup = [&](bool admitted, const char* message) { + if (admitted) { + state->signalExecuteCompleted(); + } else { + state->completed.store(true); + } + state->deleteAsyncWork(); + if (state->tsfn != nullptr) { + ::napi_release_threadsafe_function(state->tsfn, napi_tsfn_release); + state->tsfn = nullptr; + } + state->releaseDescriptor(); + + napi_value error = nullptr; + napi_value messageValue; + if (::napi_create_string_utf8(env, message, NAPI_AUTO_LENGTH, &messageValue) == napi_ok) { + ::napi_create_error(env, nullptr, messageValue, &error); + } + if (error == nullptr) { + ::napi_get_undefined(env, &error); + } + state->callReject(error); + state->release(); + }; + // The operationsInFlight claim taken above is held for the whole stream // (released at the end of backupStreamExecute, not here), so it already // rules out a concurrent cancelAllAsyncWork() racing this registration — // finishClose()'s first (unbounded) wait cannot reach the closables sweep // that would call it until this claim releases. Admission is therefore - // guaranteed to succeed; discard the result rather than threading the - // tsfn's ownership through the same reject path as the simpler async ops. - (void)(*dbHandle)->registerAsyncWork(); - - // On a queue failure the claim rolls the counter back (execute never runs). - // The state/tsfn leak on this rare N-API failure path matches the existing - // async methods (e.g. Database::Backup). - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + // expected to always succeed, but unlike the simpler async ops this state + // can't route refusal through admitAsyncWorkOrReject() (its `delete state` + // would double-free here) -- so on the vanishingly unlikely chance a future + // change adds another cancelAllAsyncWork() path for this handle, reject + // properly instead of silently proceeding with an unregistered stream. + if (!(*dbHandle)->registerAsyncWork()) { + rejectAndCleanup(false, "Database is closing"); + NAPI_RETURN_UNDEFINED(); + } + + if (::napi_queue_async_work(env, state->asyncWork) != napi_ok) { + rejectAndCleanup(true, "Failed to queue backup stream work"); + NAPI_RETURN_UNDEFINED(); + } // The worker now owns the in-flight decrement (end of execute). handedOff = true; diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index 588fc2dc7..a30d2699a 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -198,10 +198,9 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - // On a queue failure the claim above rolls the counter back (execute never - // runs); the state leak on this rare N-API failure path matches the existing - // async methods (e.g. Database::Backup). - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue checkpoint work")) { + NAPI_RETURN_UNDEFINED(); + } // The worker now owns the in-flight decrement (end of execute); stop the // claim from releasing it here. diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 2bfd4ba26..f26c531d7 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -129,7 +129,9 @@ static napi_value doClear(napi_env env, napi_callback_info info, const char* fai NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue clear work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } @@ -372,7 +374,9 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue compact work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } @@ -733,7 +737,9 @@ napi_value Database::Flush(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue flush work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } @@ -880,7 +886,11 @@ napi_value Database::Get(napi_env env, napi_callback_info info) { return returnStatus; } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue get work")) { + napi_value returnStatus; + NAPI_STATUS_THROWS(::napi_create_uint32(env, 1, &returnStatus)); + return returnStatus; + } napi_value returnStatus; NAPI_STATUS_THROWS(::napi_create_uint32(env, 1, &returnStatus)); diff --git a/src/binding/napi/async.h b/src/binding/napi/async.h index b949711aa..83be13e26 100644 --- a/src/binding/napi/async.h +++ b/src/binding/napi/async.h @@ -256,6 +256,59 @@ bool admitAsyncWorkOrReject(napi_env env, AsyncWorkHandle* handle, State* state, return false; } +/** + * Queues `state`'s async work after a successful `admitAsyncWorkOrReject()`. + * On success, returns true and the caller returns its normal pending value. + * On the rare `napi_queue_async_work()` failure, the admission above already + * incremented the target's active-work count with nothing left to run it + * down — a bare `NAPI_STATUS_THROWS` here would return without releasing + * that claim, leaking `state` and leaving the count permanently off by one. + * Since a bounded `waitForAsyncWorkCompletion()` no longer exists to paper + * over a stuck count (see the note on that method), a single stranded claim + * blocks that handle's close forever, which in turn blocks every later + * `OpenDB()` for its path. This releases the claim via + * `signalExecuteCompleted()` (unlike `admitAsyncWorkOrReject()`'s refusal + * path, this one actually decrements — the claim was real), deletes the + * async work object and `state`, rejects the promise with `message`, and + * returns false. + * + * `admitted` must be false when the caller skipped `admitAsyncWorkOrReject()` + * for this `state` (e.g. backup.cpp's `queueBackupWork(..., registerWork)` + * with `registerWork` false) — otherwise `signalExecuteCompleted()` would + * unregister a claim that was never taken, underflowing the count. + */ +template +bool queueAsyncWorkOrReject(napi_env env, State* state, const char* message, bool admitted = true) { + if (::napi_queue_async_work(env, state->asyncWork) == napi_ok) { + return true; + } + + if (admitted) { + state->signalExecuteCompleted(); + } else { + state->completed.store(true); + } + + if (state->asyncWork) { + if (::napi_delete_async_work(env, state->asyncWork) == napi_ok) { + state->asyncWork = nullptr; + } + } + + napi_value error = nullptr; + napi_value messageValue; + if (::napi_create_string_utf8(env, message, NAPI_AUTO_LENGTH, &messageValue) == napi_ok) { + ::napi_create_error(env, nullptr, messageValue, &error); + } + if (error == nullptr) { + ::napi_get_undefined(env, &error); + } + state->callReject(error); + + delete state; + return false; +} + } // namespace rocksdb_js #endif diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index 9e8dc246b..797244b7f 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -891,7 +891,9 @@ napi_value Transaction::Commit(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue commit work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } From ba0e1c071bef5d3169edeb27041e9c3de38e3129 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 26 Aug 2026 09:36:54 -0600 Subject: [PATCH 39/39] fix(close): cancel compaction before async drain Co-Authored-By: GPT-5 Codex --- src/binding/database/db_descriptor.cpp | 7 ++----- src/binding/database/db_descriptor.h | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index ed2e4acb5..e3e8e9814 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -394,11 +394,8 @@ void DBDescriptor::finishClose(bool destroying) { // Existing operations will decrement operationsInFlight and notify us when done. // Unbounded in-flight operations must abort once `closing` is published // rather than block this untimed wait for their full duration. A count - // scan polls isClosing() itself; a manual compactRange() cannot, so it - // gets an explicit cancel token. The token stays armed past this drain: - // an async compact() released its OperationGuard at setup handoff, so it - // is still running here and is not awaited until the closables sweep. - this->compactCancelRequested.store(true); + // scan polls isClosing() itself; a manual compactRange() uses the cancel + // token armed by beginClose(), before DBHandle::close() starts its drain. DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 61d2ff9d8..870bc4a5e 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -293,13 +293,11 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool transactionLogsUnregistered = false; /** - * Armed by finishClose() for its whole duration so an in-progress manual - * compactRange() on another thread aborts instead of making teardown wait - * out its full, unbounded duration. It covers both shapes: compactSync() - * holds an OperationGuard and so blocks the drain, while an async compact() - * released its guard at setup handoff and is not awaited until the closables - * sweep -- clearing the token after the drain left that second one able to - * stall teardown (and every concurrent open on the path) indefinitely. + * Armed when close is claimed so an in-progress manual + * compactRange() on another thread aborts before DBHandle::close() waits for + * its async work to drain. It covers both shapes: compactSync() holds an + * OperationGuard and so blocks the drain, while an async compact() released + * its guard at setup handoff and is not awaited until the closables sweep. * Close-initiated compaction passes `cancellable = false` rather than * clearing this, since nothing external is waiting on it. */ @@ -464,7 +462,13 @@ struct DBDescriptor final : public std::enable_shared_from_this { * under the same lock) waits instead of handing the descriptor to a new * handle that would then be closed out from under it. */ - bool beginClose() { return !this->closing.exchange(true); } + bool beginClose() { + if (this->closing.exchange(true)) { + return false; + } + this->compactCancelRequested.store(true); + return true; + } /** * Performs the actual close work (flush, close handles, release resources).