Add Dockerized Anvil test network (#723)

* Add Dockerized Anvil test network

* Make bot RPC quorum configurable

Author: KillariDev

bots/liquidator/README.md

@@ -3,8 +3,8 @@
 The liquidator discovers every security pool registered by a configured
 `SecurityPoolFactory`, shows pool and vault statistics in a local dashboard, and
 evaluates unsafe vaults in operator-selected pools. Dry-run is the default. Live
-execution requires an explicit signer, independent read RPC quorum, execution
-flag, and non-zero deployment addresses.
+execution requires an explicit signer, the configured read RPC quorum, the
+execution flag, and non-zero deployment addresses.
 
 The bot owns an ordinary vault under its signer address in each selected pool. A
 liquidation moves ETH-denominated open-interest debt, the proportional attoREP
@@ -41,6 +41,13 @@ password. Compose publishes the port only on host loopback, so connect from anot
 machine through a trusted tunnel to the host rather than changing the port binding.
 Keep `ZOLTAR_BOT_DASHBOARD_LOOPBACK_PUBLISHED` paired with that `127.0.0.1` mapping.
 
+Compose passes `ZOLTAR_BOT_RPC_QUORUM`, which defaults to `2`. This production
+policy requires two agreeing readers and two independent quorum RPC URLs in addition
+to the primary reader so one endpoint may be unavailable. For an isolated local
+development chain only, put `ZOLTAR_BOT_RPC_QUORUM=1` in this directory's `.env`
+before starting Compose. That setting permits the primary reader to operate alone
+and removes independent RPC corroboration. Values other than `1` or `2` stop startup.
+
 Save the chain and RPCs in **Chain and RPC connectivity**, finish the remaining
 configuration, and resume only after reviewing the saved settings. Run
 `docker compose down` to stop the bot and `docker compose up` to start it again.
@@ -81,8 +88,8 @@ saved. Same-chain RPC changes apply at the next scan. A configured operator file
 cannot be retargeted to another chain: create a separate paused configuration with
 a separate `runtime.stateFile`, then select its chain and endpoints in the
 dashboard. This boundary prevents transactions, staged operations, and scan state
-from crossing chains. Live execution requires two independent quorum RPCs in addition
-to the primary read RPC.
+from crossing chains. Under the default quorum policy, live execution requires two
+independent quorum RPCs in addition to the primary read RPC.
 
 A configured bot keeps its dashboard available when retryable RPC transport
 unavailability prevents startup validation. It reports `connectivity-degraded`, shows
@@ -107,13 +114,14 @@ endpoints, gas limits, and REP limits have been reviewed. When execution is
 enabled:
 
 - `connectivity.readRpcUrl` supplies the local operational view.
-- `connectivity.quorumRpcUrls` must contain at least two independent read RPCs.
+- Under the default quorum policy, `connectivity.quorumRpcUrls` must contain at
+  least two independent read RPCs.
 - For a critical pool, price, vault, or candidate snapshot, only a retryable
-  transport failure makes a reader unavailable. At least two readers must respond,
-  and every responding reader must agree exactly before a transaction is sent. One
-  transport-unavailable endpoint degrades health without stopping a healthy
-  two-reader quorum; a malformed or contradictory response is a safety fault and
-  fails closed.
+  transport failure makes a reader unavailable. The configured number of readers
+  must respond, and every responding reader must agree exactly before a transaction
+  is sent. Under the default policy, one transport-unavailable endpoint degrades
+  health without stopping a healthy two-reader quorum. A malformed or contradictory
+  response is a safety fault and fails closed.
 - `submission.mode` may be `public` or `private`. ETH-funded stale-price requests
   use the same signed-transaction delivery policy as other actions.
 - `privateKey` is stored in the local operator file only when explicitly saved.

bots/liquidator/compose.yaml

@@ -10,6 +10,7 @@ services:
     environment:
       # Keep this assertion paired with the host-loopback port mapping below.
       ZOLTAR_BOT_DASHBOARD_LOOPBACK_PUBLISHED: "true"
+      ZOLTAR_BOT_RPC_QUORUM: ${ZOLTAR_BOT_RPC_QUORUM-2}
     ports:
       - 127.0.0.1:4183:4183
     healthcheck:

bots/liquidator/src/cli/run.ts

@@ -5,6 +5,7 @@ import { createRpcEndpointPool } from '@zoltar/bot-shared/ethereum'
 import { checkConnectivity, checkSubmissionEndpoints, endpointLabel, readRpcChainId } from '@zoltar/bot-shared/monitoring/connectivity'
 import { availableSettledValues, settledQuorumValue } from '@zoltar/bot-shared/monitoring/read-quorum'
 import { ConnectivityDegradedError, operationalFailureDisposition, pollUntilStopped, retryDelayMilliseconds } from '@zoltar/bot-shared/monitoring/resilience'
+import { rpcQuorumRequirement } from '@zoltar/bot-shared/monitoring/rpc-quorum-policy'
 import { availableExecutionObservations } from '#monitoring/execution-quorum'
 import { signerCandidate } from '@zoltar/bot-shared/config/signer'
 import { loadSettings, parseDesiredPools, parseStrategy, saveSettings, serializedSettings, type OperatorSettings } from '#config/settings'
@@ -128,7 +129,7 @@ async function runOperator(loaded: Awaited<ReturnType<typeof loadSettings>>, pro
 							currentHeads: async () => {
 								const settled = await Promise.allSettled(endpoints.map(async endpoint => await createPublicClient({ chain, transport: readPool.transportFor(endpoint) }).getBlockNumber()))
 								const heads = availableSettledValues(settled)
-								if (heads.length < 2) throw new ConnectivityDegradedError('Replacement reconciliation requires at least two available independent RPC endpoints')
+								if (heads.length < rpcQuorumRequirement()) throw new ConnectivityDegradedError('Replacement reconciliation does not satisfy the configured RPC quorum requirement')
 								return heads
 							},
 							replacement: async () => replacementEvidence,

bots/liquidator/src/config/settings.ts

@@ -6,6 +6,7 @@ import { signerCandidate } from '@zoltar/bot-shared/config/signer'
 import { validateConnectivitySettings, validateIndependentReadRpcUrls, type ConnectivitySettings, type NetworkName } from '@zoltar/bot-shared/monitoring/connectivity'
 import { validateSubmissionSettings, type SubmissionSettings } from '@zoltar/bot-shared/execution/transaction-submission'
 import { parseCentralizedMarketSettings, serializeCentralizedMarketSettings, type CentralizedMarketSettings } from '@zoltar/bot-shared/monitoring/centralized-markets'
+import { configuredQuorumRpcUrlMinimum, rpcQuorumRequirement } from '@zoltar/bot-shared/monitoring/rpc-quorum-policy'
 
 export type CandidatePriority = 'largest-bonus' | 'largest-debt' | 'lowest-top-up'
 
@@ -256,6 +257,7 @@ function parseConnectivity(value: unknown): OperatorSettings['connectivity'] {
 }
 
 export function parseSettings(value: unknown): OperatorSettings {
+	rpcQuorumRequirement()
 	const root = record(value, 'operator settings')
 	if (root['version'] !== 1) throw new Error('operator settings version must be 1')
 	const deployment = record(root['deployment'], 'deployment')
@@ -331,7 +333,7 @@ export function parseSettings(value: unknown): OperatorSettings {
 	const marketAssetIds = [settings.centralizedMarkets, ...settings.childMarketConfigurations].map(configuration => configuration.assetAddress.toLowerCase())
 	if (new Set(marketAssetIds).size !== marketAssetIds.length) throw new Error('Market configurations must target distinct REP assets')
 	if (settings.runtime.execute && settings.privateKey === undefined) throw new Error('Live execution requires privateKey')
-	if (settings.runtime.execute && settings.connectivity.quorumRpcUrls.length < 2) throw new Error('Live execution requires at least two independent quorum RPCs (three read endpoints total)')
+	if (settings.runtime.execute && settings.connectivity.quorumRpcUrls.length < configuredQuorumRpcUrlMinimum()) throw new Error('Live execution requires at least two independent quorum RPCs (three read endpoints total)')
 	if (settings.runtime.execute && settings.deployment.securityPoolFactory === getAddress('0x0000000000000000000000000000000000000000')) throw new Error('Live execution requires a deployed security-pool factory')
 	if (settings.runtime.execute && settings.deployment.weth === getAddress('0x0000000000000000000000000000000000000000')) throw new Error('Live execution requires a deployed WETH contract')
 	if (settings.runtime.execute && settings.deployment.zoltar === getAddress('0x0000000000000000000000000000000000000000')) throw new Error('Live execution requires a deployed Zoltar contract')

bots/liquidator/src/core/network-connectivity.ts

@@ -1,4 +1,5 @@
 import { checkConnectivity, checkSubmissionEndpoints, endpointLabel, readRpcChainId, validateConnectivitySettings, validateIndependentReadRpcUrls } from '@zoltar/bot-shared/monitoring/connectivity'
+import { configuredQuorumRpcUrlMinimum } from '@zoltar/bot-shared/monitoring/rpc-quorum-policy'
 import type { OperatorSettings } from '#config/settings'
 
 type ConnectivityChecks = {
@@ -22,7 +23,7 @@ export async function updateNetworkConnectivity(parameters: { apply: (settings:
 	const quorumRpcUrls = validateIndependentReadRpcUrls(connectivity.readRpcUrl, rawQuorumRpcUrls.map(String))
 	const network: OperatorSettings['network'] = networkName === 'mainnet' ? { chainId: 1, explorerUrl: 'https://etherscan.io', name: networkName } : { chainId: 11_155_111, explorerUrl: 'https://sepolia.etherscan.io', name: networkName }
 	if (settings.networkConfigured && network.chainId !== settings.network.chainId) throw new Error('Use a separate operator configuration and durable state file to change chains')
-	if (settings.runtime.execute && quorumRpcUrls.length < 2) throw new Error('Live execution requires at least two independent quorum RPCs (three read endpoints total)')
+	if (settings.runtime.execute && quorumRpcUrls.length < configuredQuorumRpcUrlMinimum()) throw new Error('Live execution requires at least two independent quorum RPCs (three read endpoints total)')
 	const checks = parameters.checks ?? defaultChecks
 	await checks.checkConnectivity(connectivity, network.chainId)
 	for (const rpcUrl of quorumRpcUrls) {

bots/liquidator/src/execution/recovery.ts

@@ -5,6 +5,7 @@ import { confirmCanonicalReceiptFinality } from '@zoltar/bot-shared/execution/ca
 import { sendRawTransactionToRpc } from '@zoltar/bot-shared/monitoring/connectivity'
 import { availableSettledValues, settledQuorumValue } from '@zoltar/bot-shared/monitoring/read-quorum'
 import { ConnectivityDegradedError } from '@zoltar/bot-shared/monitoring/resilience'
+import { rpcQuorumRequirement } from '@zoltar/bot-shared/monitoring/rpc-quorum-policy'
 import { submitSignedTransaction } from '@zoltar/bot-shared/execution/transaction-submission'
 import type { OperatorSettings } from '#config/settings'
 import { stagedOperationOutcome } from '#core/staged-outcome'
@@ -115,7 +116,7 @@ export async function recoverPendingTransactions(settings: OperatorSettings, wal
 		}
 		const settledBlocks = await Promise.allSettled(clients.map(async ({ client }) => await client.getBlockNumber()))
 		const blocks = availableSettledValues(settledBlocks)
-		if (blocks.length < 2) throw new ConnectivityDegradedError(`Transaction ${intent.hash} recovery requires at least two available independent RPC endpoints`)
+		if (blocks.length < rpcQuorumRequirement()) throw new ConnectivityDegradedError(`Transaction ${intent.hash} recovery does not satisfy the configured RPC quorum requirement`)
 		const recoveryAction = ambiguousRecoveryAction(intent, blocks)
 		if (recoveryAction === 'expire-private') {
 			await canonicalBlockHash(settings, intent.maxBlockNumber + PRIVATE_INTENT_FINALITY_BLOCKS, pool)
@@ -150,7 +151,7 @@ export async function reconcilePendingStagedOperations(settings: OperatorSetting
 	for (const pending of [...state.pendingStagedOperations]) {
 		const settledHeads = await Promise.allSettled(clients.map(async ({ client }) => await client.getBlockNumber()))
 		const heads = availableSettledValues(settledHeads)
-		if (heads.length < 2) throw new ConnectivityDegradedError(`Staged operation ${pending.operationId.toString()} recovery requires at least two available independent RPC endpoints`)
+		if (heads.length < rpcQuorumRequirement()) throw new ConnectivityDegradedError(`Staged operation ${pending.operationId.toString()} recovery does not satisfy the configured RPC quorum requirement`)
 		const toBlock = heads.reduce((minimum, head) => (head < minimum ? head : minimum))
 		let outcome: (NonNullable<ReturnType<typeof stagedOperationOutcome>> & { blockHash: Hex; blockNumber: bigint; transactionHash: Hex }) | undefined
 		for (const range of stagedOperationRecoveryRanges(pending.queuedBlock, toBlock)) {

bots/liquidator/src/monitoring/execution-quorum.ts

@@ -1,14 +1,16 @@
 import { quorumValue } from '@zoltar/bot-shared/monitoring/read-quorum'
 import { ConnectivityDegradedError, operationalFailureDisposition } from '@zoltar/bot-shared/monitoring/resilience'
+import { rpcQuorumRequirement } from '@zoltar/bot-shared/monitoring/rpc-quorum-policy'
 
 export function availableExecutionObservations<T, V>(label: string, settled: readonly PromiseSettledResult<T>[], observation: (value: T) => { endpoint: string; value: V }) {
 	const safetyFailure = settled.find(result => result.status === 'rejected' && operationalFailureDisposition(result.reason) === 'safety-paused')
 	if (safetyFailure?.status === 'rejected') throw safetyFailure.reason
 	const available = settled.flatMap(result => (result.status === 'fulfilled' ? [result.value] : []))
-	if (available.length < 2) {
+	const requirement = rpcQuorumRequirement()
+	if (available.length < requirement) {
 		const failures = settled.flatMap(result => (result.status === 'rejected' ? [result.reason instanceof Error ? result.reason.message : String(result.reason)] : []))
-		throw new ConnectivityDegradedError(`${label} requires at least two available independent RPC endpoints${failures.length === 0 ? '' : `: ${failures.join('; ')}`}`)
+		throw new ConnectivityDegradedError(`${label} requires at least ${requirement === 1 ? 'one available RPC endpoint' : 'two available independent RPC endpoints'}${failures.length === 0 ? '' : `: ${failures.join('; ')}`}`)
 	}
-	quorumValue(label, available.map(observation))
+	quorumValue(label, available.map(observation), requirement)
 	return available
 }

bots/liquidator/tests/config/settings.test.ts

@@ -119,6 +119,17 @@ describe('liquidator settings', () => {
 		).toThrow('name and chainId must identify the same supported chain')
 	})
 
+	test('rejects an explicitly empty RPC quorum policy during settings parsing', () => {
+		const previous = process.env['ZOLTAR_BOT_RPC_QUORUM']
+		try {
+			process.env['ZOLTAR_BOT_RPC_QUORUM'] = ''
+			expect(() => parseSettings(settings)).toThrow('ZOLTAR_BOT_RPC_QUORUM must be 1 or 2')
+		} finally {
+			if (previous === undefined) delete process.env['ZOLTAR_BOT_RPC_QUORUM']
+			else process.env['ZOLTAR_BOT_RPC_QUORUM'] = previous
+		}
+	})
+
 	test('requires independent quorum reads for live execution', () => {
 		expect(() =>
 			parseSettings({