Merge pull request #1311 from codeflash-ai/fix/dynamic-env-var-reading

fix: use getter functions for perf env vars to support Vitest module caching

Author: Saga4

packages/codeflash/runtime/__tests__/dynamic-env-vars.test.js

@@ -0,0 +1,73 @@
+/**
+ * Test: Dynamic environment variable reading
+ *
+ * This test verifies that the performance configuration functions read
+ * environment variables at runtime rather than at module load time.
+ *
+ * This is critical for Vitest compatibility, where modules may be cached
+ * and loaded before environment variables are set.
+ *
+ * Run with: node __tests__/dynamic-env-vars.test.js
+ */
+
+const assert = require('assert');
+
+// Clear any existing env vars before loading the module
+delete process.env.CODEFLASH_PERF_LOOP_COUNT;
+delete process.env.CODEFLASH_PERF_MIN_LOOPS;
+delete process.env.CODEFLASH_PERF_TARGET_DURATION_MS;
+delete process.env.CODEFLASH_PERF_BATCH_SIZE;
+delete process.env.CODEFLASH_PERF_STABILITY_CHECK;
+delete process.env.CODEFLASH_PERF_CURRENT_BATCH;
+
+// Now load the module - at this point env vars are not set
+const capture = require('../capture');
+
+console.log('Testing dynamic environment variable reading...\n');
+
+// Test 1: Default values when env vars are not set
+console.log('Test 1: Default values');
+assert.strictEqual(capture.getPerfLoopCount(), 1, 'getPerfLoopCount default should be 1');
+assert.strictEqual(capture.getPerfMinLoops(), 5, 'getPerfMinLoops default should be 5');
+assert.strictEqual(capture.getPerfTargetDurationMs(), 10000, 'getPerfTargetDurationMs default should be 10000');
+assert.strictEqual(capture.getPerfBatchSize(), 10, 'getPerfBatchSize default should be 10');
+assert.strictEqual(capture.getPerfStabilityCheck(), false, 'getPerfStabilityCheck default should be false');
+assert.strictEqual(capture.getPerfCurrentBatch(), 0, 'getPerfCurrentBatch default should be 0');
+console.log('  PASS: All defaults correct\n');
+
+// Test 2: Values change when env vars are set AFTER module load
+// This is the critical test - if these were constants, they would still return defaults
+console.log('Test 2: Dynamic reading after module load');
+process.env.CODEFLASH_PERF_LOOP_COUNT = '100';
+process.env.CODEFLASH_PERF_MIN_LOOPS = '10';
+process.env.CODEFLASH_PERF_TARGET_DURATION_MS = '5000';
+process.env.CODEFLASH_PERF_BATCH_SIZE = '20';
+process.env.CODEFLASH_PERF_STABILITY_CHECK = 'true';
+process.env.CODEFLASH_PERF_CURRENT_BATCH = '5';
+
+assert.strictEqual(capture.getPerfLoopCount(), 100, 'getPerfLoopCount should read 100 from env');
+assert.strictEqual(capture.getPerfMinLoops(), 10, 'getPerfMinLoops should read 10 from env');
+assert.strictEqual(capture.getPerfTargetDurationMs(), 5000, 'getPerfTargetDurationMs should read 5000 from env');
+assert.strictEqual(capture.getPerfBatchSize(), 20, 'getPerfBatchSize should read 20 from env');
+assert.strictEqual(capture.getPerfStabilityCheck(), true, 'getPerfStabilityCheck should read true from env');
+assert.strictEqual(capture.getPerfCurrentBatch(), 5, 'getPerfCurrentBatch should read 5 from env');
+console.log('  PASS: Dynamic reading works correctly\n');
+
+// Test 3: Values change again when env vars are modified
+console.log('Test 3: Values update when env vars change');
+process.env.CODEFLASH_PERF_LOOP_COUNT = '500';
+process.env.CODEFLASH_PERF_BATCH_SIZE = '50';
+
+assert.strictEqual(capture.getPerfLoopCount(), 500, 'getPerfLoopCount should update to 500');
+assert.strictEqual(capture.getPerfBatchSize(), 50, 'getPerfBatchSize should update to 50');
+console.log('  PASS: Values update correctly\n');
+
+// Cleanup
+delete process.env.CODEFLASH_PERF_LOOP_COUNT;
+delete process.env.CODEFLASH_PERF_MIN_LOOPS;
+delete process.env.CODEFLASH_PERF_TARGET_DURATION_MS;
+delete process.env.CODEFLASH_PERF_BATCH_SIZE;
+delete process.env.CODEFLASH_PERF_STABILITY_CHECK;
+delete process.env.CODEFLASH_PERF_CURRENT_BATCH;
+
+console.log('All tests passed!');

packages/codeflash/runtime/capture.js

@@ -47,14 +47,30 @@ const TEST_MODULE = process.env.CODEFLASH_TEST_MODULE;
 //   Batch 1: Test1(5 loops) → Test2(5 loops) → Test3(5 loops)
 //   Batch 2: Test1(5 loops) → Test2(5 loops) → Test3(5 loops)
 //   ...until time budget exhausted
-const PERF_LOOP_COUNT = parseInt(process.env.CODEFLASH_PERF_LOOP_COUNT || '1', 10);
-const PERF_MIN_LOOPS = parseInt(process.env.CODEFLASH_PERF_MIN_LOOPS || '5', 10);
-const PERF_TARGET_DURATION_MS = parseInt(process.env.CODEFLASH_PERF_TARGET_DURATION_MS || '10000', 10);
-const PERF_BATCH_SIZE = parseInt(process.env.CODEFLASH_PERF_BATCH_SIZE || '10', 10);
-const PERF_STABILITY_CHECK = (process.env.CODEFLASH_PERF_STABILITY_CHECK || 'false').toLowerCase() === 'true';
+//
+// IMPORTANT: These are getter functions, NOT constants!
+// Vitest caches modules and may load this file before env vars are set.
+// Using getter functions ensures we read the env vars at runtime when they're actually needed.
+function getPerfLoopCount() {
+    return parseInt(process.env.CODEFLASH_PERF_LOOP_COUNT || '1', 10);
+}
+function getPerfMinLoops() {
+    return parseInt(process.env.CODEFLASH_PERF_MIN_LOOPS || '5', 10);
+}
+function getPerfTargetDurationMs() {
+    return parseInt(process.env.CODEFLASH_PERF_TARGET_DURATION_MS || '10000', 10);
+}
+function getPerfBatchSize() {
+    return parseInt(process.env.CODEFLASH_PERF_BATCH_SIZE || '10', 10);
+}
+function getPerfStabilityCheck() {
+    return (process.env.CODEFLASH_PERF_STABILITY_CHECK || 'false').toLowerCase() === 'true';
+}
 // Current batch number - set by loop-runner before each batch
 // This allows continuous loop indices even when Jest resets module state
-const PERF_CURRENT_BATCH = parseInt(process.env.CODEFLASH_PERF_CURRENT_BATCH || '0', 10);
+function getPerfCurrentBatch() {
+    return parseInt(process.env.CODEFLASH_PERF_CURRENT_BATCH || '0', 10);
+}
 
 // Stability constants (matching Python's config_consts.py)
 const STABILITY_WINDOW_SIZE = 0.35;
@@ -86,7 +102,7 @@ function checkSharedTimeLimit() {
         return false;
     }
     const elapsed = Date.now() - sharedPerfState.startTime;
-    if (elapsed >= PERF_TARGET_DURATION_MS && sharedPerfState.totalLoopsCompleted >= PERF_MIN_LOOPS) {
+    if (elapsed >= getPerfTargetDurationMs() && sharedPerfState.totalLoopsCompleted >= getPerfMinLoops()) {
         sharedPerfState.shouldStop = true;
         return true;
     }
@@ -111,7 +127,7 @@ function getInvocationLoopIndex(invocationKey) {
     // Calculate global loop index using batch number from environment
     // PERF_CURRENT_BATCH is 1-based (set by loop-runner before each batch)
     const currentBatch = parseInt(process.env.CODEFLASH_PERF_CURRENT_BATCH || '1', 10);
-    const globalIndex = (currentBatch - 1) * PERF_BATCH_SIZE + localIndex;
+    const globalIndex = (currentBatch - 1) * getPerfBatchSize() + localIndex;
 
     return globalIndex;
 }
@@ -606,7 +622,7 @@ function capture(funcName, lineId, fn, ...args) {
  */
 function capturePerf(funcName, lineId, fn, ...args) {
     // Check if we should skip looping entirely (shared time budget exceeded)
-    const shouldLoop = PERF_LOOP_COUNT > 1 && !checkSharedTimeLimit();
+    const shouldLoop = getPerfLoopCount() > 1 && !checkSharedTimeLimit();
 
     // Get test context (computed once, reused across batch)
     let testModulePath;
@@ -636,9 +652,9 @@ function capturePerf(funcName, lineId, fn, ...args) {
     // If so, just execute the function once without timing (for test assertions)
     const peekLoopIndex = (sharedPerfState.invocationLoopCounts[invocationKey] || 0);
     const currentBatch = parseInt(process.env.CODEFLASH_PERF_CURRENT_BATCH || '1', 10);
-    const nextGlobalIndex = (currentBatch - 1) * PERF_BATCH_SIZE + peekLoopIndex + 1;
+    const nextGlobalIndex = (currentBatch - 1) * getPerfBatchSize() + peekLoopIndex + 1;
 
-    if (shouldLoop && nextGlobalIndex > PERF_LOOP_COUNT) {
+    if (shouldLoop && nextGlobalIndex > getPerfLoopCount()) {
         // All loops completed, just execute once for test assertion
         return fn(...args);
     }
@@ -654,7 +670,7 @@ function capturePerf(funcName, lineId, fn, ...args) {
     // Batched looping: run BATCH_SIZE loops per capturePerf call when using loop-runner
     // For Vitest (no loop-runner), do all loops internally in a single call
     const batchSize = shouldLoop
-        ? (hasExternalLoopRunner ? PERF_BATCH_SIZE : PERF_LOOP_COUNT)
+        ? (hasExternalLoopRunner ? getPerfBatchSize() : getPerfLoopCount())
         : 1;
 
     for (let batchIndex = 0; batchIndex < batchSize; batchIndex++) {
@@ -667,7 +683,7 @@ function capturePerf(funcName, lineId, fn, ...args) {
         const loopIndex = getInvocationLoopIndex(invocationKey);
 
         // Check if we've exceeded max loops for this invocation
-        if (loopIndex > PERF_LOOP_COUNT) {
+        if (loopIndex > getPerfLoopCount()) {
             break;
         }
 
@@ -872,7 +888,11 @@ module.exports = {
     LOOP_INDEX,
     OUTPUT_FILE,
     TEST_ITERATION,
-    // Batch configuration
-    PERF_BATCH_SIZE,
-    PERF_LOOP_COUNT,
+    // Batch configuration (getter functions for dynamic env var reading)
+    getPerfBatchSize,
+    getPerfLoopCount,
+    getPerfMinLoops,
+    getPerfTargetDurationMs,
+    getPerfStabilityCheck,
+    getPerfCurrentBatch,
 };