fix(eval): clean up eval configs and add test-clado-api script (#540)

Consolidate 13 configs down to 7 with uniform settings:
- 3 weekly (CI): browseros-agent, browseros-oe-agent, browseros-oe-clado
- 4 test (local): test_gemini-computer-use, test_yutori-navigator, test_webvoyager, test_mind2web
- All configs: headless=false, captcha block, full browseros ports, restart_server_per_task

Deleted: debug-test, mind2web-test, tool-loop-test, orchestrator-executor-test,
orchestrator-executor-clado-test, fireworks-minimax-m2, webvoyager-test

Added: test-clado-api.ts script, browseros-oe-agent-weekly.json (OE with AI SDK executor)

Author: shivammittal274

…/configs/orchestrator-executor-test.json …l/configs/browseros-oe-agent-weekly.jsonpackages/browseros-agent/apps/eval/configs/orchestrator-executor-test.json renamed to packages/browseros-agent/apps/eval/configs/browseros-oe-agent-weekly.json packages/browseros-agent/apps/eval/configs/orchestrator-executor-test.json renamed to packages/browseros-agent/apps/eval/configs/browseros-oe-agent-weekly.json

@@ -2,24 +2,21 @@
   "agent": {
     "type": "orchestrator-executor",
     "orchestrator": {
-      "type": "single",
       "provider": "openai-compatible",
       "model": "accounts/fireworks/models/kimi-k2p5",
       "apiKey": "FIREWORKS_API_KEY",
-      "baseUrl": "https://api.fireworks.ai/inference/v1",
-      "supportsImages": true
+      "baseUrl": "https://api.fireworks.ai/inference/v1"
     },
     "executor": {
       "provider": "openai-compatible",
       "model": "accounts/fireworks/models/kimi-k2p5",
       "apiKey": "FIREWORKS_API_KEY",
-      "baseUrl": "https://api.fireworks.ai/inference/v1",
-      "supportsImages": true
+      "baseUrl": "https://api.fireworks.ai/inference/v1"
     }
   },
-  "dataset": "../data/webvoyager_e2e_test.jsonl",
-  "output_dir": "../results/orchestrator-executor-webvoyager-test",
-  "num_workers": 3,
+  "dataset": "../data/webbench-2of4-50.jsonl",
+  "num_workers": 10,
+  "restart_server_per_task": true,
   "browseros": {
     "server_url": "http://127.0.0.1:9110",
     "base_cdp_port": 9010,
@@ -28,8 +25,12 @@
     "load_extensions": false,
     "headless": false
   },
+  "captcha": {
+    "api_key_env": "NOPECHA_API_KEY"
+  },
+  "graders": ["performance_grader"],
   "grader_api_key_env": "OPENROUTER_API_KEY",
   "grader_base_url": "https://openrouter.ai/api/v1",
   "grader_model": "openai/gpt-4.1",
-  "timeout_ms": 1200000
+  "timeout_ms": 1800000
 }

packages/browseros-agent/apps/eval/configs/browseros-oe-clado-weekly.json

@@ -23,7 +23,10 @@
     "base_server_port": 9110,
     "base_extension_port": 9310,
     "load_extensions": false,
-    "headless": true
+    "headless": false
+  },
+  "captcha": {
+    "api_key_env": "NOPECHA_API_KEY"
   },
   "graders": ["performance_grader"],
   "grader_api_key_env": "OPENROUTER_API_KEY",

packages/browseros-agent/apps/eval/configs/debug-test.json

@@ -9,12 +9,20 @@
     "turnLimit": 100
   },
   "dataset": "../data/test-set.jsonl",
-  "output_dir": "../results/gemini-computer-use-test-set2",
   "num_workers": 1,
   "restart_server_per_task": true,
   "browseros": {
-    "server_url": "http://127.0.0.1:9110"
+    "server_url": "http://127.0.0.1:9110",
+    "base_cdp_port": 9010,
+    "base_server_port": 9110,
+    "base_extension_port": 9310,
+    "load_extensions": false,
+    "headless": false
   },
+  "captcha": {
+    "api_key_env": "NOPECHA_API_KEY"
+  },
+  "graders": ["performance_grader"],
   "grader_api_key_env": "OPENROUTER_API_KEY",
   "grader_base_url": "https://openrouter.ai/api/v1",
   "grader_model": "openai/gpt-4.1",

packages/browseros-agent/apps/eval/configs/fireworks-minimax-m2.json

@@ -6,11 +6,20 @@
     "apiKey": "OPENROUTER_API_KEY"
   },
   "dataset": "../data/mind2web.jsonl",
-  "output_dir": "../results/mind2web-full",
   "num_workers": 5,
+  "restart_server_per_task": true,
   "browseros": {
-    "server_url": "http://127.0.0.1:9110"
+    "server_url": "http://127.0.0.1:9110",
+    "base_cdp_port": 9010,
+    "base_server_port": 9110,
+    "base_extension_port": 9310,
+    "load_extensions": false,
+    "headless": false
   },
+  "captcha": {
+    "api_key_env": "NOPECHA_API_KEY"
+  },
+  "graders": ["performance_grader"],
   "grader_api_key_env": "OPENROUTER_API_KEY",
   "grader_base_url": "https://openrouter.ai/api/v1",
   "grader_model": "openai/gpt-4.1",

packages/browseros-agent/apps/eval/configs/mind2web-test.json

@@ -8,16 +8,20 @@
     "supportsImages": true
   },
   "dataset": "../data/webvoyager.jsonl",
-  "output_dir": "../results/webvoyager-cdp-server",
   "num_workers": 3,
+  "restart_server_per_task": true,
   "browseros": {
     "server_url": "http://127.0.0.1:9110",
     "base_cdp_port": 9010,
     "base_server_port": 9110,
     "base_extension_port": 9310,
     "load_extensions": false,
-    "headless": true
+    "headless": false
   },
+  "captcha": {
+    "api_key_env": "NOPECHA_API_KEY"
+  },
+  "graders": ["performance_grader"],
   "grader_api_key_env": "OPENROUTER_API_KEY",
   "grader_base_url": "https://openrouter.ai/api/v1",
   "grader_model": "openai/gpt-4.1",

packages/browseros-agent/apps/eval/configs/orchestrator-executor-clado-test.json

@@ -9,14 +9,22 @@
     "turnLimit": 100
   },
   "dataset": "../data/test-set.jsonl",
-  "output_dir": "../results/yutori-navigator",
   "num_workers": 1,
   "restart_server_per_task": true,
   "browseros": {
-    "server_url": "http://127.0.0.1:9110"
+    "server_url": "http://127.0.0.1:9110",
+    "base_cdp_port": 9010,
+    "base_server_port": 9110,
+    "base_extension_port": 9310,
+    "load_extensions": false,
+    "headless": false
   },
-  "timeout_ms": 1200000,
+  "captcha": {
+    "api_key_env": "NOPECHA_API_KEY"
+  },
+  "graders": ["performance_grader"],
   "grader_api_key_env": "OPENROUTER_API_KEY",
   "grader_base_url": "https://openrouter.ai/api/v1",
-  "grader_model": "openai/gpt-4.1"
+  "grader_model": "openai/gpt-4.1",
+  "timeout_ms": 1200000
 }