Skip to content

Commit b654ffd

Browse files
committed
improve spy game logic
1 parent d31e616 commit b654ffd

7 files changed

Lines changed: 1116 additions & 186 deletions

File tree

tutorial/opencode_build_aime/auto_research/auto_train.py

Lines changed: 410 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
CONFIG_PATH="/mnt/data_cpfs/qingxu.fu/alpha_auto_research/research_config.jsonc"
5+
PYTHON_BIN="${PYTHON_BIN:-python3}"
6+
7+
eval "$("${PYTHON_BIN}" - "${CONFIG_PATH}" <<'PY'
8+
import json
9+
import shlex
10+
import sys
11+
from pathlib import Path
12+
13+
14+
def strip_jsonc_comments(text: str) -> str:
15+
out = []
16+
in_string = False
17+
escape = False
18+
i = 0
19+
while i < len(text):
20+
ch = text[i]
21+
nxt = text[i + 1] if i + 1 < len(text) else ""
22+
if in_string:
23+
out.append(ch)
24+
if escape:
25+
escape = False
26+
elif ch == "\\":
27+
escape = True
28+
elif ch == '"':
29+
in_string = False
30+
i += 1
31+
continue
32+
if ch == '"':
33+
in_string = True
34+
out.append(ch)
35+
i += 1
36+
continue
37+
if ch == "/" and nxt == "/":
38+
while i < len(text) and text[i] != "\n":
39+
i += 1
40+
continue
41+
out.append(ch)
42+
i += 1
43+
return "".join(out)
44+
45+
46+
config_path = Path(sys.argv[1])
47+
text = strip_jsonc_comments(config_path.read_text(encoding="utf-8"))
48+
config = json.loads(text)
49+
50+
for key in ("SWANLAB_WEB_HOST", "SWANLAB_API_KEY", "SWANLAB_API_HOST"):
51+
value = config["swanlab"][key]
52+
print(f"export {key}={shlex.quote(value)}")
53+
PY
54+
)"

tutorial/opencode_build_spy_game/agent_roll.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def main():
6565

6666
# Configure and start training
6767
ajet_job = AgentJetJob(
68+
ensure_new_experiment=True,
6869
algorithm="grpo",
6970
project_name="spy-game-rl",
7071
logging="swanlab",
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
"""
2+
Single-episode debug runner for the spy game.
3+
4+
Plays one full game with all players backed by qwen-max via DashScope, then
5+
prints a focused diagnostic report:
6+
7+
* the per-team accumulated penalties
8+
* every auto-elimination (repetition rule) that fired, with the shared words
9+
* every leak that fired, with the offending player + word
10+
* a verbatim-repetition spot check that re-runs the rule's extractor on the
11+
descriptions we actually saw, so we can verify the rule is actually
12+
capable of catching what we eyeballed
13+
14+
Run:
15+
DASHSCOPE_API_KEY=... python -m tutorial.opencode_build_spy_game.agent_roll_single_episode
16+
"""
17+
18+
import json
19+
import os
20+
import random
21+
import sys
22+
from pathlib import Path
23+
from typing import Dict, List
24+
25+
from tutorial.opencode_build_spy_game.game_engine import (
26+
SpyGame,
27+
extract_content_words,
28+
REPEAT_LIMIT,
29+
LEAK_PENALTY,
30+
WIN_REWARD,
31+
_description_leaks_word,
32+
)
33+
34+
35+
PLAYER_NAMES = [
36+
"Alexander", "Benjamin", "Christopher", "Daniel", "Elizabeth",
37+
"Fitzgerald", "Gabriella", "Harrison", "Isabella", "Jonathan",
38+
"Katherine", "Leonardo", "Margaret", "Nathaniel", "Ophelia",
39+
"Penelope", "Quentin", "Rosalind", "Sebastian", "Theodora",
40+
"Ulysses", "Victoria", "Wellington", "Xander", "Yasmine",
41+
"Zachary", "Adelaide", "Beatrice", "Cornelius", "Desmond",
42+
"Eleanor", "Frederick", "Genevieve", "Humphrey", "Imogen",
43+
"Jasper", "Lillian", "Maximilian", "Nicolette", "Orlando",
44+
"Percival", "Quintessa", "Reginald", "Seraphina", "Tristan",
45+
"Valentina", "Winifred", "Xavier", "Yolanda", "Zephyr"
46+
]
47+
48+
DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
49+
DASHSCOPE_MODEL = "qwen-max"
50+
51+
52+
def _selftest_extractor() -> None:
53+
"""Re-run the repetition rule on the exact verbatim copies from the user's
54+
transcript, to prove the extractor would fire if it were called."""
55+
print("\n" + "=" * 70)
56+
print("SELFTEST: extractor on the user's round-3 verbatim copies")
57+
print("=" * 70)
58+
sentence = "It's a versatile item that pairs well with spreads for a quick and satisfying meal."
59+
a = extract_content_words(sentence)
60+
b = extract_content_words(sentence)
61+
overlap = sorted(a & b)
62+
print(f"sentence : {sentence!r}")
63+
print(f"content : {sorted(a)}")
64+
print(f"overlap : {overlap} (count={len(overlap)}, limit={REPEAT_LIMIT})")
65+
print(f"would fire: {len(overlap) > REPEAT_LIMIT}")
66+
67+
68+
def _selftest_leak() -> None:
69+
print("\n" + "=" * 70)
70+
print("SELFTEST: leak detector")
71+
print("=" * 70)
72+
cases = [
73+
("bread", "I love bread for breakfast.", True),
74+
("bread", "I love breadcrumbs for breakfast.", False),
75+
("bread", "I love BREAD for breakfast.", True),
76+
("bread", "Tasty bread, sliced.", True),
77+
("cake", "It's a versatile item.", False),
78+
]
79+
for word, desc, expect in cases:
80+
got = _description_leaks_word(desc, word)
81+
ok = "OK " if got == expect else "FAIL"
82+
print(f" [{ok}] word={word!r:10s} desc={desc!r:55s} -> {got} (expected {expect})")
83+
84+
85+
def _build_player_configs(num_players: int, names: List[str]) -> List[Dict]:
86+
api_key = os.environ.get("DASHSCOPE_API_KEY")
87+
if not api_key:
88+
raise SystemExit("DASHSCOPE_API_KEY is not set; cannot run debug episode.")
89+
return [
90+
{
91+
"name": names[i],
92+
"base_url": DASHSCOPE_BASE_URL,
93+
"api_key": api_key,
94+
"model": DASHSCOPE_MODEL,
95+
}
96+
for i in range(num_players)
97+
]
98+
99+
100+
def _summarise(result: Dict) -> None:
101+
print("\n" + "#" * 70)
102+
print("DIAGNOSTIC SUMMARY")
103+
print("#" * 70)
104+
print(f"winner : {result['winner']}")
105+
print(f"aborted_by_role : {result['aborted_by_role']}")
106+
print(f"total_rounds : {result['total_rounds']}")
107+
print(f"final_alive : {result['final_alive']}")
108+
print(f"civilian_reward : {result['civilian_reward']:+.4f}")
109+
print(f"spy_reward : {result['spy_reward']:+.4f}")
110+
print(f"team_penalties : {result['team_penalties']}")
111+
print(f"reward constants : WIN_REWARD={WIN_REWARD:+.2f} LEAK_PENALTY={LEAK_PENALTY:+.2f} REPEAT_LIMIT={REPEAT_LIMIT}")
112+
113+
auto_elims = [
114+
e for e in result["game_history"]
115+
if e.get("type") == "elimination" and "Auto-eliminated" in (e.get("reason") or "")
116+
]
117+
leaks = [
118+
e for e in result["game_history"]
119+
if e.get("type") == "elimination" and e.get("aborted")
120+
]
121+
vote_elims = [
122+
e for e in result["game_history"]
123+
if e.get("type") == "elimination"
124+
and e.get("eliminated_name") is not None
125+
and not e.get("aborted")
126+
and "Auto-eliminated" not in (e.get("reason") or "")
127+
]
128+
129+
print(f"\nrule firings: auto-elims={len(auto_elims)} leaks={len(leaks)} vote-elims={len(vote_elims)}")
130+
for e in auto_elims:
131+
print(f" AUTO-ELIM round {e['round']}: {e['eliminated_name']} ({e['eliminated_role']}) | {e['reason']}")
132+
for e in leaks:
133+
print(f" LEAK round {e['round']}: {e['eliminated_name']} ({e['eliminated_role']}) | {e['reason']}")
134+
135+
# Cross-check: scan every description against earlier-in-round descriptions
136+
# using the exact same extractor the engine uses, and report any pair the
137+
# engine should have caught but didn't.
138+
print("\ncross-check (recomputing repetition overlap on stored history):")
139+
by_round: Dict[int, List[Dict]] = {}
140+
for entry in result["game_history"]:
141+
if entry.get("type") == "description":
142+
by_round.setdefault(entry["round"], []).append(entry)
143+
missed = 0
144+
for r, entries in by_round.items():
145+
prior: set = set()
146+
for e in entries:
147+
own = extract_content_words(e["description"])
148+
shared = sorted(own & prior)
149+
stored = e.get("repeated_words", [])
150+
marker = ""
151+
if len(shared) > REPEAT_LIMIT and not stored:
152+
marker = " <-- WOULD HAVE FIRED but engine recorded none"
153+
missed += 1
154+
elif sorted(stored) != shared:
155+
marker = f" <-- mismatch: engine recorded {stored}"
156+
print(
157+
f" r{r} {e['player_name']:13s} stored_repeats={len(stored):2d} "
158+
f"recompute_overlap={len(shared):2d} {shared}{marker}"
159+
)
160+
prior |= own
161+
if missed:
162+
print(f"\n!!! repetition rule missed {missed} firings -- the engine is buggy")
163+
else:
164+
print("\nall repetitions detected by recompute were also caught by the engine.")
165+
166+
167+
def main(argv: List[str]) -> int:
168+
# Allow picking a task from the mock dataset, or default to "bread vs cake"
169+
# which matches the transcript the user shared.
170+
dataset_path = Path(__file__).with_name("mock_game_dataset.json")
171+
tasks = json.loads(dataset_path.read_text())
172+
task = next(
173+
(t for t in tasks if t["civilian_word"] == "bread" and t["spy_word"] == "cake"),
174+
tasks[0],
175+
)
176+
if len(argv) > 1:
177+
idx = int(argv[1])
178+
task = tasks[idx]
179+
180+
seed = int(os.environ.get("EPISODE_SEED", "0"))
181+
random.seed(seed)
182+
print(f"task = {task} seed = {seed}")
183+
184+
_selftest_leak()
185+
_selftest_extractor()
186+
187+
names = random.sample(PLAYER_NAMES, task["num_players"])
188+
configs = _build_player_configs(task["num_players"], names)
189+
190+
game = SpyGame(
191+
civilian_word=task["civilian_word"],
192+
spy_word=task["spy_word"],
193+
num_players=task["num_players"],
194+
num_spies=task["num_spies"],
195+
player_configs=configs,
196+
)
197+
result = game.play_game()
198+
_summarise(result)
199+
return 0
200+
201+
202+
if __name__ == "__main__":
203+
sys.exit(main(sys.argv))

tutorial/opencode_build_spy_game/agent_run.py

Lines changed: 20 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -148,32 +148,23 @@ def run_agent_and_compute_reward(task: Task, base_url: str, api_key: str) -> Wor
148148
"""
149149
api_baseurl_key = OpenaiBaseUrlAndApiKey(base_url=base_url, api_key=api_key)
150150

151-
try:
152-
# Execute game
153-
game_result = _execute_agent(task, api_baseurl_key)
154-
155-
# Compute reward (1.0 if civilians win, 0.0 if spies win)
156-
reward = _compute_reward(game_result)
157-
158-
# Return workflow output
159-
return WorkflowOutput(
160-
reward=reward,
161-
metadata={
162-
"winner": game_result["winner"],
163-
"total_rounds": game_result["total_rounds"],
164-
"civilian_word": game_result["civilian_word"],
165-
"spy_word": game_result["spy_word"],
166-
"final_alive": game_result["final_alive"]
167-
}
168-
)
169-
170-
except Exception as e:
171-
print(f"Error during game execution: {e}")
172-
# Return 0 reward on failure
173-
return WorkflowOutput(
174-
reward=0.0,
175-
metadata={
176-
"error": str(e),
177-
"winner": "error"
178-
}
179-
)
151+
# No internal try/except: any failure (NLTK missing, malformed model output,
152+
# API timeout) propagates to the training loop, which already has its own
153+
# except/return-None path. A second silent layer was hiding bugs (e.g. the
154+
# NLTK silent-disable that collapsed all rewards to 0/1).
155+
game_result = _execute_agent(task, api_baseurl_key)
156+
reward = _compute_reward(game_result)
157+
return WorkflowOutput(
158+
reward=reward,
159+
metadata={
160+
"winner": game_result["winner"],
161+
"aborted_by_role": game_result.get("aborted_by_role"),
162+
"team_penalties": game_result.get("team_penalties"),
163+
"civilian_reward": game_result["civilian_reward"],
164+
"spy_reward": game_result["spy_reward"],
165+
"total_rounds": game_result["total_rounds"],
166+
"civilian_word": game_result["civilian_word"],
167+
"spy_word": game_result["spy_word"],
168+
"final_alive": game_result["final_alive"],
169+
},
170+
)

tutorial/opencode_build_spy_game/agent_run_adv.py

Lines changed: 27 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -170,49 +170,30 @@ def run_agent_and_compute_reward(
170170
api_key=api_key_spies
171171
)
172172

173-
try:
174-
# Execute game
175-
game_result = _execute_agent(task, api_baseurl_key_civilians, api_baseurl_key_spies)
176-
177-
# Compute rewards for both teams
178-
civilian_reward, spy_reward = _compute_rewards(game_result)
179-
180-
# Create separate workflow outputs for each team
181-
workflow_output_civilians = WorkflowOutput(
182-
reward=civilian_reward,
183-
metadata={
184-
"team": "civilians",
185-
"winner": game_result["winner"],
186-
"total_rounds": game_result["total_rounds"],
187-
"civilian_word": game_result["civilian_word"],
188-
"spy_word": game_result["spy_word"],
189-
"final_alive": game_result["final_alive"]
190-
}
191-
)
192-
193-
workflow_output_spies = WorkflowOutput(
194-
reward=spy_reward,
195-
metadata={
196-
"team": "spies",
197-
"winner": game_result["winner"],
198-
"total_rounds": game_result["total_rounds"],
199-
"civilian_word": game_result["civilian_word"],
200-
"spy_word": game_result["spy_word"],
201-
"final_alive": game_result["final_alive"]
202-
}
203-
)
204-
205-
return workflow_output_civilians, workflow_output_spies
206-
207-
except Exception as e:
208-
print(f"Error during adversarial game execution: {e}")
209-
# Return neutral rewards on failure
210-
error_output_civilians = WorkflowOutput(
211-
reward=0.5,
212-
metadata={"error": str(e), "winner": "error", "team": "civilians"}
213-
)
214-
error_output_spies = WorkflowOutput(
215-
reward=0.5,
216-
metadata={"error": str(e), "winner": "error", "team": "spies"}
217-
)
218-
return error_output_civilians, error_output_spies
173+
# No internal try/except: any failure propagates to the training loop's
174+
# own except/return-None path. The previous fallback returned reward=0.5,
175+
# which under the new reward scheme (loss=0, win=1) is HIGHER than losing
176+
# -- failed rollouts were being silently rewarded above legitimate losses.
177+
game_result = _execute_agent(task, api_baseurl_key_civilians, api_baseurl_key_spies)
178+
civilian_reward, spy_reward = _compute_rewards(game_result)
179+
180+
common_meta = {
181+
"winner": game_result["winner"],
182+
"aborted_by_role": game_result.get("aborted_by_role"),
183+
"team_penalties": game_result.get("team_penalties"),
184+
"civilian_reward": game_result["civilian_reward"],
185+
"spy_reward": game_result["spy_reward"],
186+
"total_rounds": game_result["total_rounds"],
187+
"civilian_word": game_result["civilian_word"],
188+
"spy_word": game_result["spy_word"],
189+
"final_alive": game_result["final_alive"],
190+
}
191+
workflow_output_civilians = WorkflowOutput(
192+
reward=civilian_reward,
193+
metadata={"team": "civilians", **common_meta},
194+
)
195+
workflow_output_spies = WorkflowOutput(
196+
reward=spy_reward,
197+
metadata={"team": "spies", **common_meta},
198+
)
199+
return workflow_output_civilians, workflow_output_spies

0 commit comments

Comments
 (0)