|
| 1 | +""" |
| 2 | +Single-episode debug runner for the spy game. |
| 3 | +
|
| 4 | +Plays one full game with all players backed by qwen-max via DashScope, then |
| 5 | +prints a focused diagnostic report: |
| 6 | +
|
| 7 | + * the per-team accumulated penalties |
| 8 | + * every auto-elimination (repetition rule) that fired, with the shared words |
| 9 | + * every leak that fired, with the offending player + word |
| 10 | + * a verbatim-repetition spot check that re-runs the rule's extractor on the |
| 11 | + descriptions we actually saw, so we can verify the rule is actually |
| 12 | + capable of catching what we eyeballed |
| 13 | +
|
| 14 | +Run: |
| 15 | + DASHSCOPE_API_KEY=... python -m tutorial.opencode_build_spy_game.agent_roll_single_episode |
| 16 | +""" |
| 17 | + |
| 18 | +import json |
| 19 | +import os |
| 20 | +import random |
| 21 | +import sys |
| 22 | +from pathlib import Path |
| 23 | +from typing import Dict, List |
| 24 | + |
| 25 | +from tutorial.opencode_build_spy_game.game_engine import ( |
| 26 | + SpyGame, |
| 27 | + extract_content_words, |
| 28 | + REPEAT_LIMIT, |
| 29 | + LEAK_PENALTY, |
| 30 | + WIN_REWARD, |
| 31 | + _description_leaks_word, |
| 32 | +) |
| 33 | + |
| 34 | + |
| 35 | +PLAYER_NAMES = [ |
| 36 | + "Alexander", "Benjamin", "Christopher", "Daniel", "Elizabeth", |
| 37 | + "Fitzgerald", "Gabriella", "Harrison", "Isabella", "Jonathan", |
| 38 | + "Katherine", "Leonardo", "Margaret", "Nathaniel", "Ophelia", |
| 39 | + "Penelope", "Quentin", "Rosalind", "Sebastian", "Theodora", |
| 40 | + "Ulysses", "Victoria", "Wellington", "Xander", "Yasmine", |
| 41 | + "Zachary", "Adelaide", "Beatrice", "Cornelius", "Desmond", |
| 42 | + "Eleanor", "Frederick", "Genevieve", "Humphrey", "Imogen", |
| 43 | + "Jasper", "Lillian", "Maximilian", "Nicolette", "Orlando", |
| 44 | + "Percival", "Quintessa", "Reginald", "Seraphina", "Tristan", |
| 45 | + "Valentina", "Winifred", "Xavier", "Yolanda", "Zephyr" |
| 46 | +] |
| 47 | + |
| 48 | +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1" |
| 49 | +DASHSCOPE_MODEL = "qwen-max" |
| 50 | + |
| 51 | + |
| 52 | +def _selftest_extractor() -> None: |
| 53 | + """Re-run the repetition rule on the exact verbatim copies from the user's |
| 54 | + transcript, to prove the extractor would fire if it were called.""" |
| 55 | + print("\n" + "=" * 70) |
| 56 | + print("SELFTEST: extractor on the user's round-3 verbatim copies") |
| 57 | + print("=" * 70) |
| 58 | + sentence = "It's a versatile item that pairs well with spreads for a quick and satisfying meal." |
| 59 | + a = extract_content_words(sentence) |
| 60 | + b = extract_content_words(sentence) |
| 61 | + overlap = sorted(a & b) |
| 62 | + print(f"sentence : {sentence!r}") |
| 63 | + print(f"content : {sorted(a)}") |
| 64 | + print(f"overlap : {overlap} (count={len(overlap)}, limit={REPEAT_LIMIT})") |
| 65 | + print(f"would fire: {len(overlap) > REPEAT_LIMIT}") |
| 66 | + |
| 67 | + |
| 68 | +def _selftest_leak() -> None: |
| 69 | + print("\n" + "=" * 70) |
| 70 | + print("SELFTEST: leak detector") |
| 71 | + print("=" * 70) |
| 72 | + cases = [ |
| 73 | + ("bread", "I love bread for breakfast.", True), |
| 74 | + ("bread", "I love breadcrumbs for breakfast.", False), |
| 75 | + ("bread", "I love BREAD for breakfast.", True), |
| 76 | + ("bread", "Tasty bread, sliced.", True), |
| 77 | + ("cake", "It's a versatile item.", False), |
| 78 | + ] |
| 79 | + for word, desc, expect in cases: |
| 80 | + got = _description_leaks_word(desc, word) |
| 81 | + ok = "OK " if got == expect else "FAIL" |
| 82 | + print(f" [{ok}] word={word!r:10s} desc={desc!r:55s} -> {got} (expected {expect})") |
| 83 | + |
| 84 | + |
| 85 | +def _build_player_configs(num_players: int, names: List[str]) -> List[Dict]: |
| 86 | + api_key = os.environ.get("DASHSCOPE_API_KEY") |
| 87 | + if not api_key: |
| 88 | + raise SystemExit("DASHSCOPE_API_KEY is not set; cannot run debug episode.") |
| 89 | + return [ |
| 90 | + { |
| 91 | + "name": names[i], |
| 92 | + "base_url": DASHSCOPE_BASE_URL, |
| 93 | + "api_key": api_key, |
| 94 | + "model": DASHSCOPE_MODEL, |
| 95 | + } |
| 96 | + for i in range(num_players) |
| 97 | + ] |
| 98 | + |
| 99 | + |
| 100 | +def _summarise(result: Dict) -> None: |
| 101 | + print("\n" + "#" * 70) |
| 102 | + print("DIAGNOSTIC SUMMARY") |
| 103 | + print("#" * 70) |
| 104 | + print(f"winner : {result['winner']}") |
| 105 | + print(f"aborted_by_role : {result['aborted_by_role']}") |
| 106 | + print(f"total_rounds : {result['total_rounds']}") |
| 107 | + print(f"final_alive : {result['final_alive']}") |
| 108 | + print(f"civilian_reward : {result['civilian_reward']:+.4f}") |
| 109 | + print(f"spy_reward : {result['spy_reward']:+.4f}") |
| 110 | + print(f"team_penalties : {result['team_penalties']}") |
| 111 | + print(f"reward constants : WIN_REWARD={WIN_REWARD:+.2f} LEAK_PENALTY={LEAK_PENALTY:+.2f} REPEAT_LIMIT={REPEAT_LIMIT}") |
| 112 | + |
| 113 | + auto_elims = [ |
| 114 | + e for e in result["game_history"] |
| 115 | + if e.get("type") == "elimination" and "Auto-eliminated" in (e.get("reason") or "") |
| 116 | + ] |
| 117 | + leaks = [ |
| 118 | + e for e in result["game_history"] |
| 119 | + if e.get("type") == "elimination" and e.get("aborted") |
| 120 | + ] |
| 121 | + vote_elims = [ |
| 122 | + e for e in result["game_history"] |
| 123 | + if e.get("type") == "elimination" |
| 124 | + and e.get("eliminated_name") is not None |
| 125 | + and not e.get("aborted") |
| 126 | + and "Auto-eliminated" not in (e.get("reason") or "") |
| 127 | + ] |
| 128 | + |
| 129 | + print(f"\nrule firings: auto-elims={len(auto_elims)} leaks={len(leaks)} vote-elims={len(vote_elims)}") |
| 130 | + for e in auto_elims: |
| 131 | + print(f" AUTO-ELIM round {e['round']}: {e['eliminated_name']} ({e['eliminated_role']}) | {e['reason']}") |
| 132 | + for e in leaks: |
| 133 | + print(f" LEAK round {e['round']}: {e['eliminated_name']} ({e['eliminated_role']}) | {e['reason']}") |
| 134 | + |
| 135 | + # Cross-check: scan every description against earlier-in-round descriptions |
| 136 | + # using the exact same extractor the engine uses, and report any pair the |
| 137 | + # engine should have caught but didn't. |
| 138 | + print("\ncross-check (recomputing repetition overlap on stored history):") |
| 139 | + by_round: Dict[int, List[Dict]] = {} |
| 140 | + for entry in result["game_history"]: |
| 141 | + if entry.get("type") == "description": |
| 142 | + by_round.setdefault(entry["round"], []).append(entry) |
| 143 | + missed = 0 |
| 144 | + for r, entries in by_round.items(): |
| 145 | + prior: set = set() |
| 146 | + for e in entries: |
| 147 | + own = extract_content_words(e["description"]) |
| 148 | + shared = sorted(own & prior) |
| 149 | + stored = e.get("repeated_words", []) |
| 150 | + marker = "" |
| 151 | + if len(shared) > REPEAT_LIMIT and not stored: |
| 152 | + marker = " <-- WOULD HAVE FIRED but engine recorded none" |
| 153 | + missed += 1 |
| 154 | + elif sorted(stored) != shared: |
| 155 | + marker = f" <-- mismatch: engine recorded {stored}" |
| 156 | + print( |
| 157 | + f" r{r} {e['player_name']:13s} stored_repeats={len(stored):2d} " |
| 158 | + f"recompute_overlap={len(shared):2d} {shared}{marker}" |
| 159 | + ) |
| 160 | + prior |= own |
| 161 | + if missed: |
| 162 | + print(f"\n!!! repetition rule missed {missed} firings -- the engine is buggy") |
| 163 | + else: |
| 164 | + print("\nall repetitions detected by recompute were also caught by the engine.") |
| 165 | + |
| 166 | + |
| 167 | +def main(argv: List[str]) -> int: |
| 168 | + # Allow picking a task from the mock dataset, or default to "bread vs cake" |
| 169 | + # which matches the transcript the user shared. |
| 170 | + dataset_path = Path(__file__).with_name("mock_game_dataset.json") |
| 171 | + tasks = json.loads(dataset_path.read_text()) |
| 172 | + task = next( |
| 173 | + (t for t in tasks if t["civilian_word"] == "bread" and t["spy_word"] == "cake"), |
| 174 | + tasks[0], |
| 175 | + ) |
| 176 | + if len(argv) > 1: |
| 177 | + idx = int(argv[1]) |
| 178 | + task = tasks[idx] |
| 179 | + |
| 180 | + seed = int(os.environ.get("EPISODE_SEED", "0")) |
| 181 | + random.seed(seed) |
| 182 | + print(f"task = {task} seed = {seed}") |
| 183 | + |
| 184 | + _selftest_leak() |
| 185 | + _selftest_extractor() |
| 186 | + |
| 187 | + names = random.sample(PLAYER_NAMES, task["num_players"]) |
| 188 | + configs = _build_player_configs(task["num_players"], names) |
| 189 | + |
| 190 | + game = SpyGame( |
| 191 | + civilian_word=task["civilian_word"], |
| 192 | + spy_word=task["spy_word"], |
| 193 | + num_players=task["num_players"], |
| 194 | + num_spies=task["num_spies"], |
| 195 | + player_configs=configs, |
| 196 | + ) |
| 197 | + result = game.play_game() |
| 198 | + _summarise(result) |
| 199 | + return 0 |
| 200 | + |
| 201 | + |
| 202 | +if __name__ == "__main__": |
| 203 | + sys.exit(main(sys.argv)) |
0 commit comments