Skip to content

Commit 0c36537

Browse files
patel-lyzrclaude
andcommitted
fix(eval): capture gitclaw tool calls + hook/policy denials
The tool-compliance scorer was blind to gitagent: gitclaw emits tool calls as TOP-LEVEL sdk_messages (type:"tool_use", toolName/args) and tool results as type:"tool_result", not nested in an assistant message's content blocks like the Claude SDK. So toolCalls/policyDenials came back empty and the scorer scored "compliant" by default. Denials also surface as text ("Tool 'cli' denied"), not the SDK's structured permission_denials. - Parse top-level tool_use/tool_result into the transcript (→ toolCalls). - recordDenial() extracts the blocked tool name from "Tool 'X' denied/blocked" text so a policy-blocked forbidden tool is scored compliant for gitagent too. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent e2c9992 commit 0c36537

1 file changed

Lines changed: 28 additions & 0 deletions

File tree

packages/agentos-server/src/eval-runner.ts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,14 @@ export async function runAgainstHarness(
212212
const tMessages = new Map<string, TranscriptEntry[]>();
213213
const tOrder: string[] = [];
214214

215+
// gitclaw/hook denials surface as text, not the Claude SDK's structured
216+
// `permission_denials`. Pull the tool name out of "Tool 'cli' denied/blocked"
217+
// so the tool-compliance scorer can treat a policy-blocked tool as compliant.
218+
const recordDenial = (text: string): void => {
219+
const m = /\bTool ['"]?([\w.-]+)['"]?\s+(?:was\s+)?(?:denied|blocked)/i.exec(text);
220+
if (m && !permissionDenials.some((d) => d.tool_name === m[1])) permissionDenials.push({ tool_name: m[1] });
221+
};
222+
215223
const handleFrame = (frame: string) => {
216224
let evName = "";
217225
let data: any = null;
@@ -272,10 +280,30 @@ export async function runAgainstHarness(
272280
if (typeof payload.result === "string") output = payload.result;
273281
if (typeof payload.total_cost_usd === "number") costUsd = payload.total_cost_usd;
274282
if (Array.isArray(payload.permission_denials)) permissionDenials = payload.permission_denials;
283+
} else if (type === "tool_use") {
284+
// gitclaw (and other engines) emit tool calls as TOP-LEVEL messages
285+
// (`toolName`/`args`) rather than nested in an assistant message's content.
286+
const name = typeof payload.toolName === "string" ? payload.toolName : typeof payload.name === "string" ? payload.name : "";
287+
if (name) {
288+
const id = `gtu-${tOrder.length}`;
289+
tOrder.push(id);
290+
tMessages.set(id, [{ type: "tool_use", tool: name, input: payload.args ?? payload.input }]);
291+
}
292+
} else if (type === "tool_result") {
293+
// Top-level tool result. A policy/hook block arrives here as an error
294+
// ("Tool 'cli' denied") — recordDenial extracts the tool for the scorer.
295+
const content = typeof payload.content === "string"
296+
? payload.content
297+
: Array.isArray(payload.content) ? payload.content.map((x: any) => x?.text ?? "").join("") : "";
298+
const id = `gtr-${tOrder.length}`;
299+
tOrder.push(id);
300+
tMessages.set(id, [{ type: "tool_result", text: content, isError: payload.isError === true }]);
301+
recordDenial(content);
275302
} else if (typeof payload.content === "string" && payload.content.trim()) {
276303
// Engines that surface the assistant's final text as a bare `content`
277304
// string (gitagent/deepagents/wrapped envelopes) with no Claude-SDK
278305
// `result` frame. Captured into the transcript → `output || lastText`.
306+
recordDenial(payload.content);
279307
const id = `c-${tOrder.length}`;
280308
if (!tMessages.has(id)) tOrder.push(id);
281309
tMessages.set(id, [{ type: "text", text: payload.content }]);

0 commit comments

Comments
 (0)