Add agent rehearsal harness, findings, and per-run evidence for split-loop-with-lambda

Six scripted pi sub-agent runs (3 tool regimes × 2 models) rehearsed the exercise; deterministic gate = solution test + approved file. Key results: mandated CLI triples cost on the strong model (4:06 vs 0:39, PASS both); weak model fails only under the mandate (A+Q), passes when free or optional. Drift documented with code: scorer.updateSelection left inside the extracted method (A+Q, C+Q). Harness in tools/rehearse; summary in rehearsal-findings.md; per-run evidence in rehearsal-runs/; cheat sheet in rehearse-with-agent.md corrected (lambda-typed parameters unsupported).
2026-09-27 17:36:19 +00:00
parent a27e32688a
commit b993f25c53
51 changed files with 2281 additions and 2 deletions
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Measure one run: total tokens (sum of per-message usage in the pi session file),
+# wall clock (from /usr/bin/time -v output), and calls to the refactoring CLI.
+# usage: bash measure_run.sh <run_dir>
+set -uo pipefail
+RUN_DIR="$1"
+
+SESSION=$(find "$RUN_DIR/session" -name '*.jsonl' 2>/dev/null | head -1)
+echo "--- Run $(basename "$RUN_DIR") ---"
+
+if [ -z "$SESSION" ]; then
+  echo "session file: MISSING"
+else
+  echo "session file: $SESSION"
+  python3 - "$SESSION" <<'PY'
+import json, sys
+
+path = sys.argv[1]
+usage_keys = ("input", "output", "cacheRead", "cacheWrite", "reasoning")
+totals = dict.fromkeys(usage_keys, 0)
+n_assistant = 0
+n_tool_uses = 0
+tool_counts = {}
+first_ts = last_ts = None
+model = None
+
+def walk(obj):
+    """Yield every dict in a JSON structure (sessions nest messages in lists)."""
+    if isinstance(obj, dict):
+        yield obj
+        for v in obj.values():
+            yield from walk(v)
+    elif isinstance(obj, list):
+        for v in obj:
+            yield from walk(v)
+
+for line in open(path):
+    try:
+        rec = json.loads(line)
+    except json.JSONDecodeError:
+        continue
+    if first_ts is None:
+        first_ts = rec.get("timestamp")
+    last_ts = rec.get("timestamp") or last_ts
+    if rec.get("type") == "model_change":
+        model = rec.get("modelId")
+    for d in walk(rec.get("message") or {}):
+        role = d.get("role")
+        if role != "assistant":
+            continue
+        if d.get("usage"):
+            n_assistant += 1
+            u = d["usage"]
+            for k in usage_keys:
+                totals[k] += u.get(k) or 0
+        for c in d.get("content") or []:
+            if isinstance(c, dict) and c.get("type") == "toolCall":
+                n_tool_uses += 1
+                name = c.get("name") or "?"
+                tool_counts[name] = tool_counts.get(name, 0) + 1
+
+print(f"model:                {model}")
+if first_ts and last_ts:
+    from datetime import datetime
+    fmt = "%Y-%m-%dT%H:%M:%S.%f%z"
+    t0 = datetime.strptime(first_ts, fmt)
+    t1 = datetime.strptime(last_ts, fmt)
+    print(f"session span:         {(t1 - t0).total_seconds():.1f} s")
+print(f"assistant messages:   {n_assistant}")
+print(f"tool calls:           {n_tool_uses} total")
+for name, n in sorted(tool_counts.items(), key=lambda kv: -kv[1]):
+    print(f"  {name:24} {n}")
+print(f"tokens input:         {totals['input']}")
+print(f"tokens output:        {totals['output']}")
+print(f"tokens cacheRead:     {totals['cacheRead']}")
+print(f"tokens cacheWrite:    {totals['cacheWrite']}")
+print(f"tokens reasoning:     {totals['reasoning']}")
+print(f"tokens TOTAL (sum):   {sum(totals.values())}")
+PY
+fi
+
+if [ -f "$RUN_DIR/time.txt" ]; then
+  echo "--- /usr/bin/time -v (wall clock + max RSS) ---"
+  grep -E "Elapsed \(wall clock\)|Maximum resident set size|Exit status" "$RUN_DIR/time.txt"
+fi
+
+# crude proxy: how often the CLI was invoked during the run (from bash history absent;
+# the session transcript already counted tool calls, so this is just informational)
+if [ -f "$RUN_DIR/pi-stdout.txt" ]; then
+  echo "--- pi stdout (last 15 lines) ---"
+  tail -15 "$RUN_DIR/pi-stdout.txt"
+fi