Add agent rehearsal harness, findings, and per-run evidence for split-loop-with-lambda
Six scripted pi sub-agent runs (3 tool regimes × 2 models) rehearsed the exercise; deterministic gate = solution test + approved file. Key results: mandated CLI triples cost on the strong model (4:06 vs 0:39, PASS both); weak model fails only under the mandate (A+Q), passes when free or optional. Drift documented with code: scorer.updateSelection left inside the extracted method (A+Q, C+Q). Harness in tools/rehearse; summary in rehearsal-findings.md; per-run evidence in rehearsal-runs/; cheat sheet in rehearse-with-agent.md corrected (lambda-typed parameters unsupported).
This commit is contained in:
Your Name
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# Measure one run: total tokens (sum of per-message usage in the pi session file),
|
||||
# wall clock (from /usr/bin/time -v output), and calls to the refactoring CLI.
|
||||
# usage: bash measure_run.sh <run_dir>
|
||||
set -uo pipefail
|
||||
RUN_DIR="$1"
|
||||
|
||||
SESSION=$(find "$RUN_DIR/session" -name '*.jsonl' 2>/dev/null | head -1)
|
||||
echo "--- Run $(basename "$RUN_DIR") ---"
|
||||
|
||||
if [ -z "$SESSION" ]; then
|
||||
echo "session file: MISSING"
|
||||
else
|
||||
echo "session file: $SESSION"
|
||||
python3 - "$SESSION" <<'PY'
|
||||
import json, sys
|
||||
|
||||
path = sys.argv[1]
|
||||
usage_keys = ("input", "output", "cacheRead", "cacheWrite", "reasoning")
|
||||
totals = dict.fromkeys(usage_keys, 0)
|
||||
n_assistant = 0
|
||||
n_tool_uses = 0
|
||||
tool_counts = {}
|
||||
first_ts = last_ts = None
|
||||
model = None
|
||||
|
||||
def walk(obj):
|
||||
"""Yield every dict in a JSON structure (sessions nest messages in lists)."""
|
||||
if isinstance(obj, dict):
|
||||
yield obj
|
||||
for v in obj.values():
|
||||
yield from walk(v)
|
||||
elif isinstance(obj, list):
|
||||
for v in obj:
|
||||
yield from walk(v)
|
||||
|
||||
for line in open(path):
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if first_ts is None:
|
||||
first_ts = rec.get("timestamp")
|
||||
last_ts = rec.get("timestamp") or last_ts
|
||||
if rec.get("type") == "model_change":
|
||||
model = rec.get("modelId")
|
||||
for d in walk(rec.get("message") or {}):
|
||||
role = d.get("role")
|
||||
if role != "assistant":
|
||||
continue
|
||||
if d.get("usage"):
|
||||
n_assistant += 1
|
||||
u = d["usage"]
|
||||
for k in usage_keys:
|
||||
totals[k] += u.get(k) or 0
|
||||
for c in d.get("content") or []:
|
||||
if isinstance(c, dict) and c.get("type") == "toolCall":
|
||||
n_tool_uses += 1
|
||||
name = c.get("name") or "?"
|
||||
tool_counts[name] = tool_counts.get(name, 0) + 1
|
||||
|
||||
print(f"model: {model}")
|
||||
if first_ts and last_ts:
|
||||
from datetime import datetime
|
||||
fmt = "%Y-%m-%dT%H:%M:%S.%f%z"
|
||||
t0 = datetime.strptime(first_ts, fmt)
|
||||
t1 = datetime.strptime(last_ts, fmt)
|
||||
print(f"session span: {(t1 - t0).total_seconds():.1f} s")
|
||||
print(f"assistant messages: {n_assistant}")
|
||||
print(f"tool calls: {n_tool_uses} total")
|
||||
for name, n in sorted(tool_counts.items(), key=lambda kv: -kv[1]):
|
||||
print(f" {name:24} {n}")
|
||||
print(f"tokens input: {totals['input']}")
|
||||
print(f"tokens output: {totals['output']}")
|
||||
print(f"tokens cacheRead: {totals['cacheRead']}")
|
||||
print(f"tokens cacheWrite: {totals['cacheWrite']}")
|
||||
print(f"tokens reasoning: {totals['reasoning']}")
|
||||
print(f"tokens TOTAL (sum): {sum(totals.values())}")
|
||||
PY
|
||||
fi
|
||||
|
||||
if [ -f "$RUN_DIR/time.txt" ]; then
|
||||
echo "--- /usr/bin/time -v (wall clock + max RSS) ---"
|
||||
grep -E "Elapsed \(wall clock\)|Maximum resident set size|Exit status" "$RUN_DIR/time.txt"
|
||||
fi
|
||||
|
||||
# crude proxy: how often the CLI was invoked during the run (from bash history absent;
|
||||
# the session transcript already counted tool calls, so this is just informational)
|
||||
if [ -f "$RUN_DIR/pi-stdout.txt" ]; then
|
||||
echo "--- pi stdout (last 15 lines) ---"
|
||||
tail -15 "$RUN_DIR/pi-stdout.txt"
|
||||
fi
|
||||
Executable
+151
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stage 2 — A/B rehearsal: spawn a pi sub-agent per variant to do the refactoring,
|
||||
# with and without the java-refactor CLI, capped at 10 minutes wall clock each.
|
||||
#
|
||||
# usage: bash tools/rehearse/run-rehearsal.sh [A] [B] (default: both)
|
||||
#
|
||||
# For each run: fresh workspace from pristine, AGENTS.md with the shared task +
|
||||
# variant-specific tool rules, pi -p with its own session dir, then measurement
|
||||
# (tokens/wall clock) and deterministic verification (solution test + approved file).
|
||||
set -uo pipefail
|
||||
|
||||
SCRATCH=/tmp/rehearsal
|
||||
PRISTINE="$SCRATCH/pristine"
|
||||
RUNS_DIR="$SCRATCH/runs"
|
||||
REPO=/home/cardfanatic/qwan/exercises/learning-hour-maker
|
||||
SOLUTION="$REPO/ideas/split-loop-with-lambda/exercise/solution"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CAP_SECONDS=600
|
||||
|
||||
[ -d "$PRISTINE" ] || { echo "Run tools/rehearse/setup.sh first (no $PRISTINE)"; exit 1; }
|
||||
WHICH="${*:-A B}"
|
||||
|
||||
# Variant -> pi model override (empty = pi settings default).
|
||||
# +Q suffix: same variant's rules on entrim Qwen3.6-35B-A3B (weak-tool-caller datapoint).
|
||||
case "$WHICH" in
|
||||
A+Q) VARIANT=A; MODEL_ARG=(--provider entrim --model Qwen/Qwen3.6-35B-A3B) ;;
|
||||
B+Q) VARIANT=B; MODEL_ARG=(--provider entrim --model Qwen/Qwen3.6-35B-A3B) ;;
|
||||
C+Q) VARIANT=C; MODEL_ARG=(--provider entrim --model Qwen/Qwen3.6-35B-A3B) ;;
|
||||
*) VARIANT="$WHICH"; MODEL_ARG=() ;;
|
||||
esac
|
||||
RUN="$WHICH"
|
||||
|
||||
PROMPT='Read AGENTS.md in this directory and execute the refactoring task it describes. Run mvn -q test after each change. When done, stop and report the final diff of LongRangeForecasts.java.'
|
||||
|
||||
for RUN in $WHICH; do
|
||||
RUN_DIR="$RUNS_DIR/$RUN"
|
||||
WS="$RUN_DIR/workspace"
|
||||
rm -rf "$RUN_DIR"
|
||||
mkdir -p "$WS" "$RUN_DIR/session"
|
||||
cp -r "$PRISTINE/." "$WS/"
|
||||
|
||||
# --- AGENTS.md: shared design instructions (identical for A and B) ---
|
||||
cat > "$WS/AGENTS.md" <<'EOF'
|
||||
# Rehearsal workspace — refactoring task
|
||||
|
||||
You are refactoring `src/main/java/codingdojo/LongRangeForecasts.java` in this
|
||||
Maven project. Run `mvn -q test` after each change (mvn is on your PATH). It may
|
||||
fail — that is expected: the dependency on `Scorer` makes the method untestable —
|
||||
but the code must still *compile*.
|
||||
|
||||
## Goal
|
||||
|
||||
In `LongRangeForecasts.longRangeForecast()`, the loop over `interestingDates`
|
||||
collects weather by calling `scorer` (an unwanted external dependency).
|
||||
|
||||
1. Introduce a lambda for the scorer-dependent code and move its definition to
|
||||
the top of the method, leaving the invocation where it is. (No refactoring
|
||||
tool supports this — do it as a minimal manual edit.)
|
||||
2. Extract the weather-collection loop into a new method `collectExpectedWeather`
|
||||
that takes the lambda as a parameter (suggested type `LongFunction<Boolean>`).
|
||||
3. Add the lambda as a parameter to `longRangeForecast` itself: the signature
|
||||
must become `longRangeForecast(LongFunction<Boolean> dayOffsetToWeather)`,
|
||||
with a public no-arg overload `longRangeForecast()` that delegates and wraps
|
||||
the `scorer` field in a lambda. Do not keep the `String quarter` parameter.
|
||||
(`java-refactor change-method-signature` or `java-refactor introduce-param`
|
||||
may help; fall back to a minimal manual edit if no subcommand can do it.)
|
||||
4. Stop. Report the final diff.
|
||||
EOF
|
||||
|
||||
# --- variant-specific tool rules ---
|
||||
if [ "$VARIANT" = "A" ]; then
|
||||
cat >> "$WS/AGENTS.md" <<'EOF'
|
||||
|
||||
## Tool rules (variant A)
|
||||
You may only change code by invoking `java-refactor` CLI commands
|
||||
(`java-refactor --help` lists them; it is on your PATH). Never edit `.java`
|
||||
files with text edits — the ONLY exceptions are the introduce-lambda edit
|
||||
(step 1) and, if no `java-refactor` subcommand can accomplish it, a minimal
|
||||
manual edit for step 3.
|
||||
EOF
|
||||
elif [ "$VARIANT" = "B" ]; then
|
||||
cat >> "$WS/AGENTS.md" <<'EOF'
|
||||
|
||||
## Tool rules (variant B)
|
||||
You may change code with your normal file-editing tools.
|
||||
EOF
|
||||
else
|
||||
cat >> "$WS/AGENTS.md" <<'EOF'
|
||||
|
||||
## Tool rules (variant C)
|
||||
A semantic Java refactoring CLI, `java-refactor`, is on your PATH
|
||||
(`java-refactor --help` lists the subcommands). Using it is OPTIONAL — you
|
||||
may also change code with your normal file-editing tools. Choose per step
|
||||
whatever you think is most reliable.
|
||||
|
||||
Subcommands that may be relevant for this task:
|
||||
- `extract` — extract selected statements into a new private method
|
||||
(selection given as start/end line+column; `--dry-run` previews)
|
||||
- `introduce-param` / `change-method-signature` — alter a method's parameters
|
||||
- `rename` — rename a symbol project-wide
|
||||
EOF
|
||||
fi
|
||||
cp "$WS/AGENTS.md" "$RUN_DIR/agents-md.txt"
|
||||
printf '%s\n' "$PROMPT" > "$RUN_DIR/prompt.txt"
|
||||
|
||||
# --- spawn the sub-agent, fresh session dir per run ---
|
||||
# PATH: maven for both; the java-refactor wrapper ONLY for run A.
|
||||
echo "== Run $RUN: starting sub-agent (cap ${CAP_SECONDS}s) =="
|
||||
(
|
||||
cd "$WS"
|
||||
export JAVA_HOME=/tmp/jdk21
|
||||
if [ "$VARIANT" = "A" ] || [ "$VARIANT" = "C" ]; then
|
||||
export PATH="$SCRATCH/bin:/tmp/apache-maven-3.9.9/bin:$PATH"
|
||||
else
|
||||
export PATH="/tmp/apache-maven-3.9.9/bin:$PATH"
|
||||
fi
|
||||
/usr/bin/time -v -o "$RUN_DIR/time.txt" \
|
||||
timeout -k 15 "$CAP_SECONDS" \
|
||||
pi -p -a "${MODEL_ARG[@]}" --session-dir "$RUN_DIR/session" \
|
||||
--name "rehearsal-$RUN" "$PROMPT"
|
||||
) > "$RUN_DIR/pi-stdout.txt" 2> "$RUN_DIR/pi-stderr.txt"
|
||||
echo " exit=$? (124 = timed out)"
|
||||
|
||||
echo "== Run $RUN: measuring =="
|
||||
bash "$SCRIPT_DIR/measure_run.sh" "$RUN_DIR" | tee "$RUN_DIR/measure.txt"
|
||||
|
||||
echo "== Run $RUN: verifying =="
|
||||
bash "$SCRIPT_DIR/verify_outcome.sh" "$RUN_DIR" | tee "$RUN_DIR/verify.txt"
|
||||
done
|
||||
|
||||
# --- combined report ---
|
||||
REPORT="$SCRATCH/report.md"
|
||||
{
|
||||
echo "# Rehearsal report — split-loop-with-lambda ($(date -Iseconds))"
|
||||
echo
|
||||
for RUN in $WHICH; do
|
||||
RUN_DIR="$RUNS_DIR/$RUN"
|
||||
echo "## Run $RUN"
|
||||
echo
|
||||
echo '```'
|
||||
cat "$RUN_DIR/measure.txt" 2>/dev/null || echo "(measurement missing)"
|
||||
echo
|
||||
cat "$RUN_DIR/verify.txt" 2>/dev/null || echo "(verification missing)"
|
||||
echo '```'
|
||||
echo
|
||||
echo "Full artifacts: \`$RUN_DIR\` (diff.patch, pi-stdout.txt, session/)"
|
||||
echo
|
||||
done
|
||||
} > "$REPORT"
|
||||
echo "Report: $REPORT"
|
||||
cat "$REPORT"
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stage 1 — one-time setup: build java-refactoring-mcp, install the CLI wrapper,
|
||||
# create the pristine kata workspace, and smoke-test the CLI.
|
||||
# Idempotent: safe to re-run. Artifacts live in /tmp/rehearsal (scratch space).
|
||||
set -euo pipefail
|
||||
|
||||
MCP_REPO=/tmp/java-refactoring-mcp
|
||||
KATA=/tmp/IceCreamForecasting-Refactoring-Kata
|
||||
SCRATCH=/tmp/rehearsal
|
||||
MVN=/tmp/apache-maven-3.9.9/bin/mvn
|
||||
# Temurin JDK downloaded to /tmp by this script (system has JRE only, no javac)
|
||||
if [ ! -x /tmp/jdk21/bin/javac ]; then
|
||||
echo "== 0. Downloading Temurin JDK 21 to /tmp/jdk21 (system has no javac) =="
|
||||
curl -sL -o /tmp/jdk21.tar.gz \
|
||||
"https://api.adoptium.net/v3/binary/latest/21/ga/linux/x64/jdk/hotspot/normal/eclipse"
|
||||
mkdir -p /tmp/jdk21
|
||||
tar -xzf /tmp/jdk21.tar.gz -C /tmp/jdk21 --strip-components=1
|
||||
fi
|
||||
export JAVA_HOME=/tmp/jdk21
|
||||
export PATH="$JAVA_HOME/bin:$PATH"
|
||||
|
||||
mkdir -p "$SCRATCH/bin"
|
||||
|
||||
echo "== 1. Build the tool (skip if fat jars exist) =="
|
||||
CLI_JAR=$(ls "$MCP_REPO"/refactor-cli/target/refactor-cli-*-fat.jar 2>/dev/null | head -1 || true)
|
||||
MCP_JAR=$(ls "$MCP_REPO"/refactor-mcp/target/refactor-mcp-*-fat.jar 2>/dev/null | head -1 || true)
|
||||
if [ -z "${CLI_JAR:-}" ]; then
|
||||
(cd "$MCP_REPO" && "$MVN" -q package -DskipTests)
|
||||
fi
|
||||
CLI_JAR=$(ls "$MCP_REPO"/refactor-cli/target/refactor-cli-*-fat.jar | head -1)
|
||||
MCP_JAR=$(ls "$MCP_REPO"/refactor-mcp/target/refactor-mcp-*-fat.jar | head -1)
|
||||
echo " CLI jar: $CLI_JAR"
|
||||
echo " MCP jar: $MCP_JAR (for reference; not needed by this rehearsal)"
|
||||
echo " JAVA_HOME: $JAVA_HOME"
|
||||
|
||||
echo "== 2. Install CLI wrapper =="
|
||||
cat > "$SCRATCH/bin/java-refactor" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
exec "$JAVA_HOME/bin/java" -jar "$CLI_JAR" "\$@"
|
||||
EOF
|
||||
chmod +x "$SCRATCH/bin/java-refactor"
|
||||
|
||||
echo "== 3. Pristine kata workspace (never touched by runs) =="
|
||||
rm -rf "$SCRATCH/pristine"
|
||||
cp -r "$KATA/java" "$SCRATCH/pristine"
|
||||
rm -rf "$SCRATCH/pristine/target"
|
||||
|
||||
echo "== 4. Baseline: pristine compiles (tests EXPECTED to fail — Scorer is flaky by design) =="
|
||||
( cd "$SCRATCH/pristine" \
|
||||
&& PATH="$JAVA_HOME/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \
|
||||
mvn -q test-compile ) \
|
||||
&& echo " test-compile OK" \
|
||||
|| { echo " test-compile FAILED"; exit 1; }
|
||||
# Demonstrate the expected baseline failure (flaky approval via Scorer's Random + today's date):
|
||||
( cd "$SCRATCH/pristine" \
|
||||
&& PATH="$JAVA_HOME/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \
|
||||
mvn -q test ) > "$SCRATCH/baseline-test.log" 2>&1 \
|
||||
&& echo " NOTE: pristine tests unexpectedly PASSED" \
|
||||
|| echo " pristine tests fail as expected (untestable Scorer dependency) — see baseline-test.log"
|
||||
|
||||
echo "== 5. Smoke test: java-refactor --help =="
|
||||
PATH="$SCRATCH/bin:$PATH" java-refactor --help > "$SCRATCH/smoke-help.txt" 2>&1 \
|
||||
&& head -3 "$SCRATCH/smoke-help.txt"
|
||||
|
||||
echo "== 6. Smoke test: dry-run extract of the weather loop =="
|
||||
# LongRangeForecasts.java lines 29-36: the for-loop over interestingDates.
|
||||
rm -rf "$SCRATCH/smoke"
|
||||
cp -r "$SCRATCH/pristine" "$SCRATCH/smoke"
|
||||
( cd "$SCRATCH/smoke" \
|
||||
&& PATH="$SCRATCH/bin:$PATH" java-refactor extract \
|
||||
--file src/main/java/codingdojo/LongRangeForecasts.java \
|
||||
--start-line 29 --start-column 9 \
|
||||
--end-line 36 --end-column 10 \
|
||||
--name collectExpectedWeather --dry-run ) \
|
||||
> "$SCRATCH/smoke-extract.txt" 2>&1 \
|
||||
&& echo " dry-run OK (see /tmp/rehearsal/smoke-extract.txt)" \
|
||||
|| { echo " dry-run FAILED — inspect /tmp/rehearsal/smoke-extract.txt"; exit 1; }
|
||||
|
||||
echo
|
||||
echo "Stage 1 done. Artifacts in $SCRATCH:"
|
||||
ls -la "$SCRATCH"
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deterministic verification of one run's end state:
|
||||
# 1. diff vs pristine exercise -> diff.patch (the agent's final diff)
|
||||
# 2. drop in the solution's test + approved file, run mvn test
|
||||
# PASS = deterministic end state matches the reference solution's behavior
|
||||
# 3. structural check: does collectExpectedWeather exist and take the lambda?
|
||||
# usage: bash verify_outcome.sh <run_dir>
|
||||
set -uo pipefail
|
||||
RUN_DIR="$1"
|
||||
WS="$RUN_DIR/workspace"
|
||||
REPO=/home/cardfanatic/qwan/exercises/learning-hour-maker
|
||||
SOLUTION="$REPO/ideas/split-loop-with-lambda/exercise/solution"
|
||||
PRISTINE=/tmp/rehearsal/pristine
|
||||
|
||||
echo "--- Run $(basename "$RUN_DIR") ---"
|
||||
|
||||
# 1. final diff vs pristine
|
||||
diff -ru --exclude=target "$PRISTINE" "$WS" > "$RUN_DIR/diff.patch"
|
||||
if [ -s "$RUN_DIR/diff.patch" ]; then
|
||||
echo "final diff vs pristine: $(grep -c '^[+-][^+-]' "$RUN_DIR/diff.patch") changed lines -> $RUN_DIR/diff.patch"
|
||||
else
|
||||
echo "final diff vs pristine: EMPTY (agent made no changes)"
|
||||
fi
|
||||
|
||||
# 2. deterministic test: replace test tree with the solution's test + approved file
|
||||
if [ ! -d "$SOLUTION" ]; then
|
||||
echo "solution dir missing, skipping mvn verification"
|
||||
exit 0
|
||||
fi
|
||||
rm -rf "$WS/src/test"
|
||||
mkdir -p "$WS/src/test/java/codingdojo"
|
||||
cp -r "$SOLUTION/src/test/java/codingdojo/." "$WS/src/test/java/codingdojo/"
|
||||
|
||||
echo "mvn test with solution's test + approved file:"
|
||||
if ( cd "$WS" && JAVA_HOME=/tmp/jdk21 PATH="/tmp/jdk21/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \
|
||||
mvn -q test ) > "$RUN_DIR/mvn-verify.log" 2>&1; then
|
||||
echo " RESULT: PASS (deterministic)"
|
||||
else
|
||||
echo " RESULT: FAIL (see $RUN_DIR/mvn-verify.log, last lines:)"
|
||||
tail -15 "$RUN_DIR/mvn-verify.log" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# 3. structural checks on the produced LongRangeForecasts.java
|
||||
echo "structural checks:"
|
||||
LR="$WS/src/main/java/codingdojo/LongRangeForecasts.java"
|
||||
grep -q "collectExpectedWeather" "$LR" \
|
||||
&& echo " [x] method collectExpectedWeather exists" \
|
||||
|| echo " [ ] method collectExpectedWeather MISSING"
|
||||
grep -q "LongFunction<Boolean>" "$LR" \
|
||||
&& echo " [x] takes LongFunction<Boolean> parameter" \
|
||||
|| echo " [ ] LongFunction<Boolean> parameter MISSING"
|
||||
grep -q "scorer" "$LR" \
|
||||
&& echo " [i] note: 'scorer' still referenced in LongRangeForecasts.java" \
|
||||
|| echo " [x] no direct scorer reference left in LongRangeForecasts.java"
|
||||
if grep -qF -- "->" "$LR"; then
|
||||
echo " [x] lambda syntax present"
|
||||
else
|
||||
echo " [ ] lambda syntax MISSING (the classic drift: extracted method closes over scorer)"
|
||||
fi
|
||||
|
||||
# comparison against reference solution shape
|
||||
if diff -q "$SOLUTION/src/main/java/codingdojo/LongRangeForecasts.java" "$LR" > /dev/null; then
|
||||
echo " [x] main class byte-identical to reference solution"
|
||||
else
|
||||
echo " [i] main class differs from reference solution (may still be a valid variant)"
|
||||
fi
|
||||
Reference in New Issue
Block a user