Add agent rehearsal harness, findings, and per-run evidence for split-loop-with-lambda

Six scripted pi sub-agent runs (3 tool regimes × 2 models) rehearsed the
exercise; deterministic gate = solution test + approved file. Key results:
mandated CLI triples cost on the strong model (4:06 vs 0:39, PASS both);
weak model fails only under the mandate (A+Q), passes when free or optional.
Drift documented with code: scorer.updateSelection left inside the extracted
method (A+Q, C+Q). Harness in tools/rehearse; summary in
rehearsal-findings.md; per-run evidence in rehearsal-runs/; cheat sheet in
rehearse-with-agent.md corrected (lambda-typed parameters unsupported).
This commit is contained in:
Your Name
2026-09-27 17:36:19 +00:00
parent a27e32688a
commit b993f25c53
51 changed files with 2281 additions and 2 deletions
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# Deterministic verification of one run's end state:
# 1. diff vs pristine exercise -> diff.patch (the agent's final diff)
# 2. drop in the solution's test + approved file, run mvn test
# PASS = deterministic end state matches the reference solution's behavior
# 3. structural check: does collectExpectedWeather exist and take the lambda?
# usage: bash verify_outcome.sh <run_dir>
set -uo pipefail
RUN_DIR="$1"
WS="$RUN_DIR/workspace"
REPO=/home/cardfanatic/qwan/exercises/learning-hour-maker
SOLUTION="$REPO/ideas/split-loop-with-lambda/exercise/solution"
PRISTINE=/tmp/rehearsal/pristine
echo "--- Run $(basename "$RUN_DIR") ---"
# 1. final diff vs pristine
diff -ru --exclude=target "$PRISTINE" "$WS" > "$RUN_DIR/diff.patch"
if [ -s "$RUN_DIR/diff.patch" ]; then
echo "final diff vs pristine: $(grep -c '^[+-][^+-]' "$RUN_DIR/diff.patch") changed lines -> $RUN_DIR/diff.patch"
else
echo "final diff vs pristine: EMPTY (agent made no changes)"
fi
# 2. deterministic test: replace test tree with the solution's test + approved file
if [ ! -d "$SOLUTION" ]; then
echo "solution dir missing, skipping mvn verification"
exit 0
fi
rm -rf "$WS/src/test"
mkdir -p "$WS/src/test/java/codingdojo"
cp -r "$SOLUTION/src/test/java/codingdojo/." "$WS/src/test/java/codingdojo/"
echo "mvn test with solution's test + approved file:"
if ( cd "$WS" && JAVA_HOME=/tmp/jdk21 PATH="/tmp/jdk21/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \
mvn -q test ) > "$RUN_DIR/mvn-verify.log" 2>&1; then
echo " RESULT: PASS (deterministic)"
else
echo " RESULT: FAIL (see $RUN_DIR/mvn-verify.log, last lines:)"
tail -15 "$RUN_DIR/mvn-verify.log" | sed 's/^/ /'
fi
# 3. structural checks on the produced LongRangeForecasts.java
echo "structural checks:"
LR="$WS/src/main/java/codingdojo/LongRangeForecasts.java"
grep -q "collectExpectedWeather" "$LR" \
&& echo " [x] method collectExpectedWeather exists" \
|| echo " [ ] method collectExpectedWeather MISSING"
grep -q "LongFunction<Boolean>" "$LR" \
&& echo " [x] takes LongFunction<Boolean> parameter" \
|| echo " [ ] LongFunction<Boolean> parameter MISSING"
grep -q "scorer" "$LR" \
&& echo " [i] note: 'scorer' still referenced in LongRangeForecasts.java" \
|| echo " [x] no direct scorer reference left in LongRangeForecasts.java"
if grep -qF -- "->" "$LR"; then
echo " [x] lambda syntax present"
else
echo " [ ] lambda syntax MISSING (the classic drift: extracted method closes over scorer)"
fi
# comparison against reference solution shape
if diff -q "$SOLUTION/src/main/java/codingdojo/LongRangeForecasts.java" "$LR" > /dev/null; then
echo " [x] main class byte-identical to reference solution"
else
echo " [i] main class differs from reference solution (may still be a valid variant)"
fi