Add agent rehearsal harness, findings, and per-run evidence for split-loop-with-lambda
Six scripted pi sub-agent runs (3 tool regimes × 2 models) rehearsed the exercise; deterministic gate = solution test + approved file. Key results: mandated CLI triples cost on the strong model (4:06 vs 0:39, PASS both); weak model fails only under the mandate (A+Q), passes when free or optional. Drift documented with code: scorer.updateSelection left inside the extracted method (A+Q, C+Q). Harness in tools/rehearse; summary in rehearsal-findings.md; per-run evidence in rehearsal-runs/; cheat sheet in rehearse-with-agent.md corrected (lambda-typed parameters unsupported).
This commit is contained in:
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deterministic verification of one run's end state:
|
||||
# 1. diff vs pristine exercise -> diff.patch (the agent's final diff)
|
||||
# 2. drop in the solution's test + approved file, run mvn test
|
||||
# PASS = deterministic end state matches the reference solution's behavior
|
||||
# 3. structural check: does collectExpectedWeather exist and take the lambda?
|
||||
# usage: bash verify_outcome.sh <run_dir>
|
||||
set -uo pipefail
|
||||
RUN_DIR="$1"
|
||||
WS="$RUN_DIR/workspace"
|
||||
REPO=/home/cardfanatic/qwan/exercises/learning-hour-maker
|
||||
SOLUTION="$REPO/ideas/split-loop-with-lambda/exercise/solution"
|
||||
PRISTINE=/tmp/rehearsal/pristine
|
||||
|
||||
echo "--- Run $(basename "$RUN_DIR") ---"
|
||||
|
||||
# 1. final diff vs pristine
|
||||
diff -ru --exclude=target "$PRISTINE" "$WS" > "$RUN_DIR/diff.patch"
|
||||
if [ -s "$RUN_DIR/diff.patch" ]; then
|
||||
echo "final diff vs pristine: $(grep -c '^[+-][^+-]' "$RUN_DIR/diff.patch") changed lines -> $RUN_DIR/diff.patch"
|
||||
else
|
||||
echo "final diff vs pristine: EMPTY (agent made no changes)"
|
||||
fi
|
||||
|
||||
# 2. deterministic test: replace test tree with the solution's test + approved file
|
||||
if [ ! -d "$SOLUTION" ]; then
|
||||
echo "solution dir missing, skipping mvn verification"
|
||||
exit 0
|
||||
fi
|
||||
rm -rf "$WS/src/test"
|
||||
mkdir -p "$WS/src/test/java/codingdojo"
|
||||
cp -r "$SOLUTION/src/test/java/codingdojo/." "$WS/src/test/java/codingdojo/"
|
||||
|
||||
echo "mvn test with solution's test + approved file:"
|
||||
if ( cd "$WS" && JAVA_HOME=/tmp/jdk21 PATH="/tmp/jdk21/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \
|
||||
mvn -q test ) > "$RUN_DIR/mvn-verify.log" 2>&1; then
|
||||
echo " RESULT: PASS (deterministic)"
|
||||
else
|
||||
echo " RESULT: FAIL (see $RUN_DIR/mvn-verify.log, last lines:)"
|
||||
tail -15 "$RUN_DIR/mvn-verify.log" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# 3. structural checks on the produced LongRangeForecasts.java
|
||||
echo "structural checks:"
|
||||
LR="$WS/src/main/java/codingdojo/LongRangeForecasts.java"
|
||||
grep -q "collectExpectedWeather" "$LR" \
|
||||
&& echo " [x] method collectExpectedWeather exists" \
|
||||
|| echo " [ ] method collectExpectedWeather MISSING"
|
||||
grep -q "LongFunction<Boolean>" "$LR" \
|
||||
&& echo " [x] takes LongFunction<Boolean> parameter" \
|
||||
|| echo " [ ] LongFunction<Boolean> parameter MISSING"
|
||||
grep -q "scorer" "$LR" \
|
||||
&& echo " [i] note: 'scorer' still referenced in LongRangeForecasts.java" \
|
||||
|| echo " [x] no direct scorer reference left in LongRangeForecasts.java"
|
||||
if grep -qF -- "->" "$LR"; then
|
||||
echo " [x] lambda syntax present"
|
||||
else
|
||||
echo " [ ] lambda syntax MISSING (the classic drift: extracted method closes over scorer)"
|
||||
fi
|
||||
|
||||
# comparison against reference solution shape
|
||||
if diff -q "$SOLUTION/src/main/java/codingdojo/LongRangeForecasts.java" "$LR" > /dev/null; then
|
||||
echo " [x] main class byte-identical to reference solution"
|
||||
else
|
||||
echo " [i] main class differs from reference solution (may still be a valid variant)"
|
||||
fi
|
||||
Reference in New Issue
Block a user