Your Name
b993f25c53
Six scripted pi sub-agent runs (3 tool regimes × 2 models) rehearsed the exercise; deterministic gate = solution test + approved file. Key results: mandated CLI triples cost on the strong model (4:06 vs 0:39, PASS both); weak model fails only under the mandate (A+Q), passes when free or optional. Drift documented with code: scorer.updateSelection left inside the extracted method (A+Q, C+Q). Harness in tools/rehearse; summary in rehearsal-findings.md; per-run evidence in rehearsal-runs/; cheat sheet in rehearse-with-agent.md corrected (lambda-typed parameters unsupported).
67 lines
2.7 KiB
Bash
Executable File
67 lines
2.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Deterministic verification of one run's end state:
|
|
# 1. diff vs pristine exercise -> diff.patch (the agent's final diff)
|
|
# 2. drop in the solution's test + approved file, run mvn test
|
|
# PASS = deterministic end state matches the reference solution's behavior
|
|
# 3. structural check: does collectExpectedWeather exist and take the lambda?
|
|
# usage: bash verify_outcome.sh <run_dir>
|
|
set -uo pipefail
|
|
RUN_DIR="$1"
|
|
WS="$RUN_DIR/workspace"
|
|
REPO=/home/cardfanatic/qwan/exercises/learning-hour-maker
|
|
SOLUTION="$REPO/ideas/split-loop-with-lambda/exercise/solution"
|
|
PRISTINE=/tmp/rehearsal/pristine
|
|
|
|
echo "--- Run $(basename "$RUN_DIR") ---"
|
|
|
|
# 1. final diff vs pristine
|
|
diff -ru --exclude=target "$PRISTINE" "$WS" > "$RUN_DIR/diff.patch"
|
|
if [ -s "$RUN_DIR/diff.patch" ]; then
|
|
echo "final diff vs pristine: $(grep -c '^[+-][^+-]' "$RUN_DIR/diff.patch") changed lines -> $RUN_DIR/diff.patch"
|
|
else
|
|
echo "final diff vs pristine: EMPTY (agent made no changes)"
|
|
fi
|
|
|
|
# 2. deterministic test: replace test tree with the solution's test + approved file
|
|
if [ ! -d "$SOLUTION" ]; then
|
|
echo "solution dir missing, skipping mvn verification"
|
|
exit 0
|
|
fi
|
|
rm -rf "$WS/src/test"
|
|
mkdir -p "$WS/src/test/java/codingdojo"
|
|
cp -r "$SOLUTION/src/test/java/codingdojo/." "$WS/src/test/java/codingdojo/"
|
|
|
|
echo "mvn test with solution's test + approved file:"
|
|
if ( cd "$WS" && JAVA_HOME=/tmp/jdk21 PATH="/tmp/jdk21/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \
|
|
mvn -q test ) > "$RUN_DIR/mvn-verify.log" 2>&1; then
|
|
echo " RESULT: PASS (deterministic)"
|
|
else
|
|
echo " RESULT: FAIL (see $RUN_DIR/mvn-verify.log, last lines:)"
|
|
tail -15 "$RUN_DIR/mvn-verify.log" | sed 's/^/ /'
|
|
fi
|
|
|
|
# 3. structural checks on the produced LongRangeForecasts.java
|
|
echo "structural checks:"
|
|
LR="$WS/src/main/java/codingdojo/LongRangeForecasts.java"
|
|
grep -q "collectExpectedWeather" "$LR" \
|
|
&& echo " [x] method collectExpectedWeather exists" \
|
|
|| echo " [ ] method collectExpectedWeather MISSING"
|
|
grep -q "LongFunction<Boolean>" "$LR" \
|
|
&& echo " [x] takes LongFunction<Boolean> parameter" \
|
|
|| echo " [ ] LongFunction<Boolean> parameter MISSING"
|
|
grep -q "scorer" "$LR" \
|
|
&& echo " [i] note: 'scorer' still referenced in LongRangeForecasts.java" \
|
|
|| echo " [x] no direct scorer reference left in LongRangeForecasts.java"
|
|
if grep -qF -- "->" "$LR"; then
|
|
echo " [x] lambda syntax present"
|
|
else
|
|
echo " [ ] lambda syntax MISSING (the classic drift: extracted method closes over scorer)"
|
|
fi
|
|
|
|
# comparison against reference solution shape
|
|
if diff -q "$SOLUTION/src/main/java/codingdojo/LongRangeForecasts.java" "$LR" > /dev/null; then
|
|
echo " [x] main class byte-identical to reference solution"
|
|
else
|
|
echo " [i] main class differs from reference solution (may still be a valid variant)"
|
|
fi
|