#!/usr/bin/env bash # Deterministic verification of one run's end state: # 1. diff vs pristine exercise -> diff.patch (the agent's final diff) # 2. drop in the solution's test + approved file, run mvn test # PASS = deterministic end state matches the reference solution's behavior # 3. structural check: does collectExpectedWeather exist and take the lambda? # usage: bash verify_outcome.sh set -uo pipefail RUN_DIR="$1" WS="$RUN_DIR/workspace" REPO=/home/cardfanatic/qwan/exercises/learning-hour-maker SOLUTION="$REPO/ideas/slice-with-lambda/exercise/solution" PRISTINE=/tmp/rehearsal/pristine echo "--- Run $(basename "$RUN_DIR") ---" # 1. final diff vs pristine diff -ru --exclude=target "$PRISTINE" "$WS" > "$RUN_DIR/diff.patch" if [ -s "$RUN_DIR/diff.patch" ]; then echo "final diff vs pristine: $(grep -c '^[+-][^+-]' "$RUN_DIR/diff.patch") changed lines -> $RUN_DIR/diff.patch" else echo "final diff vs pristine: EMPTY (agent made no changes)" fi # 2. deterministic test: replace test tree with the solution's test + approved file if [ ! -d "$SOLUTION" ]; then echo "solution dir missing, skipping mvn verification" exit 0 fi rm -rf "$WS/src/test" mkdir -p "$WS/src/test/java/codingdojo" cp -r "$SOLUTION/src/test/java/codingdojo/." "$WS/src/test/java/codingdojo/" echo "mvn test with solution's test + approved file:" if ( cd "$WS" && JAVA_HOME=/tmp/jdk21 PATH="/tmp/jdk21/bin:/tmp/apache-maven-3.9.9/bin:$PATH" \ mvn -q test ) > "$RUN_DIR/mvn-verify.log" 2>&1; then echo " RESULT: PASS (deterministic)" else echo " RESULT: FAIL (see $RUN_DIR/mvn-verify.log, last lines:)" tail -15 "$RUN_DIR/mvn-verify.log" | sed 's/^/ /' fi # 3. structural checks on the produced LongRangeForecasts.java echo "structural checks:" LR="$WS/src/main/java/codingdojo/LongRangeForecasts.java" grep -q "collectExpectedWeather" "$LR" \ && echo " [x] method collectExpectedWeather exists" \ || echo " [ ] method collectExpectedWeather MISSING" grep -q "LongFunction" "$LR" \ && echo " [x] takes LongFunction parameter" \ || echo " [ ] LongFunction parameter MISSING" grep -q "scorer" "$LR" \ && echo " [i] note: 'scorer' still referenced in LongRangeForecasts.java" \ || echo " [x] no direct scorer reference left in LongRangeForecasts.java" if grep -qF -- "->" "$LR"; then echo " [x] lambda syntax present" else echo " [ ] lambda syntax MISSING (the classic drift: extracted method closes over scorer)" fi # comparison against reference solution shape if diff -q "$SOLUTION/src/main/java/codingdojo/LongRangeForecasts.java" "$LR" > /dev/null; then echo " [x] main class byte-identical to reference solution" else echo " [i] main class differs from reference solution (may still be a valid variant)" fi