phase 2 complete - skills ported, with smoke tests
This commit is contained in:
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Run all Phase 2 smoke tests
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
passed=0
|
||||
failed=0
|
||||
results=()
|
||||
|
||||
run_test() {
|
||||
local name="$1"
|
||||
local script="$2"
|
||||
echo ""
|
||||
echo "━━━ $name ━━━"
|
||||
if bash "$script"; then
|
||||
results+=(" ✓ $name")
|
||||
((passed++))
|
||||
else
|
||||
results+=(" ✗ $name")
|
||||
((failed++))
|
||||
fi
|
||||
}
|
||||
|
||||
run_test "structure" structure.sh
|
||||
run_test "elicit" test-elicit.sh
|
||||
run_test "distill" test-distill.sh
|
||||
run_test "propagate" test-propagate.sh
|
||||
|
||||
echo ""
|
||||
echo "━━━ summary ━━━"
|
||||
for r in "${results[@]}"; do
|
||||
echo "$r"
|
||||
done
|
||||
echo ""
|
||||
echo "passed: $passed failed: $failed"
|
||||
|
||||
[ "$failed" -eq 0 ]
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Structural smoke tests for Phase 2 skills
|
||||
# Verifies files exist, frontmatter is valid, and reference symlinks resolve
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
pass=0
|
||||
fail=0
|
||||
|
||||
check() {
|
||||
local desc="$1"
|
||||
shift
|
||||
if "$@" >/dev/null 2>&1; then
|
||||
printf ' \033[32m✓\033[0m %s\n' "$desc"
|
||||
((pass++))
|
||||
else
|
||||
printf ' \033[31m✗\033[0m %s\n' "$desc"
|
||||
((fail++))
|
||||
fi
|
||||
}
|
||||
|
||||
# Inverse check: passes when the command FAILS (e.g. grep finds nothing)
|
||||
check_absent() {
|
||||
local desc="$1"
|
||||
shift
|
||||
if "$@" >/dev/null 2>&1; then
|
||||
printf ' \033[31m✗\033[0m %s\n' "$desc"
|
||||
((fail++))
|
||||
else
|
||||
printf ' \033[32m✓\033[0m %s\n' "$desc"
|
||||
((pass++))
|
||||
fi
|
||||
}
|
||||
|
||||
for skill in elicit distill propagate; do
|
||||
echo "=== $skill ==="
|
||||
check "SKILL.md exists" test -f ".pi/skills/$skill/SKILL.md"
|
||||
check "frontmatter: name: $skill" grep -q "^name: $skill" ".pi/skills/$skill/SKILL.md"
|
||||
check "frontmatter: disable-model-invocation" grep -q "^disable-model-invocation:" ".pi/skills/$skill/SKILL.md"
|
||||
check "frontmatter: license" grep -q "^license:" ".pi/skills/$skill/SKILL.md"
|
||||
check "frontmatter: metadata.upstream" grep -q "upstream:" ".pi/skills/$skill/SKILL.md"
|
||||
check "references/ directory exists" test -d ".pi/skills/$skill/references"
|
||||
done
|
||||
|
||||
echo "=== reference symlinks ==="
|
||||
check "elicit: language-reference.md resolves" test -s ".pi/skills/elicit/references/language-reference.md"
|
||||
check "elicit: library-spec-signals.md resolves" test -s ".pi/skills/elicit/references/library-spec-signals.md"
|
||||
check "distill: language-reference.md resolves" test -s ".pi/skills/distill/references/language-reference.md"
|
||||
check "distill: worked-examples.md resolves" test -s ".pi/skills/distill/references/worked-examples.md"
|
||||
check "propagate: test-generation.md resolves" test -s ".pi/skills/propagate/references/test-generation.md"
|
||||
|
||||
echo "=== cross-references in SKILL.md ==="
|
||||
check "elicit: refs use references/ prefix" grep -q '(references/language-reference.md)' ".pi/skills/elicit/SKILL.md"
|
||||
check "elicit: sub-ref uses references/ prefix" grep -q '(references/library-spec-signals.md)' ".pi/skills/elicit/SKILL.md"
|
||||
check "distill: refs use references/ prefix" grep -q '(references/language-reference.md)' ".pi/skills/distill/SKILL.md"
|
||||
check "distill: sub-ref uses references/ prefix" grep -q '(references/worked-examples.md)' ".pi/skills/distill/SKILL.md"
|
||||
check "propagate: refs use references/ prefix" grep -q 'references/test-generation.md' ".pi/skills/propagate/SKILL.md"
|
||||
|
||||
echo "=== invocation references ==="
|
||||
check "elicit: uses /skill:tend" grep -q '/skill:tend' ".pi/skills/elicit/SKILL.md"
|
||||
check "elicit: uses /skill:weed" grep -q '/skill:weed' ".pi/skills/elicit/SKILL.md"
|
||||
check "distill: uses /skill:tend" grep -q '/skill:tend' ".pi/skills/distill/SKILL.md"
|
||||
check "distill: uses /skill:weed" grep -q '/skill:weed' ".pi/skills/distill/SKILL.md"
|
||||
check "propagate: uses /skill:distill" grep -q '/skill:distill' ".pi/skills/propagate/SKILL.md"
|
||||
check "propagate: uses /skill:elicit" grep -q '/skill:elicit' ".pi/skills/propagate/SKILL.md"
|
||||
|
||||
echo "=== no upstream path leaks ==="
|
||||
check_absent "elicit: no ../../references/" grep -q '\.\./\.\./references/' ".pi/skills/elicit/SKILL.md"
|
||||
check_absent "distill: no ../../references/" grep -q '\.\./\.\./references/' ".pi/skills/distill/SKILL.md"
|
||||
check_absent "elicit: no \`tend\` skill" grep -q '`tend` skill' ".pi/skills/elicit/SKILL.md"
|
||||
check_absent "distill: no \`tend\` skill" grep -q '`tend` skill' ".pi/skills/distill/SKILL.md"
|
||||
|
||||
echo ""
|
||||
echo "passed: $pass failed: $fail"
|
||||
[ "$fail" -eq 0 ]
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Smoke test: /skill:distill loads and the model responds with distillation methodology
|
||||
# Expects: the response discusses extracting entities/rules from code
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
MODEL="Qwen3.6-35B-A3B-MXFP4_MOE.gguf"
|
||||
TIMEOUT=180
|
||||
|
||||
echo "=== distill: invoking pi ==="
|
||||
|
||||
output=$(timeout "$TIMEOUT" pi -p --no-session --model "$MODEL" \
|
||||
"/skill:distill I have a TypeScript function that checks if a user's subscription is active by comparing expiry date to now, and if expired sets status to 'expired'. How would you begin distilling this into an Allium spec? Keep it under 100 words." 2>&1)
|
||||
rc=$?
|
||||
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo " FAIL: pi exited with code $rc"
|
||||
echo "$output" | tail -20
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$output" ]; then
|
||||
echo " FAIL: empty response"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$output"
|
||||
echo ""
|
||||
|
||||
# Check the response is relevant to distillation
|
||||
if echo "$output" | grep -iqE 'entit|rule|status|domain|abstract|spec|allium|subscription|temporal|transition'; then
|
||||
echo " PASS: response contains distillation-relevant content"
|
||||
exit 0
|
||||
else
|
||||
echo " FAIL: response does not appear to follow distillation methodology"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Smoke test: /skill:elicit loads and the model responds with elicitation methodology
|
||||
# Expects: the response references scoping, entities, or asks discovery questions
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
MODEL="Qwen3.6-35B-A3B-MXFP4_MOE.gguf"
|
||||
TIMEOUT=180
|
||||
|
||||
echo "=== elicit: invoking pi ==="
|
||||
|
||||
output=$(timeout "$TIMEOUT" pi -p --no-session --model "$MODEL" \
|
||||
"/skill:elicit I want to specify a simple counter that increments and resets. What are the first questions you would ask? Keep it under 100 words." 2>&1)
|
||||
rc=$?
|
||||
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo " FAIL: pi exited with code $rc"
|
||||
echo "$output" | tail -20
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$output" ]; then
|
||||
echo " FAIL: empty response"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$output"
|
||||
echo ""
|
||||
|
||||
# Check the response is relevant to elicitation
|
||||
if echo "$output" | grep -iqE 'scope|boundar|entit|actor|specif|question|what.*system|who.*user'; then
|
||||
echo " PASS: response contains elicitation-relevant content"
|
||||
exit 0
|
||||
else
|
||||
echo " FAIL: response does not appear to follow elicitation methodology"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Smoke test: /skill:propagate loads and the model responds with test generation methodology
|
||||
# Expects: the response discusses test obligations, assertions, or test categories
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
MODEL="Qwen3.6-35B-A3B-MXFP4_MOE.gguf"
|
||||
TIMEOUT=180
|
||||
|
||||
echo "=== propagate: invoking pi ==="
|
||||
|
||||
output=$(timeout "$TIMEOUT" pi -p --no-session --model "$MODEL" \
|
||||
"/skill:propagate Given an Allium entity Order with status: pending | confirmed | shipped and a rule ConfirmOrder that transitions pending to confirmed, what test obligations would you derive? Keep it under 100 words." 2>&1)
|
||||
rc=$?
|
||||
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo " FAIL: pi exited with code $rc"
|
||||
echo "$output" | tail -20
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$output" ]; then
|
||||
echo " FAIL: empty response"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$output"
|
||||
echo ""
|
||||
|
||||
# Check the response is relevant to test generation
|
||||
if echo "$output" | grep -iqE 'test|assert|obligation|transition|valid|invalid|state|verif|propert'; then
|
||||
echo " PASS: response contains propagation-relevant content"
|
||||
exit 0
|
||||
else
|
||||
echo " FAIL: response does not appear to follow propagation methodology"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user