Rename, tests Fail, not Error

Parsing no longer depends on ui widget

.gitignore

@@ -1,2 +1,7 @@
 node_modules
+.pnpm-store
 .idea
+*.log
+
+# Pi extension package
+pi-test-status/node_modules

.pi/extensions/test-status.ts

@@ -1,44 +0,0 @@
-import { isBashToolResult, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
-
-export default function testStatusExtension(pi: ExtensionAPI) {
-  // After every tool execution, check if tests just ran
-  pi.on("tool_result", async (event, ctx) => {
-    // Only care about bash commands
-    if (!isBashToolResult(event)) return;
-
-    const command: string = event.input.command;
-
-    // Only care about test commands
-    if (!command.includes("npm test") && !command.includes("vitest") && !command.includes("npx vitest")) {
-      return;
-    }
-
-    const output = event.content[0]?.text ?? "";
-    const theme = ctx.ui.theme;
-
-    // Detect pass/fail from vitest summary output
-    const testsPassed = /Tests\s+.+passed/.test(output) && !/Tests\s+.+failed/.test(output);
-    const testsFailed = /Tests\s+.+failed/.test(output) || event.details?.exitCode !== 0;
-
-    if (testsPassed) {
-      // Extract summary line for inline detail (e.g., "1 passed (242ms)")
-      const summaryLine = output
-        .split("\n")
-        .find((line: string) => /Test Files\s+.+passed/.test(line));
-      const summary = summaryLine?.trim() ?? "All tests passed";
-
-      ctx.ui.setWidget("test-status", [
-        `${theme.fg("success", "●")} ${theme.fg("text", summary)}`,
-      ]);
-    } else if (testsFailed) {
-      const summaryLine = output
-        .split("\n")
-        .find((line: string) => /Tests\s+.+failed/.test(line));
-      const summary = summaryLine?.trim() ?? "Tests failed";
-
-      ctx.ui.setWidget("test-status", [
-        `${theme.fg("error", "●")} ${theme.fg("text", summary)}`,
-      ]);
-    }
-  });
-}

.pi/llm-metrics.log

@@ -1,15 +0,0 @@
-{"timestamp":"2026-06-20T11:03:05.568Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":1,"inputTokens":1038,"outputTokens":60,"totalTokens":1098,"prefillTokensPerSec":351.03,"generationTokensPerSec":189.87,"combinedTokensPerSec":335.47,"totalDurationMs":3273,"timeToFirstTokenMs":2957,"rawTimestamps":{"ttftMs":2957,"allTtftMs":[2957],"generationDurationMs":316,"turns":[{"turnId":"turn-0","durationMs":3273,"ttftMs":2957}]}}
-{"timestamp":"2026-06-20T11:08:53.529Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":3,"inputTokens":2915,"outputTokens":510,"totalTokens":3425,"prefillTokensPerSec":887.91,"generationTokensPerSec":56.86,"combinedTokensPerSec":279.52,"totalDurationMs":12253,"timeToFirstTokenMs":3283,"rawTimestamps":{"ttftMs":3283,"allTtftMs":[3283],"generationDurationMs":8970,"turns":[{"turnId":"turn-0","durationMs":3866},{"turnId":"turn-1","durationMs":3909},{"turnId":"turn-2","durationMs":4478,"ttftMs":3283}]}}
-{"timestamp":"2026-06-20T11:12:12.819Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":3,"inputTokens":21584,"outputTokens":1172,"totalTokens":22756,"prefillTokensPerSec":2383.66,"generationTokensPerSec":77.02,"combinedTokensPerSec":937.58,"totalDurationMs":24271,"timeToFirstTokenMs":9055,"rawTimestamps":{"ttftMs":9055,"allTtftMs":[9055],"generationDurationMs":15216,"turns":[{"turnId":"turn-0","durationMs":3053},{"turnId":"turn-1","durationMs":8673},{"turnId":"turn-2","durationMs":12545,"ttftMs":9055}]}}
-{"timestamp":"2026-06-20T11:16:46.039Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":2,"inputTokens":20185,"outputTokens":788,"totalTokens":20973,"prefillTokensPerSec":8734.31,"generationTokensPerSec":70.01,"combinedTokensPerSec":1545.88,"totalDurationMs":13567,"timeToFirstTokenMs":2311,"rawTimestamps":{"ttftMs":2311,"allTtftMs":[2311],"generationDurationMs":11256,"turns":[{"turnId":"turn-0","durationMs":9944},{"turnId":"turn-1","durationMs":3623,"ttftMs":2311}]}}
-{"timestamp":"2026-06-20T11:17:45.757Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":2,"inputTokens":21490,"outputTokens":157,"totalTokens":21647,"prefillTokensPerSec":6002.79,"generationTokensPerSec":20.34,"combinedTokensPerSec":1916,"totalDurationMs":11298,"timeToFirstTokenMs":3580,"rawTimestamps":{"ttftMs":3580,"allTtftMs":[3580],"generationDurationMs":7718,"turns":[{"turnId":"turn-0","durationMs":7166},{"turnId":"turn-1","durationMs":4132,"ttftMs":3580}]}}
-{"timestamp":"2026-06-20T11:19:06.241Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":2,"inputTokens":10578,"outputTokens":1189,"totalTokens":11767,"prefillTokensPerSec":1126.76,"generationTokensPerSec":106.47,"combinedTokensPerSec":572.46,"totalDurationMs":20555,"timeToFirstTokenMs":9388,"rawTimestamps":{"ttftMs":9388,"allTtftMs":[9388],"generationDurationMs":11167,"turns":[{"turnId":"turn-0","durationMs":6211},{"turnId":"turn-1","durationMs":14344,"ttftMs":9388}]}}
-{"timestamp":"2026-06-20T11:23:18.506Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":4,"inputTokens":77692,"outputTokens":3607,"totalTokens":81299,"prefillTokensPerSec":21695.62,"generationTokensPerSec":68.5,"combinedTokensPerSec":1445.68,"totalDurationMs":56236,"timeToFirstTokenMs":3581,"rawTimestamps":{"ttftMs":3581,"allTtftMs":[3581,17325,3277],"generationDurationMs":52655,"turns":[{"turnId":"turn-0","durationMs":6524},{"turnId":"turn-1","durationMs":4884,"ttftMs":3581},{"turnId":"turn-2","durationMs":32914,"ttftMs":17325},{"turnId":"turn-3","durationMs":11914,"ttftMs":3277}]}}
-{"timestamp":"2026-06-20T11:24:47.584Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":1,"inputTokens":38092,"outputTokens":198,"totalTokens":38290,"prefillTokensPerSec":7511.73,"generationTokensPerSec":224.74,"combinedTokensPerSec":6433.13,"totalDurationMs":5952,"timeToFirstTokenMs":5071,"rawTimestamps":{"ttftMs":5071,"allTtftMs":[5071],"generationDurationMs":881,"turns":[{"turnId":"turn-0","durationMs":5952,"ttftMs":5071}]}}
-{"timestamp":"2026-06-20T11:26:19.496Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":3,"inputTokens":21740,"outputTokens":720,"totalTokens":22460,"prefillTokensPerSec":4758.15,"generationTokensPerSec":34.98,"combinedTokensPerSec":893.01,"totalDurationMs":25151,"timeToFirstTokenMs":4569,"rawTimestamps":{"ttftMs":4569,"allTtftMs":[4569,8170,8094],"generationDurationMs":20582,"turns":[{"turnId":"turn-0","durationMs":5604,"ttftMs":4569},{"turnId":"turn-1","durationMs":9459,"ttftMs":8170},{"turnId":"turn-2","durationMs":10088,"ttftMs":8094}]}}
-{"timestamp":"2026-06-20T11:28:23.807Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":1,"inputTokens":114,"outputTokens":31,"totalTokens":145,"prefillTokensPerSec":20.5,"generationTokensPerSec":156.57,"combinedTokensPerSec":25.18,"totalDurationMs":5759,"timeToFirstTokenMs":5561,"rawTimestamps":{"ttftMs":5561,"allTtftMs":[5561],"generationDurationMs":198,"turns":[{"turnId":"turn-0","durationMs":5759,"ttftMs":5561}]}}
-{"timestamp":"2026-06-20T11:29:28.327Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":2,"inputTokens":79835,"outputTokens":216,"totalTokens":80051,"prefillTokensPerSec":15568.45,"generationTokensPerSec":33.59,"combinedTokensPerSec":6926.03,"totalDurationMs":11558,"timeToFirstTokenMs":5128,"rawTimestamps":{"ttftMs":5128,"allTtftMs":[5128],"generationDurationMs":6430,"turns":[{"turnId":"turn-0","durationMs":5985},{"turnId":"turn-1","durationMs":5573,"ttftMs":5128}]}}
-{"timestamp":"2026-06-20T11:30:00.138Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":4,"inputTokens":4564,"outputTokens":411,"totalTokens":4975,"prefillTokensPerSec":813.4,"generationTokensPerSec":25.37,"combinedTokensPerSec":228.06,"totalDurationMs":21814,"timeToFirstTokenMs":5611,"rawTimestamps":{"ttftMs":5611,"allTtftMs":[5611],"generationDurationMs":16203,"turns":[{"turnId":"turn-0","durationMs":4208},{"turnId":"turn-1","durationMs":3337},{"turnId":"turn-2","durationMs":7936},{"turnId":"turn-3","durationMs":6333,"ttftMs":5611}]}}
-{"timestamp":"2026-06-20T11:30:48.044Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":3,"inputTokens":4491,"outputTokens":917,"totalTokens":5408,"prefillTokensPerSec":666.62,"generationTokensPerSec":90.94,"combinedTokensPerSec":321.5,"totalDurationMs":16821,"timeToFirstTokenMs":6737,"rawTimestamps":{"ttftMs":6737,"allTtftMs":[6737,2505,3913],"generationDurationMs":10084,"turns":[{"turnId":"turn-0","durationMs":8498,"ttftMs":6737},{"turnId":"turn-1","durationMs":3187,"ttftMs":2505},{"turnId":"turn-2","durationMs":5136,"ttftMs":3913}]}}
-{"timestamp":"2026-06-20T11:32:53.704Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":2,"inputTokens":2255,"outputTokens":149,"totalTokens":2404,"prefillTokensPerSec":245.64,"generationTokensPerSec":23.34,"combinedTokensPerSec":154.47,"totalDurationMs":15563,"timeToFirstTokenMs":9180,"rawTimestamps":{"ttftMs":9180,"allTtftMs":[9180],"generationDurationMs":6383,"turns":[{"turnId":"turn-0","durationMs":5664},{"turnId":"turn-1","durationMs":9899,"ttftMs":9180}]}}
-{"timestamp":"2026-06-20T11:33:29.044Z","provider":"openrouter","model":"deepseek/deepseek-v4-flash","turnCount":3,"inputTokens":1321,"outputTokens":305,"totalTokens":1626,"prefillTokensPerSec":215.08,"generationTokensPerSec":15.8,"combinedTokensPerSec":63.91,"totalDurationMs":25441,"timeToFirstTokenMs":6142,"rawTimestamps":{"ttftMs":6142,"allTtftMs":[6142],"generationDurationMs":19299,"turns":[{"turnId":"turn-0","durationMs":11526},{"turnId":"turn-1","durationMs":7061},{"turnId":"turn-2","durationMs":6854,"ttftMs":6142}]}}

doc/test-output-parsers.md

@@ -0,0 +1,91 @@
+# Test Output Parsers
+
+## Motivation
+
+The test-status widget needs to decide pass/fail and show a meaningful summary.
+Relying on output regex alone for the pass/fail decision is brittle — different
+test frameworks, locales, custom reporters, or unusual test names can produce
+false positives or negatives. Using the process exit code as the canonical
+signal is universal and correct across all tools.
+
+This document outlines a modular architecture where pass/fail is decided by exit
+code, and output parsing is purely cosmetic enrichment.
+
+## Architecture
+
+```
+┌─────────────────────────────┐
+│  Decision (exit code)       │  ← single source of truth for pass/fail
+│  - Always runs              │
+│  - Produces: pass/fail bool │
+└──────────┬──────────────────┘
+           │ feeds status icon
+           ▼
+┌─────────────────────────────┐
+│  Display                    │
+│  - Icon from exit code      │
+│  - Summary from parser      │
+└──────────┬──────────────────┘
+           │ enriched by
+           ▼
+┌─────────────────────────────┐
+│  Parser Registry            │  ← extensible, tool-agnostic
+│  - Tries parsers in order   │
+│  - First match wins         │
+│  - Each parser has:         │
+│    • detect(output) → bool  │
+│    • summarize(output)      │
+│      → str | null           │
+│  - Fallback: generic text   │
+└─────────────────────────────┘
+```
+
+## Parser Module Interface
+
+Each parser is a self-contained module that exports two functions:
+
+| Function | Signature | Role |
+|---|---|---|
+| `detect` | `(output: string) => boolean` | Returns `true` if this tool's output is present. Checks for distinctive markers (e.g., vitest's "RUN" banner, pytest's `===` separator, cargo test's "running" lines). |
+| `summarize` | `(output: string) => string \| null` | Extracts a one-line human-readable summary (e.g., "3 passed (242ms)", "2 failed, 1 passed"). Returns `null` if no summary can be extracted. |
+
+## Lifecycle
+
+1. A tool execution completes (any bash command).
+2. Check if the command is a known test command (current behaviour).
+3. **Decision**: Read `event.details?.exitCode` — `0` means pass, anything else
+   means fail. This is the single source of truth.
+4. **Display — icon**: Set widget icon (● green or ● red) based on exit code.
+5. **Display — summary**: Walk the parser registry. For each parser, call
+   `detect(output)`. On first match, call `summarize(output)` for the text. If
+   no parser matches or `summarize` returns `null`, fall back to a generic
+   string ("All tests passed" / "Tests failed").
+
+## Adding a New Tool
+
+To add support for a new test framework, create a new file in the parsers
+directory that implements the two-function interface. Register it in the
+registry list. Zero changes to the core decision or display logic.
+
+Examples of tool-specific markers for `detect`:
+
+| Tool | Detect marker |
+|---|---|
+| Vitest | `" RUN "` banner |
+| Jest | `"PASS "` / `"FAIL "` prefixes |
+| pytest | `"=== test session starts ==="` |
+| Go test | `"ok  "` or `"FAIL  "` line prefixes |
+| cargo test | `"running "` followed by digit |
+| Mocha | `"passing"` / `"failing"` in summary |
+| RSpec | `"Finished in "` |
+
+## Benefits Over Current Approach
+
+| Aspect | Current (inline regex) | Proposed (modular parsers) |
+|---|---|---|
+| Pass/fail signal | Regex on output — brittle | Exit code — universal & correct |
+| Output parsing | Inline, single format | Registry of modular parsers |
+| Adding new tools | Edit the one function, risk regressions | Add a new file, register it |
+| False negatives | Possible (word "failed" in test name) | Eliminated (exit code is canonical) |
+| False positives | Possible (weird output formatting) | Eliminated |
+| Parsing failure impact | Can break pass/fail (wrong icon) | Only loses cosmetic detail (correct icon preserved) |

package.json

@@ -1,13 +1,18 @@
 {
   "name": "red-green-refactor-cycle-socrates-uk",
   "version": "0.0.1",
+  "private": true,
   "type": "module",
   "scripts": {
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
   },
   "devDependencies": {
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
+  },
+  "dependencies": {
+    "@earendil-works/pi-coding-agent": "^0.80.2"
   }
 }

pi-test-status/README.md

@@ -0,0 +1,57 @@
+# pi-test-status
+
+A [Pi](https://pi.dev) extension that watches for test runs and shows a pass/fail widget in the Pi UI.
+
+## Features
+
+- Detects `vitest` and `npm test` commands in bash tool calls
+- Parses vitest output to determine pass/fail status
+- Shows a live widget indicator (● green for pass, ● red for fail)
+- Displays the summary line from the test output
+
+## Installation
+
+```bash
+pi install npm:pi-test-status
+```
+
+Or with a scope:
+
+```bash
+pi install npm:@your-scope/pi-test-status
+```
+
+## Development
+
+```bash
+pi -e ./extensions/index.ts
+```
+
+Or from the package root:
+
+```bash
+pi -e .
+```
+
+## Usage
+
+Once installed, the extension automatically activates when you run tests:
+
+```bash
+# In Pi, just run your tests — the widget appears automatically
+npm test
+npx vitest
+vitest run
+```
+
+The widget shows:
+- Green ● with summary on test pass
+- Red ● with summary on test failure
+
+## How it works
+
+The extension subscribes to the `tool_result` event. When a bash tool call matches a test command, it parses the vitest output for pass/fail indicators and updates a Pi widget with the result.
+
+## License
+
+MIT

pi-test-status/extensions/index.ts

@@ -0,0 +1,28 @@
+import {type ExtensionAPI, isBashToolResult} from "@earendil-works/pi-coding-agent";
+import {isTestCommand, parseTestOutput} from "./parsing/typescript-command-output";
+
+// noinspection JSUnusedGlobalSymbols
+export default function testStatusExtension(pi: ExtensionAPI) {
+    // After every tool execution, check if tests just ran
+    pi.on("tool_result", async (event, ctx) => {
+        // Only care about bash commands
+        if (!isBashToolResult(event)) return;
+        if (!isTestCommand(event.input.command as string) ) return;
+
+        const textContent = event.content.find((c): c is { type: "text"; text: string } => c.type === "text");
+        const output = textContent?.text ?? "";
+        const theme = ctx.ui.theme;
+
+        const testResult = parseTestOutput(output);
+
+        if (testResult.status === "Success") {
+            ctx.ui.setWidget("test-status", [
+                `${theme.fg("success", "●")} ${theme.fg("text", testResult.summary)}`,
+            ]);
+        } else if (testResult.status === "Fail" || event.isError) {
+            ctx.ui.setWidget("test-status", [
+                `${theme.fg("error", "●")} ${theme.fg("text", testResult.summary)}`,
+            ]);
+        }
+    });
+}

pi-test-status/extensions/parsing/__tests__/vitest-output.test.ts

@@ -0,0 +1,60 @@
+import { describe, it, expect } from "vitest";
+import { parseTestOutput } from "../typescript-command-output";
+
+describe("parseTestOutput", () => {
+  it("parses a passing vitest run", () => {
+    const output = `
+ RAN  v3.2.6 /path
+
+ ✓ src/greet.test.ts (1 test) 1ms
+
+ Test Files  1 passed (1)
+      Tests  1 passed (1)
+   Start at  11:12:54
+   Duration  308ms
+`;
+
+    const result = parseTestOutput(output);
+
+    expect(result.status).toBe("Success");
+    expect(result.summary).toContain("1 passed");
+  });
+
+  it("parses a failing vitest run", () => {
+    const output = `
+ RAN  v3.2.6 /path
+
+ ✓ src/greet.test.ts (1 test) 1ms
+ ✗ src/foo.test.ts (1 test) 2ms
+
+ Test Files  1 failed (1)
+      Tests  1 failed (1)
+   Start at  11:15:00
+   Duration  400ms
+`;
+
+    const result = parseTestOutput(output);
+
+    expect(result.status).toBe("Fail");
+    expect(result.summary).toContain("1 failed");
+  });
+
+  it("returns Error and a fallback summary for unrecognised output", () => {
+    const result = parseTestOutput("some random output");
+
+    expect(result.status).toBe("Fail");
+    expect(result.summary).toBe("Tests failed");
+  });
+
+  it("returns Success with a fallback summary when passed but no detail line", () => {
+    const output = `
+ ✓ src/greet.test.ts (1 test) 1ms
+ Tests  1 passed (242ms)
+`;
+
+    const result = parseTestOutput(output);
+
+    expect(result.status).toBe("Success");
+    expect(result.summary).toBe("All tests passed");
+  });
+});

pi-test-status/extensions/parsing/typescript-command-output.ts

@@ -0,0 +1,25 @@
+export type TestResult =
+  | { status: "Success"; summary: string }
+  | { status: "Fail"; summary: string };
+
+export const isTestCommand = (command: string) => {
+  return command.includes("npm test") || command.includes("vitest");
+};
+
+export function parseTestOutput(output: string): TestResult {
+  const testsPassed =
+    /Tests\s+.+passed/.test(output) && !/Tests\s+.+failed/.test(output);
+  const testsFailed = /Tests\s+.+failed/.test(output);
+
+  if (testsPassed && !testsFailed) {
+    const line = output
+      .split("\n")
+      .find((l) => /Test Files\s+.+passed/.test(l));
+    return { status: "Success", summary: line?.trim() ?? "All tests passed" };
+  }
+
+  const line = output
+    .split("\n")
+    .find((l) => /Tests\s+.+failed/.test(l));
+  return { status: "Fail", summary: line?.trim() ?? "Tests failed" };
+}

pi-test-status/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "pi-test-status",
+  "version": "0.1.0",
+  "description": "Pi extension that shows test status (pass/fail) as a widget after running tests",
+  "keywords": ["pi-package", "pi-extension", "test-status", "vitest"],
+  "type": "module",
+  "main": "extensions/index.ts",
+  "files": [
+    "extensions",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "echo \"No tests yet\"",
+    "typecheck": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*"
+  },
+  "pi": {
+    "extensions": ["./extensions"]
+  },
+  "license": "MIT"
+}

pi-test-status/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "noEmit": true
+  },
+  "include": ["extensions"]
+}

pnpm-workspace.yaml

@@ -0,0 +1,3 @@
+packages:
+  - "."
+  - "pi-test-status"

vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: [
+      "src/**/*.test.ts",
+      "pi-test-status/extensions/**/*.test.ts",
+    ],
+  },
+});