app: coverage floor (86) + framework-glue exclusions
Measured before choosing: 86.17%, identical via `mix test --cover` and via the exported sample + `mix test.coverage`, stable across seeds. Floor is the integer below that, with the rule written next to it: a FLOOR, ratcheted up as behavior-test yaks land, never lowered (a drop means the change or an exclusion is wrong). Exclusions are the modules the suite cannot or should not exercise, each justified in the config comment: `use`-generated glue (Repo, Mailer, Telemetry, ErrorHTML/ErrorJSON), the `use FirehoseWeb` macro, production release tasks (Release), the lib_dev credo check (NoConnShadowing), and test scaffolding (DataCase, ConnCase, AccountsFixtures, Test.FakeBlog). Everything else counts, CoreComponents/controllers/LiveViews included. Wiring matches blogex and the fast/push split: `test.coverage` runs the suite once with `--cover --export-coverage default` (deleting stale exports first, since `mix test.coverage` imports every .coverdata it finds and would pool them), then the `coverage` sensor enforces the floor from that export. The sensor guards on the export existing because `mix test.coverage` exits 0 when it finds nothing — a floor check that did not run must not pass.
This commit is contained in:
+26
-8
@@ -3,7 +3,7 @@
|
|||||||
MISE_BIN ?= mise
|
MISE_BIN ?= mise
|
||||||
MISE_EXEC = $(MISE_BIN) exec --
|
MISE_EXEC = $(MISE_BIN) exec --
|
||||||
|
|
||||||
.PHONY: check precommit deps compile test format credo verify
|
.PHONY: check precommit deps compile test test.coverage coverage format credo verify
|
||||||
|
|
||||||
# Run all static analysis checks (no database required)
|
# Run all static analysis checks (no database required)
|
||||||
check: credo format
|
check: credo format
|
||||||
@@ -24,6 +24,20 @@ compile:
|
|||||||
test: deps compile
|
test: deps compile
|
||||||
$(MISE_EXEC) mix test
|
$(MISE_EXEC) mix test
|
||||||
|
|
||||||
|
# Run the suite once with coverage exported, so the coverage floor check (and,
|
||||||
|
# later, CRAP) reads the export instead of running the suite a second time.
|
||||||
|
# Stale exports are removed first: the floor must never report an older run.
|
||||||
|
test.coverage: deps compile
|
||||||
|
rm -f cover/*.coverdata
|
||||||
|
$(MISE_EXEC) mix test --cover --export-coverage default
|
||||||
|
|
||||||
|
# Enforce the coverage floor (mix.exs -> test_coverage) from that export. No
|
||||||
|
# test run. `mix test.coverage` exits 0 when it finds no export at all, so the
|
||||||
|
# guard below is load-bearing: a floor check that did not run must not pass.
|
||||||
|
coverage:
|
||||||
|
@test -f cover/default.coverdata || { echo "coverage: no exported coverage - run 'make test.coverage' first" >&2; exit 1; }
|
||||||
|
$(MISE_EXEC) mix test.coverage
|
||||||
|
|
||||||
# Format code
|
# Format code
|
||||||
format:
|
format:
|
||||||
$(MISE_EXEC) mix format
|
$(MISE_EXEC) mix format
|
||||||
@@ -106,15 +120,19 @@ credo:
|
|||||||
# target reports as wired.
|
# target reports as wired.
|
||||||
# - sobelow (yak: sobelow in the Lens pass)
|
# - sobelow (yak: sobelow in the Lens pass)
|
||||||
# - coverage (yak: coverage floors + framework-glue exclusions policy)
|
# - coverage (yak: coverage floors + framework-glue exclusions policy)
|
||||||
|
# WIRED below (test.coverage + coverage).
|
||||||
# - crap (yak: CRAP scores with ex_crap (both apps))
|
# - crap (yak: CRAP scores with ex_crap (both apps))
|
||||||
# - mutation (yak: mutation testing with muex — a sensor, not a gate)
|
# - mutation (yak: mutation testing with muex — a sensor, not a gate)
|
||||||
#
|
#
|
||||||
# Runtime note: the coverage yak should make this export coverage from a single
|
# Runtime note: the coverage sensor exports from a single suite run
|
||||||
# suite run (test --cover --export-coverage) that CRAP reuses, instead of
|
# (`test.coverage` -> `mix test --cover --export-coverage default`) that CRAP is
|
||||||
# running the suite twice.
|
# meant to reuse, instead of running the suite twice.
|
||||||
VERIFY_SENSORS :=
|
#
|
||||||
|
# The suite runs FIRST, because it is what produces the coverage export the
|
||||||
|
# sensors read; sensors still run cheapest-first among themselves.
|
||||||
|
VERIFY_SENSORS := coverage
|
||||||
|
|
||||||
verify:
|
verify:
|
||||||
@echo "==> verify(app): sensors [$(if $(VERIFY_SENSORS),$(VERIFY_SENSORS),none wired yet)]"
|
@echo "==> verify(app): suite (coverage exported), then sensors [$(VERIFY_SENSORS)]"
|
||||||
@for sensor in $(VERIFY_SENSORS); do $(MAKE) $$sensor || exit 1; done
|
@$(MAKE) test.coverage
|
||||||
@$(MAKE) test
|
@for sensor in $(VERIFY_SENSORS); do echo "==> verify(app): sensor $$sensor"; $(MAKE) $$sensor || exit 1; done
|
||||||
|
|||||||
+57
@@ -10,6 +10,7 @@ defmodule Firehose.MixProject do
|
|||||||
start_permanent: Mix.env() == :prod,
|
start_permanent: Mix.env() == :prod,
|
||||||
aliases: aliases(),
|
aliases: aliases(),
|
||||||
deps: deps(),
|
deps: deps(),
|
||||||
|
test_coverage: test_coverage(),
|
||||||
compilers: [:phoenix_live_view] ++ Mix.compilers(),
|
compilers: [:phoenix_live_view] ++ Mix.compilers(),
|
||||||
listeners: [Phoenix.CodeReloader]
|
listeners: [Phoenix.CodeReloader]
|
||||||
]
|
]
|
||||||
@@ -99,4 +100,60 @@ defmodule Firehose.MixProject do
|
|||||||
precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"]
|
precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"]
|
||||||
]
|
]
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# The coverage floor the Lens pass enforces (`make verify` -> `make coverage`
|
||||||
|
# in app/). Rule: the threshold is a FLOOR. Raise it as behavior-test yaks
|
||||||
|
# land; never lower it. A drop means the change (or an exclusion) is wrong,
|
||||||
|
# not the floor.
|
||||||
|
#
|
||||||
|
# Measured 2026-10-26 (PostgreSQL on localhost:5432, exclusions below):
|
||||||
|
# 86.17%, identical via `mix test --cover` and via
|
||||||
|
# `mix test --cover --export-coverage default` + `mix test.coverage`, and
|
||||||
|
# stable across seeds. Floor is the integer below that.
|
||||||
|
#
|
||||||
|
# `test.coverage` deletes stale exports before sampling: `mix test.coverage`
|
||||||
|
# imports *every* .coverdata it finds, so leftover exports from earlier runs
|
||||||
|
# pool into the total and quietly inflate it.
|
||||||
|
#
|
||||||
|
# Excluded are modules the suite cannot or should not exercise; counting them
|
||||||
|
# would move the number without saying anything about app behavior:
|
||||||
|
#
|
||||||
|
# * framework glue generated by `use`:
|
||||||
|
# Firehose.Repo `use Ecto.Repo`
|
||||||
|
# Firehose.Mailer `use Swoosh.Mailer`
|
||||||
|
# FirehoseWeb.Telemetry Phoenix's telemetry supervisor + metric list
|
||||||
|
# FirehoseWeb.ErrorHTML Phoenix error renderers, both delegating to
|
||||||
|
# FirehoseWeb.ErrorJSON Phoenix.Controller.status_message_from_template/1
|
||||||
|
# * FirehoseWeb the `use FirehoseWeb` macro: imports and aliases
|
||||||
|
# only, nothing that executes
|
||||||
|
# * Firehose.Release production release tasks (`mix ecto.migrate` on
|
||||||
|
# Dokku); the suite never runs them
|
||||||
|
# * Firehose.Checks.NoConnShadowing
|
||||||
|
# lib_dev credo check, exercised by `mix credo`
|
||||||
|
# and not by tests
|
||||||
|
# * test scaffolding Firehose.DataCase, FirehoseWeb.ConnCase,
|
||||||
|
# Firehose.AccountsFixtures, Firehose.Test.FakeBlog:
|
||||||
|
# fixtures measure the tests, not the app
|
||||||
|
#
|
||||||
|
# Everything else counts — CoreComponents, controllers, LiveViews included:
|
||||||
|
# that is where the behavior-test work shows up as a higher floor.
|
||||||
|
defp test_coverage do
|
||||||
|
[
|
||||||
|
summary: [threshold: 86],
|
||||||
|
ignore_modules: [
|
||||||
|
Firehose.Repo,
|
||||||
|
Firehose.Mailer,
|
||||||
|
Firehose.Release,
|
||||||
|
FirehoseWeb,
|
||||||
|
FirehoseWeb.Telemetry,
|
||||||
|
FirehoseWeb.ErrorHTML,
|
||||||
|
FirehoseWeb.ErrorJSON,
|
||||||
|
Firehose.Checks.NoConnShadowing,
|
||||||
|
Firehose.DataCase,
|
||||||
|
FirehoseWeb.ConnCase,
|
||||||
|
Firehose.AccountsFixtures,
|
||||||
|
Firehose.Test.FakeBlog
|
||||||
|
]
|
||||||
|
]
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
Reference in New Issue
Block a user