From 3ef38bc8fc75eac5c7e2d4d92f1c3820d157990d Mon Sep 17 00:00:00 2001 From: Willem van den Ende Date: Thu, 29 Oct 2026 10:07:55 +0000 Subject: [PATCH] yak(mutation-testing-with-muex-blogex-pure-app-code-1341): mutation testing with muex (blogex + pure app code) --- app/mix.exs | 66 ++++++++- app/mix.lock | 1 + app/mutation_scope.txt | 0 app/mutation_scope_fast.txt | 0 app/test/mutation_config_test.exs | 49 +++++++ app/test/mutation_scope_test.exs | 53 +++++++ blogex/mix.exs | 67 ++++++++- blogex/mix.lock | 1 + blogex/mutation_scope.txt | 8 + blogex/mutation_scope_fast.txt | 3 + blogex/test/blogex/link_validator_test.exs | 18 +++ blogex/test/mutation_config_test.exs | 82 +++++++++++ blogex/test/mutation_scope_test.exs | 64 ++++++++ doc/mutation-testing.md | 163 +++++++++++++++++++++ 14 files changed, 572 insertions(+), 3 deletions(-) create mode 100644 app/mutation_scope.txt create mode 100644 app/mutation_scope_fast.txt create mode 100644 app/test/mutation_config_test.exs create mode 100644 app/test/mutation_scope_test.exs create mode 100644 blogex/mutation_scope.txt create mode 100644 blogex/mutation_scope_fast.txt create mode 100644 blogex/test/mutation_config_test.exs create mode 100644 blogex/test/mutation_scope_test.exs create mode 100644 doc/mutation-testing.md diff --git a/app/mix.exs b/app/mix.exs index a709c56..b08ac8b 100644 --- a/app/mix.exs +++ b/app/mix.exs @@ -83,6 +83,7 @@ defmodule Firehose.MixProject do branch: "main"}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:ex_crap, "~> 0.1.1", only: [:dev, :test], runtime: false}, + {:muex, "~> 0.12", only: [:dev, :test], runtime: false}, {:sobelow, "~> 0.16", only: [:dev, :test], runtime: false} ] end @@ -93,6 +94,14 @@ defmodule Firehose.MixProject do # $ mix setup # # See the documentation for `Mix` for more info on aliases. + # + # `mutate` / `mutate.fast` are a sensor, not a gate: they are deliberately + # absent from Makefile's VERIFY_SENSORS and always pass `--fail-at 0`, so a + # survivor is reported rather than fatal. The scope lives in the plain-text + # files beside this one; the aliases read them at run time, so the text file + # stays the single source of truth. Both are empty today because the app's + # candidate modules are only tested through the database-backed DataCase; + # an empty scope raises instead of silently running over nothing. defp aliases do [ setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"], @@ -106,10 +115,65 @@ defmodule Firehose.MixProject do "esbuild firehose --minify", "phx.digest" ], - precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"] + precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"], + mutate: &mutate/1, + "mutate.fast": &mutate_fast/1 ] end + # No macro/`use` mutators: the web layer (where the DSLs live) is out of + # scope, and an empty scope never reaches muex at all. Spellings are muex + # 0.12's CLI names. + @mutation_mutators "arithmetic,boolean,comparison,conditional,function_call,literal,return_value" + + defp mutate(args), do: run_mutation("mutation_scope.txt", args) + + defp mutate_fast(args), do: run_mutation("mutation_scope_fast.txt", args) + + defp run_mutation(scope_file, args) do + Mix.Task.run("muex", mutation_args(scope_file, args)) + end + + @doc false + def mutation_args(scope_file, extra_args \\ []) do + files = read_mutation_scope(scope_file) + + # `--fail-at 0` comes first so an ad-hoc override cannot turn the sensor + # into a gate. Extra args come before our defaults, and OptionParser keeps + # the first value, so `mix mutate --files lib/foo.ex` targets one file. + ["--fail-at", "0"] ++ + extra_args ++ + ["--files", Enum.join(files, ","), "--mutators", @mutation_mutators] + end + + @doc false + def read_mutation_scope(scope_file) do + path = Path.expand(scope_file, Path.dirname(Mix.Project.project_file())) + + files = + case File.read(path) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + + {:error, reason} -> + Mix.raise( + "mutation scope #{scope_file} is unreadable (#{:file.format_error(reason)}); " <> + "refusing to run the mutation sensor over an unknown scope" + ) + end + + if files == [] do + Mix.raise( + "mutation scope #{scope_file} is empty; refusing to run the mutation sensor over nothing" + ) + end + + files + end + # The coverage floor the Lens pass enforces (`make verify` -> `make coverage` # in app/). Rule: the threshold is a FLOOR. Raise it as behavior-test yaks # land; never lower it. A drop means the change (or an exclusion) is wrong, diff --git a/app/mix.lock b/app/mix.lock index ba6f0ea..f4673dc 100644 --- a/app/mix.lock +++ b/app/mix.lock @@ -34,6 +34,7 @@ "microprints": {:git, "https://gitea.apps.sustainabledelivery.com/QWAN/microprints-phoenix.git", "f2a361555310518870ad3c093050169f30ff870c", [branch: "main"]}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.11.0", "a713551624815c0435237b93d90ea8b9b14254690c66f732d0ef8930f76ff1d9", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "c6279ba2d6aa3a383a1d4cfbe7b59f42e6efd400f58d8e2acfeac48a438693ab"}, + "muex": {:hex, :muex, "0.12.0", "80d8f1b2f7bfcba8ecb46b7b43dce55b555d33da54b84e74feedc4079fa2bd36", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a4b395b87d23b8d34f4508d0c13494b734f1fff8156eca2360fd279a0799d52e"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, diff --git a/app/mutation_scope.txt b/app/mutation_scope.txt new file mode 100644 index 0000000..e69de29 diff --git a/app/mutation_scope_fast.txt b/app/mutation_scope_fast.txt new file mode 100644 index 0000000..e69de29 diff --git a/app/test/mutation_config_test.exs b/app/test/mutation_config_test.exs new file mode 100644 index 0000000..4b13891 --- /dev/null +++ b/app/test/mutation_config_test.exs @@ -0,0 +1,49 @@ +defmodule Firehose.MutationConfigTest do + # Same wiring pins as blogex, plus the app-specific one: because the app + # scope is deliberately empty (Q45), the aliases must fail loudly. A sensor + # that reports a score for a run that tested nothing is worse than no sensor. + use ExUnit.Case + + test "muex is a dev/test-only, non-runtime dependency" do + deps = Mix.Project.config()[:deps] + + assert {:muex, requirement, opts} = dep!(deps, :muex) + assert requirement =~ "0.12" + assert opts[:only] == [:dev, :test] + assert opts[:runtime] == false + end + + test "mutate and mutate.fast aliases are defined" do + aliases = Mix.Project.config()[:aliases] + + assert is_function(aliases[:mutate], 1) + assert is_function(aliases[:"mutate.fast"], 1) + end + + test "an empty app scope fails loudly instead of running over nothing" do + assert_raise Mix.Error, ~r/empty/, fn -> + Firehose.MixProject.mutation_args("mutation_scope.txt", []) + end + end + + test "ad-hoc args do not bypass the empty-scope failure" do + assert_raise Mix.Error, ~r/empty/, fn -> + Firehose.MixProject.mutation_args("mutation_scope.txt", [ + "--files", + "lib/firehose/accounts/user.ex" + ]) + end + end + + defp dep!(deps, name) do + case Enum.find(deps, fn + {^name, _requirement} -> true + {^name, _requirement, _opts} -> true + _ -> false + end) do + {^name, requirement} -> {name, requirement, []} + {^name, requirement, opts} -> {name, requirement, opts} + nil -> flunk("expected #{inspect(name)} in deps/0") + end + end +end diff --git a/app/test/mutation_scope_test.exs b/app/test/mutation_scope_test.exs new file mode 100644 index 0000000..f4e8a50 --- /dev/null +++ b/app/test/mutation_scope_test.exs @@ -0,0 +1,53 @@ +defmodule Firehose.MutationScopeTest do + # The app's candidate pure modules (Firehose.Accounts.User, Scope and + # UserToken) are only exercised through Firehose.DataCase: their tests use + # the SQL sandbox, fixtures and Repo. Per the yak's Q45 decision, an app + # mutation run would therefore need PostgreSQL, so the scope is deliberately + # empty and the alias fails loudly rather than silently running over nothing. + # + # These pins keep that decision visible: if the scope grows, this file must + # change with it, and it may never grow into the web layer. + use ExUnit.Case + + @root Path.expand("..", __DIR__) + + test "scope files exist" do + assert File.exists?(Path.join(@root, "mutation_scope.txt")) + assert File.exists?(Path.join(@root, "mutation_scope_fast.txt")) + end + + test "app scope is empty because the candidate modules' tests need PostgreSQL" do + assert scope("mutation_scope.txt") == [] + end + + test "fast scope is a subset of full scope" do + out_of_scope = + MapSet.difference( + MapSet.new(scope("mutation_scope_fast.txt")), + MapSet.new(scope("mutation_scope.txt")) + ) + + assert MapSet.size(out_of_scope) == 0, + "mutate.fast names files outside the audited scope: #{inspect(MapSet.to_list(out_of_scope))}" + end + + test "scope never includes web-layer code" do + web_entries = + Enum.filter( + scope("mutation_scope.txt") ++ scope("mutation_scope_fast.txt"), + &String.starts_with?(&1, "lib/firehose_web") + ) + + assert web_entries == [], + "the web layer (controllers, LiveViews, components) is out of scope: #{inspect(web_entries)}" + end + + defp scope(file) do + @root + |> Path.join(file) + |> File.read!() + |> String.split("\n", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + end +end diff --git a/blogex/mix.exs b/blogex/mix.exs index 23ba5e4..885a612 100644 --- a/blogex/mix.exs +++ b/blogex/mix.exs @@ -48,7 +48,8 @@ defmodule Blogex.MixProject do {:plug, "~> 1.15"}, {:ex_doc, "~> 0.34", only: :dev, runtime: false}, {:rustler, "~> 0.38.0", optional: true}, - {:ex_crap, "~> 0.1.1", only: [:dev, :test], runtime: false} + {:ex_crap, "~> 0.1.1", only: [:dev, :test], runtime: false}, + {:muex, "~> 0.12", only: [:dev, :test], runtime: false} ] end @@ -91,12 +92,74 @@ defmodule Blogex.MixProject do # Same `precommit` alias name and shape as app/mix.exs, so both apps share # one loop: compile (warnings fatal), format, test. Heavy sensors belong in # the Lens pass (`make verify`), not here. + # + # `mutate` / `mutate.fast` are a sensor, not a gate: they are deliberately + # absent from Makefile's VERIFY_SENSORS and always pass `--fail-at 0`, so a + # survivor is reported rather than fatal. The scope lives in the plain-text + # files beside this one; the aliases read them at run time, so the text file + # stays the single source of truth. defp aliases do [ - precommit: ["compile --warning-as-errors", "format", "test"] + precommit: ["compile --warning-as-errors", "format", "test"], + mutate: &mutate/1, + "mutate.fast": &mutate_fast/1 ] end + # No macro/`use` mutators: the scoped modules are plain functions, and the + # macro-heavy modules (blog.ex, router.ex, components.ex, layout.ex) are out + # of scope entirely. Spellings are muex 0.12's CLI names. + @mutation_mutators "arithmetic,boolean,comparison,conditional,function_call,literal,return_value" + + defp mutate(args), do: run_mutation("mutation_scope.txt", args) + + defp mutate_fast(args), do: run_mutation("mutation_scope_fast.txt", args) + + defp run_mutation(scope_file, args) do + Mix.Task.run("muex", mutation_args(scope_file, args)) + end + + @doc false + def mutation_args(scope_file, extra_args \\ []) do + files = read_mutation_scope(scope_file) + + # `--fail-at 0` comes first so an ad-hoc override cannot turn the sensor + # into a gate. Extra args come before our defaults, and OptionParser keeps + # the first value, so `mix mutate --files lib/blogex/seo.ex` targets one + # file while `mix mutate` covers the whole scope file. + ["--fail-at", "0"] ++ + extra_args ++ + ["--files", Enum.join(files, ","), "--mutators", @mutation_mutators] + end + + @doc false + def read_mutation_scope(scope_file) do + path = Path.expand(scope_file, Path.dirname(Mix.Project.project_file())) + + files = + case File.read(path) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + + {:error, reason} -> + Mix.raise( + "mutation scope #{scope_file} is unreadable (#{:file.format_error(reason)}); " <> + "refusing to run the mutation sensor over an unknown scope" + ) + end + + if files == [] do + Mix.raise( + "mutation scope #{scope_file} is empty; refusing to run the mutation sensor over nothing" + ) + end + + files + end + defp package do [ licenses: ["MIT"], diff --git a/blogex/mix.lock b/blogex/mix.lock index 02a418f..3a6d5f5 100644 --- a/blogex/mix.lock +++ b/blogex/mix.lock @@ -11,6 +11,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "mdex_native": {:hex, :mdex_native, "0.2.4", "a74b9d799d3fc487b1f97b70ee9b04108c7af0514920d56828e173d19312c2a9", [:mix], [{:rustler, "~> 0.32", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.7", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "d71185c43c5209c6939f4b65f903f71c892512c93ea6843171227855d2d976f5"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "muex": {:hex, :muex, "0.12.0", "80d8f1b2f7bfcba8ecb46b7b43dce55b555d33da54b84e74feedc4079fa2bd36", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a4b395b87d23b8d34f4508d0c13494b734f1fff8156eca2360fd279a0799d52e"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_publisher": {:hex, :nimble_publisher, "2.0.0", "f4fc100949ccdf1831094ff303b79f27c6f04b2ead829409426ee0d9ed3aa55b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: true]}, {:mdex_native, "~> 0.1", [hex: :mdex_native, repo: "hexpm", optional: false]}], "hexpm", "88997f74b763c24a15d20a90d7cd4ef1fd01172a2b4651e54d8021a44659f2c8"}, "phoenix": {:hex, :phoenix, "1.8.15", "dcdb304113660ec97e1579458824d3c34b5d2eed07a0fdf25a836a9515b63146", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 2.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "7b83ed6b3d544f24a29277eab7f051be38b76f390bb511bb6ddb7ec6e8e05b95"}, diff --git a/blogex/mutation_scope.txt b/blogex/mutation_scope.txt new file mode 100644 index 0000000..bc2eb71 --- /dev/null +++ b/blogex/mutation_scope.txt @@ -0,0 +1,8 @@ +lib/blogex/link_validator.ex +lib/blogex/seo.ex +lib/blogex/feed.ex +lib/blogex/post.ex +lib/blogex/external_links.ex +lib/blogex/registry.ex +lib/blogex/not_found_error.ex +lib/blogex/link_error.ex diff --git a/blogex/mutation_scope_fast.txt b/blogex/mutation_scope_fast.txt new file mode 100644 index 0000000..00ba1a1 --- /dev/null +++ b/blogex/mutation_scope_fast.txt @@ -0,0 +1,3 @@ +lib/blogex/link_validator.ex +lib/blogex/seo.ex +lib/blogex/post.ex diff --git a/blogex/test/blogex/link_validator_test.exs b/blogex/test/blogex/link_validator_test.exs index b06599b..1fc8198 100644 --- a/blogex/test/blogex/link_validator_test.exs +++ b/blogex/test/blogex/link_validator_test.exs @@ -245,6 +245,24 @@ defmodule Blogex.LinkValidatorTest do end describe "validate_body/2" do + # The guard is a conjunction, not a disjunction: both arguments must have + # the documented types. Mutating `and` to `or` would let a string blog id + # (or a non-binary body) through the guard to crash somewhere else, so pin + # the contract at the door. + test "rejects a non-atom blog id at the function clause guard" do + assert_raise FunctionClauseError, fn -> + # apply/3 keeps the compiler's type checker quiet: the point is the + # runtime guard, and the test deliberately passes an invalid type. + apply(LinkValidator, :validate_body, ["Just text, no links.", "engineering"]) + end + end + + test "rejects a non-binary body at the function clause guard" do + assert_raise FunctionClauseError, fn -> + apply(LinkValidator, :validate_body, [[], :engineering]) + end + end + test "returns :ok when body has no internal blog links" do body = "Just text, no links." diff --git a/blogex/test/mutation_config_test.exs b/blogex/test/mutation_config_test.exs new file mode 100644 index 0000000..edeead4 --- /dev/null +++ b/blogex/test/mutation_config_test.exs @@ -0,0 +1,82 @@ +defmodule Blogex.MutationConfigTest do + # The mutation sensor is opt-in: a dependency and two aliases, deliberately + # absent from `make verify`'s VERIFY_SENSORS. These pins keep the wiring + # honest (dev/test only, never runtime) and keep the aliases reading the + # scope files rather than duplicating a list in mix.exs. + use ExUnit.Case + + test "muex is a dev/test-only, non-runtime dependency" do + deps = Mix.Project.config()[:deps] + + assert {:muex, requirement, opts} = dep!(deps, :muex) + assert requirement =~ "0.12" + assert opts[:only] == [:dev, :test] + assert opts[:runtime] == false + end + + test "mutate and mutate.fast aliases are defined" do + aliases = Mix.Project.config()[:aliases] + + assert is_function(aliases[:mutate], 1) + assert is_function(aliases[:"mutate.fast"], 1) + end + + test "mutation args always include the scope, the curated mutators and --fail-at 0" do + args = Blogex.MixProject.mutation_args("mutation_scope_fast.txt", []) + scope = Blogex.MixProject.read_mutation_scope("mutation_scope_fast.txt") + + assert "--fail-at" in args + assert Enum.at(args, Enum.find_index(args, &(&1 == "--fail-at")) + 1) == "0" + assert files_arg(args) == Enum.join(scope, ",") + + mutators = Enum.at(args, Enum.find_index(args, &(&1 == "--mutators")) + 1) + + assert mutators == + "arithmetic,boolean,comparison,conditional,function_call,literal,return_value" + end + + test "extra CLI args pass through so a single file can be targeted" do + args = + Blogex.MixProject.mutation_args("mutation_scope_fast.txt", ["--files", "lib/blogex/seo.ex"]) + + assert args + |> Enum.chunk_every(2, 1, :discard) + |> Enum.any?(fn + ["--files", "lib/blogex/seo.ex"] -> true + _ -> false + end) + end + + test "an empty scope fails loudly instead of running over nothing" do + path = + Path.join(System.tmp_dir!(), "muex-empty-scope-#{System.unique_integer([:positive])}.txt") + + File.write!(path, "\n \n") + on_exit(fn -> File.rm(path) end) + + assert_raise Mix.Error, ~r/empty/, fn -> + Blogex.MixProject.read_mutation_scope(path) + end + end + + defp dep!(deps, name) do + case Enum.find(deps, fn + {^name, _requirement} -> true + {^name, _requirement, _opts} -> true + _ -> false + end) do + {^name, requirement} -> {name, requirement, []} + {^name, requirement, opts} -> {name, requirement, opts} + nil -> flunk("expected #{inspect(name)} in deps/0") + end + end + + defp files_arg(args) do + args + |> Enum.chunk_every(2, 1, :discard) + |> Enum.find_value(fn + ["--files", files] -> files + _ -> nil + end) + end +end diff --git a/blogex/test/mutation_scope_test.exs b/blogex/test/mutation_scope_test.exs new file mode 100644 index 0000000..4f5f8c7 --- /dev/null +++ b/blogex/test/mutation_scope_test.exs @@ -0,0 +1,64 @@ +defmodule Blogex.MutationScopeTest do + # The mutation scope files are the single source of truth for what `mix + # mutate` / `mix mutate.fast` are allowed to touch. These pins keep the + # sensor honest: a scope that quietly shrinks to an empty (or stale) set + # would report a score for a check it never ran. + use ExUnit.Case + + @root Path.expand("..", __DIR__) + @full "mutation_scope.txt" + @fast "mutation_scope_fast.txt" + + # Pure modules only. These are excluded deliberately: + # blogex.ex facade + moduledoc, nothing to mutate + # blogex/blog.ex the `__using__` macro (NimblePublisher expansion) + # router.ex Phoenix router DSL + # components.ex Phoenix component DSL + # layout.ex Phoenix/LiveView layout DSL + @shell_modules [ + "lib/blogex.ex", + "lib/blogex/blog.ex", + "lib/blogex/router.ex", + "lib/blogex/components.ex", + "lib/blogex/layout.ex" + ] + + test "full scope lists only existing files" do + missing = Enum.reject(scope(@full), &File.exists?(Path.join(@root, &1))) + + assert missing == [], + "mutation scope names files that do not exist: #{inspect(missing)}" + end + + test "full scope is not empty" do + assert scope(@full) != [] + end + + test "fast scope is not empty" do + assert scope(@fast) != [] + end + + test "fast scope is a subset of full scope" do + out_of_scope = MapSet.difference(MapSet.new(scope(@fast)), MapSet.new(scope(@full))) + + assert MapSet.size(out_of_scope) == 0, + "mutate.fast names files outside the audited scope: #{inspect(MapSet.to_list(out_of_scope))}" + end + + test "scope excludes the macro and shell modules" do + scoped = MapSet.new(scope(@full) ++ scope(@fast)) + present = Enum.filter(@shell_modules, &MapSet.member?(scoped, &1)) + + assert present == [], + "shell/macro modules belong outside the mutation scope: #{inspect(present)}" + end + + defp scope(file) do + @root + |> Path.join(file) + |> File.read!() + |> String.split("\n", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + end +end diff --git a/doc/mutation-testing.md b/doc/mutation-testing.md new file mode 100644 index 0000000..73342e8 --- /dev/null +++ b/doc/mutation-testing.md @@ -0,0 +1,163 @@ +# Mutation testing with muex + +Mutation testing is a **sensor, not a gate**. It deliberately breaks +production code one mutant at a time and re-runs the suite: a mutant *killed* +by a test is evidence the test pins that behaviour; a mutant that *survives* +is a question for a Lens session. Runs always pass `--fail-at 0`, so a survivor +is reported and triaged, never a failing build. + +Nothing here is wired into `VERIFY_SENSORS`. `make check`, `make verify`, +coverage floors and `.githooks/pre-push` are unchanged by this yak. + +## How to run it + +Both apps declare two aliases in `mix.exs`; both read a plain-text scope file +at run time, so the file stays the single source of truth. + +| Command | Scope | Blogex wall clock (2026-10-28) | +|---------|-------|--------------------------------| +| `mix mutate.fast` | `mutation_scope_fast.txt` (3 modules) | ~3 s | +| `mix mutate` | `mutation_scope.txt` (8 modules) | ~4 s | + +The app declares the same aliases but both scope files are empty (see +"Scope: app" below), so `mix mutate` there raises a clear Mix error instead of +running a sensor over nothing. + +Extra arguments pass through to muex, so a one-off target is explicit: + +```bash +cd blogex +mix mutate --files lib/blogex/seo.ex # one module, ad hoc +mix mutate.fast --no-optimize # full mutant list, slower +``` + +The aliases pin `--fail-at 0` first, so an ad-hoc flag cannot turn the sensor +into a gate. `mix mutate` does not enable `--cover`: muex runs each mutant's +suite under `MIX_ENV=test`, and coverage would let a code-deleting mutant trip +the project's coverage floor and abort the run. + +## Scope: blogex + +| In scope (`mutation_scope.txt`) | Why | +|---------------------------------|-----| +| `lib/blogex/link_validator.ex` | Pure link parsing/validation; the yak's example module. | +| `lib/blogex/seo.ex` | Pure sitemap/JSON-LD/OpenGraph string building. | +| `lib/blogex/feed.ex` | Pure feed generation. | +| `lib/blogex/post.ex` | Pure post parsing/visibility helpers. | +| `lib/blogex/external_links.ex` | Pure HTML rewriting for external links. | +| `lib/blogex/registry.ex` | Module-registry lookup; no I/O. | +| `lib/blogex/not_found_error.ex` | Exception definition. | +| `lib/blogex/link_error.ex` | Exception definition. | + +| Out of scope | Why | +|--------------|-----| +| `lib/blogex.ex` | Facade + moduledoc: delegation and docs, no logic to mutate. | +| `lib/blogex/blog.ex` | A `defmacro __using__` that expands `NimblePublisher`; mutating macro bodies produces noise, not behaviour. | +| `lib/blogex/router.ex` | Phoenix router DSL (`use`, `scope`, `get`); macro-generated. | +| `lib/blogex/components.ex` | Phoenix component DSL (`use`, `attr`, `slot`, `~H`). | +| `lib/blogex/layout.ex` | LiveView layout DSL. | + +`mutation_scope_fast.txt` is a subset (`link_validator`, `seo`, `post`) so the +inner loop stays seconds long; a pin in +`test/mutation_scope_test.exs` fails if it ever drifts outside the audited +scope. Another pin fails if any scoped path stops existing, so a rename cannot +silently leave the sensor reporting a score for a reduced set. + +## Mutators + +The aliases pass an explicit curated set (muex 0.12 CLI names): + +``` +arithmetic, boolean, comparison, conditional, function_call, literal, return_value +``` + +Macro/`use`-based mutators are deliberately absent. The scoped modules are plain +functions, and the macro-heavy modules are out of scope; macro mutations produce +false-positive survivors rather than behavioural questions. The `arithmetic`, +`boolean`, `comparison` and `conditional` families are the behavioural core; +`function_call`, `literal` and `return_value` catch removed calls, changed +constants and stubbed return values. + +## Baseline (blogex, 2026-10-28) + +Command: `mix mutate` (default muex optimization, `--fail-at 0`). +Result: **19 mutants, 12 killed, 0 survived, 0 invalid, 7 equivalent — 100%.** +Wall clock: 3.5 s. The run that found the survivor below was 11 killed / 1 +survived (91.67%) before its pin landed. + +| Module | Mutants | Killed | Survived | Equivalent | +|--------|--------:|-------:|---------:|-----------:| +| `link_validator.ex` | 7 | 4 | 0 | 3 | +| `external_links.ex` | 6 | 5 | 0 | 1 | +| `registry.ex` | 4 | 1 | 0 | 3 | +| `post.ex` | 1 | 1 | 0 | 0 | +| `seo.ex` | 1 | 1 | 0 | 0 | +| `feed.ex` | 0 | 0 | 0 | 0 | +| `not_found_error.ex` | 0 | 0 | 0 | 0 | +| `link_error.ex` | 0 | 0 | 0 | 0 | + +muex's file analyser gives `not_found_error.ex` a complexity score of 0 and +skips it under the default filter. The scope file still lists it: it is pure +code and stays audited if it grows logic. `feed.ex` is analysed but the curated +mutator set generates nothing for it. + +### Survivors found and killed + +| Mutant | Resolution | +|--------|------------| +| `link_validator.ex:201` — `Boolean: and to or` in `validate_body/3`'s guard `when is_binary(body) and is_atom(blog)` | Genuine: the mutant let a string blog id through the guard. Pinned by two `FunctionClauseError` tests in `test/blogex/link_validator_test.exs`; `mix mutate.fast` then reports 0 survivors. | + +## False-positive classes + +These are not gaps in behaviour; do not weaken a test to "kill" them. + +- **Commutative argument swaps.** `function_call` mutates `a and b` to + `b and a`, `a == b` to `b == a`, and `a || b` to `b || a`. These compile to + the same behaviour, and muex's trivial-compiler-equivalence check marks them + `equivalent` (7 of the 19 baseline mutants). They are noise from the + mutator, not survivors. +- **Removed log lines.** A removed `Logger` call changes no return value; the + suite cannot observe it. Such a mutant is documented here instead of adding + a log-capture assertion that pins an implementation detail. +- **`and`/`or` in guards is *not* automatically a false positive.** The + baseline survivor above was a real guard-contract gap; only commutative + rewrites are equivalent. +- **Invalid mutants.** Mutated code that does not compile is counted + `invalid`, not `survived`, and excluded from the score. + +## Deeper runs + +muex's default `balanced` optimization reduced 1304 generated mutants to 12 +tested ones (99.1%) on the blogex full scope. That is what keeps the sensor in +seconds, but it is sampled, not exhaustive. `mix mutate --no-optimize` tests +everything; on the three-module fast scope it took ~2 min 15 s and produced +434 mutants (329 killed, 19 survived, 83 invalid, 3 equivalent) before the +guard pin. Use it when triaging a specific module, not in a loop. + +Re-baseline after a yak that changes the scoped modules (notably the Boundary +work, which adds `use` lines inside them); do not compare scores across such a +change. + +## Scope: app + +**Deliberately empty; no app baseline was run.** The candidate pure modules +are `Firehose.Accounts.User`, `Firehose.Accounts.Scope` and +`Firehose.Accounts.UserToken`, but every test that exercises them lives in +`test/firehose/accounts_test.exs`, which uses `Firehose.DataCase`, fixtures and +`Repo`; the SQL sandbox needs PostgreSQL. The yak's Q45 decision was: use those +modules only after confirming their tests need no database, otherwise leave the +app scope empty. They need one, so the scope is empty. + +Consequences, all pinned by `app/test/mutation_scope_test.exs` and +`app/test/mutation_config_test.exs`: + +- Both app scope files list nothing, so `mix mutate` / `mix mutate.fast` raise + `mutation scope … is empty; refusing to run the mutation sensor over nothing`. +- Ad-hoc `--files` arguments do not bypass the guard: the scope is validated + before muex arguments are built. +- No scope entry may ever begin with `lib/firehose_web` (controllers, + LiveViews, components). + +There is no app mutation baseline to report. Do not add one until the scoped +modules have tests that run without the database; change the scope file and +these pins in the same commit.