yak(mutation-testing-with-muex-blogex-pure-app-code-1341): mutation testing with muex (blogex + pure app code)

app/mix.exs

@@ -83,6 +83,7 @@ defmodule Firehose.MixProject do
        branch: "main"},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
       {:ex_crap, "~> 0.1.1", only: [:dev, :test], runtime: false},
+      {:muex, "~> 0.12", only: [:dev, :test], runtime: false},
       {:sobelow, "~> 0.16", only: [:dev, :test], runtime: false}
     ]
   end
@@ -93,6 +94,14 @@ defmodule Firehose.MixProject do
   #     $ mix setup
   #
   # See the documentation for `Mix` for more info on aliases.
+  #
+  # `mutate` / `mutate.fast` are a sensor, not a gate: they are deliberately
+  # absent from Makefile's VERIFY_SENSORS and always pass `--fail-at 0`, so a
+  # survivor is reported rather than fatal. The scope lives in the plain-text
+  # files beside this one; the aliases read them at run time, so the text file
+  # stays the single source of truth. Both are empty today because the app's
+  # candidate modules are only tested through the database-backed DataCase;
+  # an empty scope raises instead of silently running over nothing.
   defp aliases do
     [
       setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"],
@@ -106,10 +115,65 @@ defmodule Firehose.MixProject do
         "esbuild firehose --minify",
         "phx.digest"
       ],
-      precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"]
+      precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"],
+      mutate: &mutate/1,
+      "mutate.fast": &mutate_fast/1
     ]
   end
 
+  # No macro/`use` mutators: the web layer (where the DSLs live) is out of
+  # scope, and an empty scope never reaches muex at all. Spellings are muex
+  # 0.12's CLI names.
+  @mutation_mutators "arithmetic,boolean,comparison,conditional,function_call,literal,return_value"
+
+  defp mutate(args), do: run_mutation("mutation_scope.txt", args)
+
+  defp mutate_fast(args), do: run_mutation("mutation_scope_fast.txt", args)
+
+  defp run_mutation(scope_file, args) do
+    Mix.Task.run("muex", mutation_args(scope_file, args))
+  end
+
+  @doc false
+  def mutation_args(scope_file, extra_args \\ []) do
+    files = read_mutation_scope(scope_file)
+
+    # `--fail-at 0` comes first so an ad-hoc override cannot turn the sensor
+    # into a gate. Extra args come before our defaults, and OptionParser keeps
+    # the first value, so `mix mutate --files lib/foo.ex` targets one file.
+    ["--fail-at", "0"] ++
+      extra_args ++
+      ["--files", Enum.join(files, ","), "--mutators", @mutation_mutators]
+  end
+
+  @doc false
+  def read_mutation_scope(scope_file) do
+    path = Path.expand(scope_file, Path.dirname(Mix.Project.project_file()))
+
+    files =
+      case File.read(path) do
+        {:ok, contents} ->
+          contents
+          |> String.split("\n", trim: true)
+          |> Enum.map(&String.trim/1)
+          |> Enum.reject(&(&1 == ""))
+
+        {:error, reason} ->
+          Mix.raise(
+            "mutation scope #{scope_file} is unreadable (#{:file.format_error(reason)}); " <>
+              "refusing to run the mutation sensor over an unknown scope"
+          )
+      end
+
+    if files == [] do
+      Mix.raise(
+        "mutation scope #{scope_file} is empty; refusing to run the mutation sensor over nothing"
+      )
+    end
+
+    files
+  end
+
   # The coverage floor the Lens pass enforces (`make verify` -> `make coverage`
   # in app/). Rule: the threshold is a FLOOR. Raise it as behavior-test yaks
   # land; never lower it. A drop means the change (or an exclusion) is wrong,

app/mix.lock

@@ -34,6 +34,7 @@
   "microprints": {:git, "https://gitea.apps.sustainabledelivery.com/QWAN/microprints-phoenix.git", "f2a361555310518870ad3c093050169f30ff870c", [branch: "main"]},
   "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
   "mint": {:hex, :mint, "1.11.0", "a713551624815c0435237b93d90ea8b9b14254690c66f732d0ef8930f76ff1d9", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "c6279ba2d6aa3a383a1d4cfbe7b59f42e6efd400f58d8e2acfeac48a438693ab"},
+  "muex": {:hex, :muex, "0.12.0", "80d8f1b2f7bfcba8ecb46b7b43dce55b555d33da54b84e74feedc4079fa2bd36", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a4b395b87d23b8d34f4508d0c13494b734f1fff8156eca2360fd279a0799d52e"},
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},

app/test/mutation_config_test.exs

@@ -0,0 +1,49 @@
+defmodule Firehose.MutationConfigTest do
+  # Same wiring pins as blogex, plus the app-specific one: because the app
+  # scope is deliberately empty (Q45), the aliases must fail loudly. A sensor
+  # that reports a score for a run that tested nothing is worse than no sensor.
+  use ExUnit.Case
+
+  test "muex is a dev/test-only, non-runtime dependency" do
+    deps = Mix.Project.config()[:deps]
+
+    assert {:muex, requirement, opts} = dep!(deps, :muex)
+    assert requirement =~ "0.12"
+    assert opts[:only] == [:dev, :test]
+    assert opts[:runtime] == false
+  end
+
+  test "mutate and mutate.fast aliases are defined" do
+    aliases = Mix.Project.config()[:aliases]
+
+    assert is_function(aliases[:mutate], 1)
+    assert is_function(aliases[:"mutate.fast"], 1)
+  end
+
+  test "an empty app scope fails loudly instead of running over nothing" do
+    assert_raise Mix.Error, ~r/empty/, fn ->
+      Firehose.MixProject.mutation_args("mutation_scope.txt", [])
+    end
+  end
+
+  test "ad-hoc args do not bypass the empty-scope failure" do
+    assert_raise Mix.Error, ~r/empty/, fn ->
+      Firehose.MixProject.mutation_args("mutation_scope.txt", [
+        "--files",
+        "lib/firehose/accounts/user.ex"
+      ])
+    end
+  end
+
+  defp dep!(deps, name) do
+    case Enum.find(deps, fn
+           {^name, _requirement} -> true
+           {^name, _requirement, _opts} -> true
+           _ -> false
+         end) do
+      {^name, requirement} -> {name, requirement, []}
+      {^name, requirement, opts} -> {name, requirement, opts}
+      nil -> flunk("expected #{inspect(name)} in deps/0")
+    end
+  end
+end

app/test/mutation_scope_test.exs

@@ -0,0 +1,53 @@
+defmodule Firehose.MutationScopeTest do
+  # The app's candidate pure modules (Firehose.Accounts.User, Scope and
+  # UserToken) are only exercised through Firehose.DataCase: their tests use
+  # the SQL sandbox, fixtures and Repo. Per the yak's Q45 decision, an app
+  # mutation run would therefore need PostgreSQL, so the scope is deliberately
+  # empty and the alias fails loudly rather than silently running over nothing.
+  #
+  # These pins keep that decision visible: if the scope grows, this file must
+  # change with it, and it may never grow into the web layer.
+  use ExUnit.Case
+
+  @root Path.expand("..", __DIR__)
+
+  test "scope files exist" do
+    assert File.exists?(Path.join(@root, "mutation_scope.txt"))
+    assert File.exists?(Path.join(@root, "mutation_scope_fast.txt"))
+  end
+
+  test "app scope is empty because the candidate modules' tests need PostgreSQL" do
+    assert scope("mutation_scope.txt") == []
+  end
+
+  test "fast scope is a subset of full scope" do
+    out_of_scope =
+      MapSet.difference(
+        MapSet.new(scope("mutation_scope_fast.txt")),
+        MapSet.new(scope("mutation_scope.txt"))
+      )
+
+    assert MapSet.size(out_of_scope) == 0,
+           "mutate.fast names files outside the audited scope: #{inspect(MapSet.to_list(out_of_scope))}"
+  end
+
+  test "scope never includes web-layer code" do
+    web_entries =
+      Enum.filter(
+        scope("mutation_scope.txt") ++ scope("mutation_scope_fast.txt"),
+        &String.starts_with?(&1, "lib/firehose_web")
+      )
+
+    assert web_entries == [],
+           "the web layer (controllers, LiveViews, components) is out of scope: #{inspect(web_entries)}"
+  end
+
+  defp scope(file) do
+    @root
+    |> Path.join(file)
+    |> File.read!()
+    |> String.split("\n", trim: true)
+    |> Enum.map(&String.trim/1)
+    |> Enum.reject(&(&1 == ""))
+  end
+end