yak(mutation-testing-with-muex-blogex-pure-app-code-1341): mutation testing with muex (blogex + pure app code)

blogex/mix.exs

@@ -48,7 +48,8 @@ defmodule Blogex.MixProject do
       {:plug, "~> 1.15"},
       {:ex_doc, "~> 0.34", only: :dev, runtime: false},
       {:rustler, "~> 0.38.0", optional: true},
-      {:ex_crap, "~> 0.1.1", only: [:dev, :test], runtime: false}
+      {:ex_crap, "~> 0.1.1", only: [:dev, :test], runtime: false},
+      {:muex, "~> 0.12", only: [:dev, :test], runtime: false}
     ]
   end
 
@@ -91,12 +92,74 @@ defmodule Blogex.MixProject do
   # Same `precommit` alias name and shape as app/mix.exs, so both apps share
   # one loop: compile (warnings fatal), format, test. Heavy sensors belong in
   # the Lens pass (`make verify`), not here.
+  #
+  # `mutate` / `mutate.fast` are a sensor, not a gate: they are deliberately
+  # absent from Makefile's VERIFY_SENSORS and always pass `--fail-at 0`, so a
+  # survivor is reported rather than fatal. The scope lives in the plain-text
+  # files beside this one; the aliases read them at run time, so the text file
+  # stays the single source of truth.
   defp aliases do
     [
-      precommit: ["compile --warning-as-errors", "format", "test"]
+      precommit: ["compile --warning-as-errors", "format", "test"],
+      mutate: &mutate/1,
+      "mutate.fast": &mutate_fast/1
     ]
   end
 
+  # No macro/`use` mutators: the scoped modules are plain functions, and the
+  # macro-heavy modules (blog.ex, router.ex, components.ex, layout.ex) are out
+  # of scope entirely. Spellings are muex 0.12's CLI names.
+  @mutation_mutators "arithmetic,boolean,comparison,conditional,function_call,literal,return_value"
+
+  defp mutate(args), do: run_mutation("mutation_scope.txt", args)
+
+  defp mutate_fast(args), do: run_mutation("mutation_scope_fast.txt", args)
+
+  defp run_mutation(scope_file, args) do
+    Mix.Task.run("muex", mutation_args(scope_file, args))
+  end
+
+  @doc false
+  def mutation_args(scope_file, extra_args \\ []) do
+    files = read_mutation_scope(scope_file)
+
+    # `--fail-at 0` comes first so an ad-hoc override cannot turn the sensor
+    # into a gate. Extra args come before our defaults, and OptionParser keeps
+    # the first value, so `mix mutate --files lib/blogex/seo.ex` targets one
+    # file while `mix mutate` covers the whole scope file.
+    ["--fail-at", "0"] ++
+      extra_args ++
+      ["--files", Enum.join(files, ","), "--mutators", @mutation_mutators]
+  end
+
+  @doc false
+  def read_mutation_scope(scope_file) do
+    path = Path.expand(scope_file, Path.dirname(Mix.Project.project_file()))
+
+    files =
+      case File.read(path) do
+        {:ok, contents} ->
+          contents
+          |> String.split("\n", trim: true)
+          |> Enum.map(&String.trim/1)
+          |> Enum.reject(&(&1 == ""))
+
+        {:error, reason} ->
+          Mix.raise(
+            "mutation scope #{scope_file} is unreadable (#{:file.format_error(reason)}); " <>
+              "refusing to run the mutation sensor over an unknown scope"
+          )
+      end
+
+    if files == [] do
+      Mix.raise(
+        "mutation scope #{scope_file} is empty; refusing to run the mutation sensor over nothing"
+      )
+    end
+
+    files
+  end
+
   defp package do
     [
       licenses: ["MIT"],

blogex/mix.lock

@@ -11,6 +11,7 @@
   "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"},
   "mdex_native": {:hex, :mdex_native, "0.2.4", "a74b9d799d3fc487b1f97b70ee9b04108c7af0514920d56828e173d19312c2a9", [:mix], [{:rustler, "~> 0.32", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.7", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "d71185c43c5209c6939f4b65f903f71c892512c93ea6843171227855d2d976f5"},
   "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
+  "muex": {:hex, :muex, "0.12.0", "80d8f1b2f7bfcba8ecb46b7b43dce55b555d33da54b84e74feedc4079fa2bd36", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a4b395b87d23b8d34f4508d0c13494b734f1fff8156eca2360fd279a0799d52e"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},
   "nimble_publisher": {:hex, :nimble_publisher, "2.0.0", "f4fc100949ccdf1831094ff303b79f27c6f04b2ead829409426ee0d9ed3aa55b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: true]}, {:mdex_native, "~> 0.1", [hex: :mdex_native, repo: "hexpm", optional: false]}], "hexpm", "88997f74b763c24a15d20a90d7cd4ef1fd01172a2b4651e54d8021a44659f2c8"},
   "phoenix": {:hex, :phoenix, "1.8.15", "dcdb304113660ec97e1579458824d3c34b5d2eed07a0fdf25a836a9515b63146", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 2.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "7b83ed6b3d544f24a29277eab7f051be38b76f390bb511bb6ddb7ec6e8e05b95"},

blogex/mutation_scope.txt

@@ -0,0 +1,8 @@
+lib/blogex/link_validator.ex
+lib/blogex/seo.ex
+lib/blogex/feed.ex
+lib/blogex/post.ex
+lib/blogex/external_links.ex
+lib/blogex/registry.ex
+lib/blogex/not_found_error.ex
+lib/blogex/link_error.ex

blogex/mutation_scope_fast.txt

@@ -0,0 +1,3 @@
+lib/blogex/link_validator.ex
+lib/blogex/seo.ex
+lib/blogex/post.ex

blogex/test/blogex/link_validator_test.exs

@@ -245,6 +245,24 @@ defmodule Blogex.LinkValidatorTest do
   end
 
   describe "validate_body/2" do
+    # The guard is a conjunction, not a disjunction: both arguments must have
+    # the documented types. Mutating `and` to `or` would let a string blog id
+    # (or a non-binary body) through the guard to crash somewhere else, so pin
+    # the contract at the door.
+    test "rejects a non-atom blog id at the function clause guard" do
+      assert_raise FunctionClauseError, fn ->
+        # apply/3 keeps the compiler's type checker quiet: the point is the
+        # runtime guard, and the test deliberately passes an invalid type.
+        apply(LinkValidator, :validate_body, ["Just text, no links.", "engineering"])
+      end
+    end
+
+    test "rejects a non-binary body at the function clause guard" do
+      assert_raise FunctionClauseError, fn ->
+        apply(LinkValidator, :validate_body, [[], :engineering])
+      end
+    end
+
     test "returns :ok when body has no internal blog links" do
       body = "Just text, no links."
 

blogex/test/mutation_config_test.exs

@@ -0,0 +1,82 @@
+defmodule Blogex.MutationConfigTest do
+  # The mutation sensor is opt-in: a dependency and two aliases, deliberately
+  # absent from `make verify`'s VERIFY_SENSORS. These pins keep the wiring
+  # honest (dev/test only, never runtime) and keep the aliases reading the
+  # scope files rather than duplicating a list in mix.exs.
+  use ExUnit.Case
+
+  test "muex is a dev/test-only, non-runtime dependency" do
+    deps = Mix.Project.config()[:deps]
+
+    assert {:muex, requirement, opts} = dep!(deps, :muex)
+    assert requirement =~ "0.12"
+    assert opts[:only] == [:dev, :test]
+    assert opts[:runtime] == false
+  end
+
+  test "mutate and mutate.fast aliases are defined" do
+    aliases = Mix.Project.config()[:aliases]
+
+    assert is_function(aliases[:mutate], 1)
+    assert is_function(aliases[:"mutate.fast"], 1)
+  end
+
+  test "mutation args always include the scope, the curated mutators and --fail-at 0" do
+    args = Blogex.MixProject.mutation_args("mutation_scope_fast.txt", [])
+    scope = Blogex.MixProject.read_mutation_scope("mutation_scope_fast.txt")
+
+    assert "--fail-at" in args
+    assert Enum.at(args, Enum.find_index(args, &(&1 == "--fail-at")) + 1) == "0"
+    assert files_arg(args) == Enum.join(scope, ",")
+
+    mutators = Enum.at(args, Enum.find_index(args, &(&1 == "--mutators")) + 1)
+
+    assert mutators ==
+             "arithmetic,boolean,comparison,conditional,function_call,literal,return_value"
+  end
+
+  test "extra CLI args pass through so a single file can be targeted" do
+    args =
+      Blogex.MixProject.mutation_args("mutation_scope_fast.txt", ["--files", "lib/blogex/seo.ex"])
+
+    assert args
+           |> Enum.chunk_every(2, 1, :discard)
+           |> Enum.any?(fn
+             ["--files", "lib/blogex/seo.ex"] -> true
+             _ -> false
+           end)
+  end
+
+  test "an empty scope fails loudly instead of running over nothing" do
+    path =
+      Path.join(System.tmp_dir!(), "muex-empty-scope-#{System.unique_integer([:positive])}.txt")
+
+    File.write!(path, "\n   \n")
+    on_exit(fn -> File.rm(path) end)
+
+    assert_raise Mix.Error, ~r/empty/, fn ->
+      Blogex.MixProject.read_mutation_scope(path)
+    end
+  end
+
+  defp dep!(deps, name) do
+    case Enum.find(deps, fn
+           {^name, _requirement} -> true
+           {^name, _requirement, _opts} -> true
+           _ -> false
+         end) do
+      {^name, requirement} -> {name, requirement, []}
+      {^name, requirement, opts} -> {name, requirement, opts}
+      nil -> flunk("expected #{inspect(name)} in deps/0")
+    end
+  end
+
+  defp files_arg(args) do
+    args
+    |> Enum.chunk_every(2, 1, :discard)
+    |> Enum.find_value(fn
+      ["--files", files] -> files
+      _ -> nil
+    end)
+  end
+end

blogex/test/mutation_scope_test.exs

@@ -0,0 +1,64 @@
+defmodule Blogex.MutationScopeTest do
+  # The mutation scope files are the single source of truth for what `mix
+  # mutate` / `mix mutate.fast` are allowed to touch. These pins keep the
+  # sensor honest: a scope that quietly shrinks to an empty (or stale) set
+  # would report a score for a check it never ran.
+  use ExUnit.Case
+
+  @root Path.expand("..", __DIR__)
+  @full "mutation_scope.txt"
+  @fast "mutation_scope_fast.txt"
+
+  # Pure modules only. These are excluded deliberately:
+  #   blogex.ex      facade + moduledoc, nothing to mutate
+  #   blogex/blog.ex the `__using__` macro (NimblePublisher expansion)
+  #   router.ex      Phoenix router DSL
+  #   components.ex  Phoenix component DSL
+  #   layout.ex      Phoenix/LiveView layout DSL
+  @shell_modules [
+    "lib/blogex.ex",
+    "lib/blogex/blog.ex",
+    "lib/blogex/router.ex",
+    "lib/blogex/components.ex",
+    "lib/blogex/layout.ex"
+  ]
+
+  test "full scope lists only existing files" do
+    missing = Enum.reject(scope(@full), &File.exists?(Path.join(@root, &1)))
+
+    assert missing == [],
+           "mutation scope names files that do not exist: #{inspect(missing)}"
+  end
+
+  test "full scope is not empty" do
+    assert scope(@full) != []
+  end
+
+  test "fast scope is not empty" do
+    assert scope(@fast) != []
+  end
+
+  test "fast scope is a subset of full scope" do
+    out_of_scope = MapSet.difference(MapSet.new(scope(@fast)), MapSet.new(scope(@full)))
+
+    assert MapSet.size(out_of_scope) == 0,
+           "mutate.fast names files outside the audited scope: #{inspect(MapSet.to_list(out_of_scope))}"
+  end
+
+  test "scope excludes the macro and shell modules" do
+    scoped = MapSet.new(scope(@full) ++ scope(@fast))
+    present = Enum.filter(@shell_modules, &MapSet.member?(scoped, &1))
+
+    assert present == [],
+           "shell/macro modules belong outside the mutation scope: #{inspect(present)}"
+  end
+
+  defp scope(file) do
+    @root
+    |> Path.join(file)
+    |> File.read!()
+    |> String.split("\n", trim: true)
+    |> Enum.map(&String.trim/1)
+    |> Enum.reject(&(&1 == ""))
+  end
+end