Simple feature flags in Elixir

Feature flags are simple ways to control how features are rolled out to users and run A/B tests. Instead of using third-party services or libraries we can easily implement a useful feature flag system in our Elixir app.

Calculating feature flags

First we set up the feature flags module which calculates the feature flags for users:

# lib/my_app/feature_flags.ex
defmodule MyApp.FeatureFlags do
  @moduledoc """
  Context for managing feature flags.

  An md5 hash combining user ID and the flag is used to determine the rollout
  of feature flags to users, as md5 is readily available in both Postgres and
  Elixir.
  """
  import Ecto.Query

  alias MyApp.Users.User

  @flag_ratios %{
    "feature_a" => 1.0,
    "feature_b" => 0.5,
    "feature_c" => 0.0
  }

  @doc """
  Returns the flag ratios.
  """
  def flag_ratios, do: @flag_ratios

  @doc """
  Returns the feature flags enabled for a given user based on rollout ratios.

  ## Examples

      iex> MyApp.FeatureFlags.list_feature_flags(
      ...>   %MyApp.Users.User{id: "11111111-1111-1111-1111-111111111113"},
      ...>   %{"feature_a" => 0.3, "feature_b" => 0.6}
      ...> )
      ["feature_b"]
  """
  def list_feature_flags(user, flag_ratios \\ flag_ratios()) do
    flag_ratios
    |> Enum.filter(fn {flag, ratio} ->
      <<score::size(16), _rest::binary>> = :crypto.hash(:md5, "#{user.id}:#{flag}")

      score / 65_536 < ratio
    end)
    |> Enum.map(fn {flag, _ratio} -> flag end)
  end

  @doc """
  Returns an Ecto query filtering users who have the given feature flag enabled.

  Raises `KeyError` if the flag is not present in the flag ratio map.

  ## Examples

      iex> %Ecto.Query{} = MyApp.FeatureFlags.users_with_feature_flag_query(
      ...>   "feature_b",
      ...>   %{"feature_a" => 0.3, "feature_b" => 0.6}
      ...> )
  """
  def users_with_feature_flag_query(flag, flag_ratios \\ flag_ratios()) do
    ratio = Map.fetch!(flag_ratios, flag)

    where(
      User,
      [u],
      fragment(
        """
        (
          (
            ('x' || substr(md5(?::text || ':' || ?), 1, 4))::bit(16)::int
          ) / 65536.0
        ) < ?
        """,
        u.id,
        ^flag,
        ^ratio
      )
    )
  end

  @doc """
  Returns an Ecto query filtering users who don't have the given feature flag enabled.

  Raises `KeyError` if the flag is not present in the flag ratio map.

  ## Examples

      iex> %Ecto.Query{} = MyApp.FeatureFlags.users_without_feature_flag_query(
      ...>   "feature_b",
      ...>   %{"feature_a" => 0.3, "feature_b" => 0.6}
      ...> )
  """
  def users_without_feature_flag_query(flag, flag_ratios \\ flag_ratios()) do
    ratio = Map.fetch!(flag_ratios, flag)

    where(
      User,
      [u],
      fragment(
        """
        (
          ('x' || substr(md5(?::text || ':' || ?), 1, 4))::bit(16)::int
        ) / 65536.0 >= ?
        """,
        u.id,
        ^flag,
        ^ratio
      )
    )
  end
end

# lib/test/my_app/feature_flags_test.exs
defmodule MyApp.FeatureFlagsTest do
  use MyApp.DataCase

  doctest MyApp.FeatureFlags

  alias MyApp.{FeatureFlags, Repo}

  test "list_feature_flags/2" do
    user_1 = user_fixture("11111111-1111-1111-1111-111111111131")
    user_2 = user_fixture("11111111-1111-1111-1111-111111111111")
    user_3 = user_fixture("11111111-1111-1111-1111-111111111114")
    user_4 = user_fixture("11111111-1111-1111-1111-111111111116")
    flag_ratios = %{"a" => 0.0, "b" => 0.0}

    assert FeatureFlags.list_feature_flags(user_1, flag_ratios) == []
    assert FeatureFlags.list_feature_flags(user_2, flag_ratios) == []
    assert FeatureFlags.list_feature_flags(user_3, flag_ratios) == []
    assert FeatureFlags.list_feature_flags(user_4, flag_ratios) == []

    flag_ratios = %{"a" => 0.25, "b" => 0.5}

    assert FeatureFlags.list_feature_flags(user_1, flag_ratios) == ["a", "b"]
    assert FeatureFlags.list_feature_flags(user_2, flag_ratios) == ["b"]
    assert FeatureFlags.list_feature_flags(user_3, flag_ratios) == []
    assert FeatureFlags.list_feature_flags(user_4, flag_ratios) == []

    flag_ratios = %{"a" => 0.5, "b" => 0.5}

    assert FeatureFlags.list_feature_flags(user_1, flag_ratios) == ["a", "b"]
    assert FeatureFlags.list_feature_flags(user_2, flag_ratios) == ["b"]
    assert FeatureFlags.list_feature_flags(user_3, flag_ratios) == ["a"]
    assert FeatureFlags.list_feature_flags(user_4, flag_ratios) == []

    flag_ratios = %{"a" => 1.0, "b" => 1.0}

    assert FeatureFlags.list_feature_flags(user_1, flag_ratios) == ["a", "b"]
    assert FeatureFlags.list_feature_flags(user_2, flag_ratios) == ["a", "b"]
    assert FeatureFlags.list_feature_flags(user_3, flag_ratios) == ["a", "b"]
    assert FeatureFlags.list_feature_flags(user_4, flag_ratios) == ["a", "b"]
  end

  test "users_with_feature_flag_query/2" do
    user_1 = user_fixture("11111111-1111-1111-1111-111111111111")
    user_2 = user_fixture("11111111-1111-1111-1111-111111111117")
    flag_ratios = %{"a" => 0.25, "b" => 0.5}

    # Ensure feature flags are correctly assigned to users
    assert FeatureFlags.list_feature_flags(user_1, flag_ratios) == ["b"]
    assert FeatureFlags.list_feature_flags(user_2, flag_ratios) == ["a"]

    assert [loaded_user] = Repo.all(FeatureFlags.users_with_feature_flag_query("a", flag_ratios))

    assert loaded_user.id == user_2.id

    assert [loaded_user] = Repo.all(FeatureFlags.users_with_feature_flag_query("b", flag_ratios))

    assert loaded_user.id == user_1.id
  end

  test "users_without_feature_flag_query/2" do
    user_1 = user_fixture("11111111-1111-1111-1111-111111111111")
    user_2 = user_fixture("11111111-1111-1111-1111-111111111117")
    flag_ratios = %{"a" => 0.25, "b" => 0.5}

    # Ensure feature flags are correctly assigned to users
    assert FeatureFlags.list_feature_flags(user_1, flag_ratios) == ["b"]
    assert FeatureFlags.list_feature_flags(user_2, flag_ratios) == ["a"]

    assert [loaded_user] = Repo.all(FeatureFlags.users_without_feature_flag_query("a", flag_ratios))

    assert loaded_user.id == user_1.id

    assert [loaded_user] = Repo.all(FeatureFlags.users_without_feature_flag_query("b", flag_ratios))

    assert loaded_user.id == user_2.id
  end

  defp user_fixture(id), do: Repo.insert!(%MyApp.Users.User{id: id})
end

We use the user's UUID because it is stable and unique. Other stable and unique identifiers can be used, but be careful. Using an identifier that can change like email, can move users from cohort to cohort invalidating the metrics. Using an identifier that's not unique, such as inserted_at with limited precision, can cause users to receive the same hash and end up in the same cohort instead of distributing equally. Whatever identifier we use, hashing it gives us a deterministic value that we can use to assign users to feature cohorts.

The first 16 bits of the hash are converted into an integer between 0 and 65,535, we then divide by the total number of possible values, 65,536, to get a value from 0.0 to less than 1.0. We can use that to compare with our rollout ratio, e.g. 0.25 means that 25% of the users will be in the feature cohort.

Gating features

To gate the features all we need to do is:

case "flag_b" in FeatureFlags.list_feature_flags(user) do
  true -> # Do flag A stuff
  false -> # Don't do flag A stuff
end

A practical example could be only sending a notification to the feature cohort:

case "activity_notification" in FeatureFlags.list_feature_flags(user) do
  true -> Notifier.send(user, activity)
  false -> :ok
end

Feature flags in an API

We can return a list of features in an API if we use a third party client like a mobile app:

# lib/my_app_web/controllers/v1/feature_flags_controller.ex
defmodule MyAppWeb.V1.FeatureFlagsController do
  use MyAppWeb, :controller

  alias MyApp.FeatureFlags

  def show(conn, _params) do
    flags = FeatureFlags.list_feature_flags(conn.assigns.current_user)

    render(conn, "show.json", flags: flags)
  end
end

# lib/my_app_web/controllers/v1/feature_flags_json.ex
defmodule MyAppWeb.V1.FeatureFlagsJSON do
  def show(%{flags: flags}) do
    %{
      object: "feature_flags",
      flags: flags
    }
  end
end

# lib/test/my_app_web/controllers/v1/feature_flags_controller_test.exs
defmodule MyAppWeb.V1.FeatureFlagsControllerTest do
  use MyAppAPIWeb.ConnCase

  setup :setup_authenticated_user_conn

  @tag user: %{id: "11111111-1111-1111-1111-111111111111"}
  test "GET /v1/user/feature-flags", %{conn: conn} do
    conn = get(conn, ~p"/v1/user/feature-flags")

    assert json = json_response(conn, 200)

    assert json["object"] == "feature_flags"
    assert json["flags"] == ["feature_a", "feature_b"]
  end
end

A/B testing

To track performance of a feature, we can build queries that show whether it had the impact we looked for. In the following let's say we have tasks users complete and we want to see how a new feature impacts the metrics:

defmodule MyApp.Analytics do
  @moduledoc """
  Context for analytics.
  """
  import Ecto.Query

  alias MyApp.{FeatureFlags, Repo, Tasks.Task}

  @doc """
  Calculates the task performance metrics based on the flag.
  """
  def get_task_performance(flag, since, opts \\ []) do
    flag_ratios = Keyword.get(opts, :flag_ratios, FeatureFlags.flag_ratios())

    %{
      enabled:
        get_task_completion_metrics(
          FeatureFlags.users_with_feature_flag_query(flag, flag_ratios),
          since
        ),
      disabled:
        get_task_completion_metrics(
          FeatureFlags.users_without_feature_flag_query(flag, flag_ratios),
          since
        )
    }
  end

  # This query excludes users who didn't create a task in the experimentation
  # period. This is fine as we only want to measure completion rate for users
  # who create tasks.
  defp get_task_completion_metrics(users_query, since) do
    users_query = select(users_query, [u], u.id)

    Task
    |> where([t], t.inserted_at >= ^since)
    |> where([t], t.user_id in subquery(users_query))
    |> group_by([t], t.user_id)
    |> select([t], %{
      user_id: t.user_id,
      total: count(t.id),
      completed: filter(count(t.id), not is_nil(t.completed_at)),
      completion_rate:
        fragment(
          """
          COUNT(?) FILTER (WHERE ? IS NOT NULL)::float
          / NULLIF(COUNT(?), 0)
          """,
          t.id,
          t.completed_at,
          t.id
        ),
      average_completion_time:
        filter(
          avg(
            fragment(
              "EXTRACT(EPOCH FROM (? - ?))",
              t.completed_at,
              t.inserted_at
            )
          ),
          not is_nil(t.completed_at)
        )
    })
    |> subquery()
    |> from()
    |> select([u], %{
      users: count(u.user_id),
      total_tasks: type(sum(u.total), :integer),
      completed_tasks: type(sum(u.completed), :integer),
      average_user_completion_rate: avg(u.completion_rate),
      average_completion_time: avg(u.average_completion_time)
    })
    |> Repo.one()
  end
end

# lib/test/my_app/analytics_test.exs
defmodule MyApp.AnalyticsTest do
  use MyApp.DataCase

  alias MyApp.{Analytics, FeatureFlags, Repo}

  test "get_task_performance/2" do
    user_a = user_fixture("11111111-1111-1111-1111-111111111112")
    user_b = user_fixture("11111111-1111-1111-1111-111111111113")
    flag_ratios = %{"experiment" => 0.5}
    since = DateTime.utc_now()

    # Ensure feature flags are correctly assigned to users
    assert FeatureFlags.list_feature_flags(user_a, flag_ratios) == ["experiment"]
    assert FeatureFlags.list_feature_flags(user_b, flag_ratios) == []

    # User A completes 1 of 2 tasks (50%) with 1 older task.
    task_fixture(user_a, %{inserted_at: DateTime.add(since, -1, :second), completed_at: DateTime.utc_now()})
    task_fixture(user_a, %{completed_at: DateTime.utc_now()})
    task_fixture(user_a, %{completed_at: nil})

    # User B completes 3 of 3 tasks (100%).
    task_fixture(user_b, %{completed_at: DateTime.utc_now()})
    task_fixture(user_b, %{completed_at: DateTime.utc_now()})
    task_fixture(user_b, %{completed_at: DateTime.utc_now()})

    result =
      Analytics.get_task_performance("experiment", since,
        flag_ratios: flag_ratios
      )

    assert result.enabled.users == 1
    assert result.enabled.total_tasks == 2
    assert result.enabled.completed_tasks == 1
    assert result.enabled.average_user_completion_rate == 0.5
    assert result.disabled.users == 1
    assert result.disabled.total_tasks == 3
    assert result.disabled.completed_tasks == 3
    assert result.disabled.average_user_completion_rate == 1.0
  end

  defp user_fixture(id), do: Repo.insert!(%MyApp.Users.User{id: id})

  defp task_fixture(user, attrs) do
    user
    |> Ecto.build_assoc(:tasks)
    |> Map.merge(attrs)
    |> Repo.insert!()
  end
end

Note that changing the rollout ratio of the feature flag starts a new experiment period, since a change in ratio means that users will switch cohorts. That's why we pass in a datetime with the since argument. This also means that if you want to look at historical experiments you must use the exact same ratios and limit the datetime range.

One thing I want to warn about with A/B tests is that it rarely provides significant insights. Either the product doesn't have enough traffic or the metrics captured don't matter. I have only ever seen it working for marginal improvements. If your product doesn't have traction then A/B testing isn't going to save it. Personally, I believe building what you think is good is better than building what metrics suggests is good.

You need enough users and meaningful events for reliable signal.

A/B/C testing

While I want you to understand that testing with more than two groups can make the signal in the metrics far weaker, it's very simple with our feature flags design:

@flag_ratios %{
  "feature_a" => 0.67,
  "feature_b" => 0.5
}

This gives us three equal groups with the following query:

# lib/my_app/analytics.ex
def feature_metric_group_query(group, flag_ratios \\ FeatureFlags.flag_ratios()) do
  group_a =
    "feature_a"
    |> FeatureFlags.users_with_feature_flag_query(flag_ratios)
    |> select([u], u.id)

  group_b =
    "feature_b"
    |> FeatureFlags.users_with_feature_flag_query(flag_ratios)
    |> select([u], u.id)

  case group do
    # Control has no feature_a
    :control -> where(User, [u], u.id not in subquery(group_a))
    # Variant A has feature_a but not feature_b
    :variant_a -> where(User, [u], u.id in subquery(group_a) and u.id not in subquery(group_b))
    # Variant B has feature_a and feature_b
    :variant_b -> where(User, [u], u.id in subquery(group_a) and u.id in subquery(group_b))
    group -> raise ArgumentError, "expected valid group, got: #{inspect(group)}"
  end
end

# lib/test/my_app/analytics_test.exs
test "feature_metric_group_query/2" do
  user_control = user_fixture("11111111-1111-1111-1111-111111111113")
  user_a = user_fixture("11111111-1111-1111-1111-111111111112")
  user_b = user_fixture("11111111-1111-1111-1111-111111111111")
  flag_ratios = %{"feature_a" => 0.67, "feature_b" => 0.5}

  # Ensure feature flags are correctly assigned to users
  assert FeatureFlags.list_feature_flags(user_control, flag_ratios) == []
  assert FeatureFlags.list_feature_flags(user_a, flag_ratios) == ["feature_a"]
  assert FeatureFlags.list_feature_flags(user_b, flag_ratios) == ["feature_a", "feature_b"]

  assert [loaded_user] = Repo.all(Analytics.feature_metric_group_query(:control, flag_ratios))
  assert loaded_user.id == user_control.id

  assert [loaded_user] = Repo.all(Analytics.feature_metric_group_query(:variant_a, flag_ratios))
  assert loaded_user.id == user_a.id

  assert [loaded_user] = Repo.all(Analytics.feature_metric_group_query(:variant_b, flag_ratios))
  assert loaded_user.id == user_b.id
end

If we want it to be more precise we could use 2.0 / 3.0 instead of 0.67. To gate the features we do this:

flags = FeatureFlags.list_feature_flags(user)

cond do
  "flag_a" not in flags -> # Control, don't do anything different
  "flag_b" not in flags -> # Do variant A stuff
  true -> # Do variant B stuff
end

You can easily expand this to A/B/C/D testing:

@flag_ratios %{
  # Control has no feature_a
  "feature_a" => 0.75,
  # Variant A has feature_a but not feature_b or feature_c
  "feature_b" => 0.67,
  # Variant B has feature_a and feature_b, but not feature_c
  # Variant C has feature_a, feature_b, and feature_c
  "feature_c" => 0.5
}

We progressively divide the remaining users into approximately equal groups. First, 25% become the control group. Of the remaining 75%, approximately 33% become variant A. We then divide the remaining users equally between variants B and C. This gives us approximately 25% of users in each group.

You must be very careful though. Testing more than two groups can make it significantly harder to tell what the signal really is

Word of caution

I did mention this earlier, but I want to really drive home the point. In 1975, Charles Goodhart wrote what later got popularized as this paraphrased quote:

When a measure becomes a target, it ceases to be a good measure.

If you have a relatively small project you shouldn't consider A/B testing at all, and if you have a large project you should use it sparingly. There is no replacement for human ingenuity and determination. Rather than discovering the future, we must build it, and we should attempt to build what we believe is the best version rather than what metrics tell us.

Metrics are useful and important, but we cannot make it our north star. The north star must be our own beliefs. Often A/B tests are misused to optimize engagement metrics that aren't necessarily meaningful. Optimizing for more engagement doesn't mean we are building a better product. Instead I think we should focus more on how to improve the baseline user experience.

For example, we can use A/B testing to experiment if:

• Users complete tasks with fewer errors, reducing user frustration.
• Users successfully find and use a feature, improving discoverability.
• Users can complete tasks in less time or with fewer steps without reducing quality.
• Users can recover from errors by themselves, improving success rate.

All metrics need context. Completing a task faster doesn't necessarily mean it's an improvement if the output is worse. The goal is to find a genuine signal that things are working better, not simply that a number goes up.

Ultimately, I think one of the best ways to evaluate what we build is to dogfood our own product. We can use metrics to understand consequences of decisions, but we shouldn't let them make decisions for us.