Dan Schultzer

ECharts in Phoenix LiveView

2025-02-15T00:00:00Z

To display charts in Phoenix, we can either generate them server-side (e.g., using Contex) or client-side with JavaScript. I would prefer a server-side solution with pure SVG/CSS, but currently, nothing beats JavaScript charting libraries. However, we can still have the backend do most of the work and live update the charts with a minimal JS hook.

In this post, I’ll go over how to implement ECharts in a Phoenix LiveView app with live updates.

Set up ECharts

Add echarts.min.js to your assets/vendor directory, and set up assets/js/hooks.js to render charts on events pushed from the backend:

// assets/js/hooks.js
let Hooks = {};

import * as echarts from "../vendor/echarts.min"

Hooks.Chart = {
  render(chart, option) {
    // The legend selection should not be overridden with subsequent updates
    if (chart.getOption() && option.legend && option.legend.selected) {
      delete option.legend.selected
    }

    chart.setOption(option)
  },

  mounted() {
    let chart = echarts.init(this.el)

    this.handleEvent(`chart-option-${this.el.id}`, (option) =>
      this.render(chart, option)
    )
  }
}

export default Hooks;

Initialize the LiveSocket in assets/js/app.js with the hooks:

// assets/js/app.js

// ...

import "phoenix_html";
// Establish Phoenix Socket and LiveView configuration.
import { Socket } from "phoenix"
import { LiveSocket } from "phoenix_live_view"
import topbar from "../vendor/topbar"
import Hooks from "./hooks" // 1. Import Hooks

let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("content");
let liveSocket = new LiveSocket("/live", Socket, {
  longPollFallbackMs: 2500,
  params: { _csrf_token: csrfToken },
  hooks: Hooks // 2. Add it to the liveSocket
});

// ...

This is all the JavaScript we need. Our backend will push the chart options used to render the graphs.

Implement some dummy data that we want to chart:

# lib/my_app/data.ex
defmodule MyApp.Data do
  def get_line_stack_data(cumulative \\ nil) do
    {
      ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"],
      Enum.map([
        %{
          name: "Email",
          values: [120, 132, 101, 134, 90, 230, 210]
        },
        %{
          name: "Union Ads",
          values: [220, 182, 191, 234, 290, 330, 310]
        },
        %{
          name: "Video Ads",
          values: [150, 232, 201, 154, 190, 330, 410]
        },
        %{
          name: "Direct",
          values: [320, 332, 301, 334, 390, 330, 320],
        },
        %{
          name: "Search Engine",
          values: [820, 932, 901, 934, 1290, 1330, 1320]
        }
      ], fn line_data ->
        values = Enum.map(line_data.values, & Float.round(&1 * :rand.uniform(), 0))
        values = cumulative && Enum.scan(values, 0, & &2 + &1) || values

        %{line_data | values: values}
      end)
    }
  end

  def get_gauge_multi_title_data do
    %{
      good: Float.round(:rand.uniform() * 100, 2),
      better: Float.round(:rand.uniform() * 100, 2),
      perfect: Float.round(:rand.uniform() * 100, 2)
    }
  end

  def get_process_count, do: :erlang.system_info(:process_count)
end

Remember to add a route to DashboardLive in the router:

# lib/my_app_web/router.ex
defmodule MyAppWeb.Router do
  use MyAppWeb, :router

  scope "/", MyAppWeb do
    pipe_through :browser

    live "/", DashboardLive
  end
end

Now onto the DashboardLive LiveView module:

# lib/my_app_web/live/dashboard_live.ex
defmodule MyAppWeb.DashboardLive do
  use MyAppWeb, :live_view

  alias MyApp.Data
  alias Phoenix.LiveView.AsyncResult

  @impl true
  def handle_params(_params, _uri, socket) do
    {:noreply,
      socket
      |> assign(:options, %{line_stack: %{cumulative: true}})
      |> assign(:reload_timers, %{})
      |> assign_async_schedule([:line_stack, :gauge_multi_title], fn opts ->
        %{
          line_stack: Data.get_line_stack_data(opts[:line_stack][:cumulative]),
          gauge_multi_title: Data.get_gauge_multi_title_data()
        }
      end)
      |> assign_async_schedule(:process_gauge, fn _opts ->
        %{process_gauge: Data.get_process_count()}
      end)}
  end

  # ...
end

In the above, we are setting an options assign so we can switch between cumulative and single data points for our stacked line graph. We also assign the reload timers so we can cancel the scheduled poll when the options change. The assign_async_schedule/4 function works the same way as assign_async/3, but also polls new data every 5 seconds.

The reason we use polling instead of PubSub is that chart data is usually an aggregate of a lot of data, and it doesn’t make sense to listen to individual records. We only need to re-render the charts every once in a while.

Continue to write out the assign_async_schedule/4 logic:

# lib/my_app_web/live/dashboard_live.ex
defmodule MyAppWeb.DashboardLive do
  # ...

  defp assign_async_schedule(socket, key_or_keys, fun, update_in \\ :timer.seconds(5)) do
    keys = List.wrap(key_or_keys)

    options =
      keys
      |> Enum.map(& {&1, %{}})
      |> Enum.into(%{})
      |> Map.merge(socket.assigns.options)

    socket =
      socket
      |> start_async_schedule(key_or_keys, fun, update_in)
      |> assign(:options, options)

    Enum.reduce(keys, socket, fn key, socket ->
      assign(socket, key, AsyncResult.loading())
    end)
  end

  defp start_async_schedule(socket, key_or_keys, fun, update_in) do
    options = Map.take(socket.assigns.options, List.wrap(key_or_keys))

    start_async(socket, key_or_keys, fn ->
      {fun.(options), fun, update_in}
    end)
  end
  
  @impl true
  def handle_async(key_or_keys, {:ok, {values, fun, update_in}}, socket) do
    socket =
      key_or_keys
      |> List.wrap()
      |> Enum.reduce(socket, fn key, socket ->
        async = Map.fetch!(socket.assigns, key)
        value = Map.fetch!(values, key)

        socket
        |> push_chart_event(key, value)
        |> assign(key, AsyncResult.ok(async, value))
      end)
      |> assign(:reload_timers, Map.put(socket.assigns.reload_timers, key_or_keys, start_timer(key_or_keys, fun, update_in)))

    {:noreply, socket}
  end

  def handle_async(key_or_keys, {:exit, reason} , socket) do
    socket =
      key_or_keys
      |> List.wrap()
      |> Enum.reduce(socket, fn key, socket ->
        async = Map.fetch!(socket.assigns, key)

        assign(socket, key, AsyncResult.failed(async, reason))
      end)

    {:noreply, socket}
  end

  defp start_timer(key_or_keys, fun, update_in, send_after \\ nil) do
    ref = Process.send_after(self(), {:async, key_or_keys, fun, update_in}, send_after || update_in)

    {ref, {fun, update_in}}
  end

  @impl true
  def handle_info({:async, key_or_keys, fun, update_in}, socket) do
    socket = start_async_schedule(socket, key_or_keys, fun, update_in)

    {:noreply, socket}
  end

  # ...
end

We also need to handle the options by resetting the timer and loading new data immediately:

# lib/my_app_web/live/dashboard_live.ex
defmodule MyAppWeb.DashboardLive do
  # ...

  @impl true
  def handle_event("options", %{"name" => "line_stack", "cumulative" => cumulative}, socket) do
    key_or_keys = [:line_stack, :gauge_multi_title]
    key_options = Map.put(socket.assigns.options[:line_stack], :cumulative, cumulative == "true")
    {timer, {fun, update_in}} = Map.fetch!(socket.assigns.reload_timers, key_or_keys)

    Process.cancel_timer(timer)

    socket =
      socket
      |> assign(:options, Map.put(socket.assigns.options, :line_stack, key_options))
      |> assign(:reload_timers, Map.put(socket.assigns.reload_timers, key_or_keys, start_timer(key_or_keys, fun, update_in, 1)))

    {:noreply, socket}
  end

  # ...
end

All that’s left to do is to push the ECharts options:

# lib/my_app_web/live/dashboard_live.ex
defmodule MyAppWeb.DashboardLive do
  # ...

  defp push_chart_event(socket, :line_stack, {columns, data}) do
    option =
      %{
        tooltip: %{
          trigger: "axis"
        },
        legend: %{
          data: Enum.map(data, & &1.name),
          selected: %{"Email" => false}
        },
        grid: %{
          left: "3%",
          right: "4%",
          bottom: "3%",
          containLabel: true
        },
        toolbox: %{
          feature: %{
            saveAsImage: %{}
          }
        },
        xAxis: %{
          type: "category",
          boundaryGap: false,
          data: columns
        },
        yAxis: %{
          type: "value"
        },
        series: Enum.map(data, &%{
            name: &1.name,
            type: "line",
            stack: "Total",
            data: &1.values
          })
      }

    push_event(socket, "chart-option-line_stack", option)
  end

  defp push_chart_event(socket, :gauge_multi_title, data) do
    option = %{
      series: [
        %{
          type: "gauge",
          anchor: %{
            show: true,
            showAbove: true,
            size: 18,
            itemStyle: %{
              color: "#FAC858"
            }
          },
          pointer: %{
            icon: "path://M2.9,0.7L2.9,0.7c1.4,0,2.6,1.2,2.6,2.6v115c0,1.4-1.2,2.6-2.6,2.6l0,0c-1.4,0-2.6-1.2-2.6-2.6V3.3C0.3,1.9,1.4,0.7,2.9,0.7z",
            width: 8,
            length: "80%",
            offsetCenter: [0, "8%"]
          },
          progress: %{
            show: true,
            overlap: true,
            roundCap: true
          },
          axisLine: %{
            roundCap: true
          },
          data: [
            %{
              value: data.good,
              name: "Good",
              title: %{
                offsetCenter: ["-40%", "80%"],
                fontSize: 10
              },
              detail: %{
                offsetCenter: ["-40%", "95%"],
                fontSize: 10,
                height: 10,
                width: 20
              }
            },
            %{
              value: data.better,
              name: "Better",
              title: %{
                offsetCenter: ["0%", "80%"],
                fontSize: 10
              },
              detail: %{
                offsetCenter: ["0%", "95%"],
                fontSize: 10,
                height: 10,
                width: 20
              }
            },
            %{
              value: data.perfect,
              name: "Perfect",
              title: %{
                offsetCenter: ["40%", "80%"],
                fontSize: 10
              },
              detail: %{
                offsetCenter: ["40%", "95%"],
                fontSize: 10,
                height: 10,
                width: 20
              }
            }
          ],
          title: %{
            fontSize: 14
          },
          detail: %{
            width: 40,
            height: 14,
            fontSize: 14,
            color: "#fff",
            backgroundColor: "inherit",
            borderRadius: 3,
            formatter: "{value}%"
          }
        }
      ]
    }

    push_event(socket, "chart-option-guage_multi_title", option)
  end

  defp push_chart_event(socket, :process_gauge, data) do
    option = %{
      series: [
        %{
          name: "Processes",
          type: "gauge",
          data: [
            %{
              value: data
            }
          ]
        }
      ]
    }

    push_event(socket, "chart-option-process_gauge", option)
  end
end

We’re setting up all the chart options here and pushing them as an event. This allows us to dynamically configure everything from the backend. We can even switch the chart type ad-hoc.

All that’s left to do is to set up the template:

<%!-- # lib/my_app_web/live/dashboard_live.html.heex --%>
<div class="w-full grid grid-row gap-4">
  <div class="w-full grid grid-row gap-4 p-4">
    <.header>
      Line stack
      <:subtitle>
        <.button :if={@options[:line_stack][:cumulative]} phx-click="options" phx-value-name={:line_stack} phx-value-cumulative="false">
          Single
        </.button>
        <.button :if={@options[:line_stack][:cumulative] != true} phx-click="options" phx-value-name={:line_stack} phx-value-cumulative="true">
          Rolling
        </.button>
      </:subtitle>
    </.header>

    <.async_result assign={@line_stack}>
      <:loading>Loading...</:loading>
      <:failed><.error>Failed to load chart</.error></:failed>
    </.async_result>

    <div id="line_stack" phx-hook="Chart" class="w-full h-[20rem]" phx-update="ignore" />
  </div>

  <div class="grid w-full grid-cols-1 gap-4 xl:grid-cols-2">
    <div class="grid grid-row gap-4 p-4">
      <.header>
        Gauge multi title
      </.header>

      <.async_result assign={@gauge_multi_title}>
        <:loading>Loading..</:loading>
        <:failed><.error>Failed to load chart</.error></:failed>
      </.async_result>

      <div id="guage_multi_title" phx-hook="Chart" class="w-full h-[20rem]" phx-update="ignore" />
    </div>

    <div class="grid grid-row gap-4 p-4">
      <.header>
        Processes
      </.header>

      <.async_result assign={@process_gauge}>
        <:loading>Loading...</:loading>
        <:failed><.error>Failed to load chart</.error></:failed>
      </.async_result>

      <div id="process_gauge" phx-hook="Chart" class="w-full h-[20rem]" phx-update="ignore" />
    </div>
  </div>
</div>

Now we have live updating charts managed entirely by the LiveView module!

Next you’ll want to produce good chart data.

Tests for MyAppWeb.DashboardLive.

Normalize chart data using Ecto and PostgreSQL

2025-02-15T00:00:00Z

If you don’t yet have charts in your Phoenix app, you may want to set up ECharts with Phoenix LiveView.

Producing good datasets for charts is essential. A graph is only as good as the data it uses. Here is a collection of helpful snippets I have used to chart data.

Finding the right step size

Numerical

For numerical steps, you can do a simple calculation with (end - start) / steps to find the step size, but unless you have fixed start and end values, you will end up with odd numbers (e.g. 167.3998).

Instead, we want to snap the steps to 1, 2.5, 5 (scaled to the proper size) so it reads well for the end-user. We’re using Decimal to calculate the step size below:

defmodule MyApp.ChartUtils do
  @moduledoc """
  Utility functions for charts.
  """

  @doc """
  Finds an appropriate numerical step size.
  """
  def step_size(min, max, steps \\ 100) do
    step_size =
      max
      |> Decimal.sub(min)
      |> Decimal.div(steps)
      # We don't want a step size lower than `1` adjusted to the lowest value exponent
      |> Decimal.max(Decimal.new(1, 1, min.exp))

    # Calculating the exponent we need for our step size
    pow10 = Integer.pow(10, length(Integer.digits(step_size.coef)) - 1)
    exp = - 1 - Decimal.scale(step_size)

    # Find the right fit in our list of step sizes to snap to
    [
      Decimal.new(1, 10 * pow10, exp),
      Decimal.new(1, 25 * pow10, exp),
      Decimal.new(1, 50 * pow10, exp),
      Decimal.new(1, 100 * pow10, exp)
    ]
    |> Enum.find(&Decimal.lte?(step_size, &1))
    |> Decimal.normalize()
  end
end

Tests for step_size/3.

Deleted record audit log with Ecto and PostgreSQL

2025-02-10T00:00:00Z

José Valim wrote about how to implement soft delete using a PostgreSQL trigger. The best reason for soft delete is when you need the ability to undelete records, e.g., you give a grace period of 30 days before the record is deleted permanently.

If the deletion of records is meant to be irreversible, this would be an anti-pattern. Soft delete for audit logging is problematic because you must be defensive in how you access data, and you must ensure you always use the right view or table partition.

For an audit log, we want to allow the record to be deleted from the table and add a log entry to another table. This could be done entirely in your application code, but this will not catch any deletions by other applications or clients.

As José writes in his blog post, the simple solution is to store a JSON blob of the data.

Deleted record audit log

For our purpose, we want to keep a paper trail of who deleted a record, when it was deleted, and why it was deleted.

We’ll create a migration that creates a deleted record log table and adds a trigger on tables to insert a log entry on DELETE operations.

defmodule MyApp.Repo.Migrations.CreateDeletedRecordLogsTable do
  use Ecto.Migration

  @trigger_on_tables ~w(table_a table_b table_c)

  def up do
    create table(:deleted_record_logs, primary_key: false) do
      add :record_table, :string, null: false, primary_key: true
      add :record_id, :string, null: false, primary_key: true
      add :record_data, :jsonb, null: false
      add :transaction_id, :bigint, null: false
      add :deleted_at, :utc_datetime_usec, null: false
      add :deleted_by, :string, null: false
      add :delete_reason, :text, null: false
    end

    execute """
      CREATE OR REPLACE FUNCTION log_deleted_record()
      RETURNS TRIGGER AS $$
      BEGIN
        INSERT INTO deleted_record_logs (
          record_table,
          record_id,
          record_data,
          transaction_id,
          deleted_at,
          deleted_by,
          delete_reason
        )
        VALUES (
          TG_TABLE_NAME,
          OLD.id,
          row_to_json(OLD)::jsonb,
          txid_current(),
          NOW(),
          current_setting('app.deleted_by', true),
          current_setting('app.delete_reason', true)
        );
        RETURN OLD;
      END;
      $$ LANGUAGE plpgsql;
      """

    for table <- @trigger_on_tables do
      execute """
        CREATE TRIGGER after_delete_trigger_#{table}
        AFTER DELETE ON #{table}
        FOR EACH ROW
        EXECUTE FUNCTION log_deleted_record();
        """
    end
  end

  def down do
    drop table(:deleted_record_logs)

    for table <- @trigger_on_tables do
      execute "DROP TRIGGER after_delete_trigger_#{table} ON #{table};"
    end

    execute "DROP FUNCTION log_deleted_record();"
  end
end

A log_deleted_record function will trigger on deletes of records for the referenced tables and insert into the logs table:

Record table name
Record ID
Record data as a JSON blob
Transaction ID
When it was deleted
Who deleted it
Why it was deleted

The transaction ID is used to group records that were deleted within the same transaction.

The app.deleted_by and app.delete_reason run-time parameters should be set in the transaction before deleting the record. We require both deleted_by and delete_reason to not be NULL. If these are not specified in the run-time parameters, the deletion will halt with an insert error.

This is how you would delete a record:

Repo.transaction(fn ->
  Repo.query!("SET LOCAL app.deleted_by = 'my name")
  Repo.query!("SET LOCAL app.delete_reason = 'invalid record'")
  Repo.delete!(record)
end)

Using SET LOCAL ensures that these run-time parameters are only set for the current transaction.

Delete record context helper

To make it easier to work with, we want to set up a context function to prepare the run-time parameters for any deletes:

defmodule MyApp.Logs do
  @moduledoc false

  alias MyApp.Repo

  def with_delete_record_context!(deleted_by, delete_reason, callback_fn) do
    Repo.transaction(fn ->
      Repo.query!("SET LOCAL app.deleted_by = '#{escape_string(deleted_by)}'")
      Repo.query!("SET LOCAL app.delete_reason = '#{escape_string(delete_reason)}'")

      res = callback_fn.()

      Repo.query!("RESET app.deleted_by")
      Repo.query!("RESET app.delete_reason")

      res
    end)
  end

  defp escape_string(string) do
    :binary.replace(string, "'", "''", [:global])
  end
end

Now we can wrap our deletes:

MyApp.Logs.with_delete_record_context!("myname", "invalid record", fn ->
  Repo.delete!(record_1)
  Repo.delete!(record_2)
end)

Final words

With these triggers in place, we have strong guarantees for our audit log. Even if someone manually deletes one of the records, it will still be logged with deleted_by and delete_reason. I’ve rolled this out to a large production app where deletions are very consequential, and it has worked flawlessly.

Tests for MyApp.Logs.

Distributed Erlang without DNS on AWS ECS

2024-11-24T00:00:00Z

Distributed Erlang is required when running multiple nodes with Phoenix LiveView and using LongPoll fallback. The Phoenix code generator adds DNSCluster by default, which is great when you can set up DNS records. Unfortunately, in our production setup, we have a separation between the network account and the service account, which makes it impossible to set up Service Discovery (see this issue for more).

What if we simply list the private IPs of the ECS tasks using the AWS API? We could use ClusterEcs, but I prefer the simplicity of DNSCluster, and one issue with ClusterEcs is that it uses ExAWS instead of AWS, which we exclusively use.

Prerequisites

You can skip this section if you have already set up your project to be ready for distributed Erlang.

Chasing a Phoenix LiveView long poll reload issue

2024-11-03T00:00:00Z

TLDR; The long poll fallback in Phoenix LiveView requires distribution. All Phoenix nodes must be connected to each other.

The issue

In a Phoenix 1.7 LiveView production app, we received reports of users experiencing a reload loop. The page loads, and after 7 seconds, a flash error message with Something went wrong! Hang in there while we get back on track appears. The page is force reloaded, and this repeats indefinitely.

The Phoenix app is running in AWS ECS, with Cloudflare in front.

Investigation

First, I had to reproduce the issue. I found out that this was related to long poll, and it was easy to reproduce by setting the session storage keys:

sessionStorage.setItem("phx:fallback:ge", true);
sessionStorage.setItem("phx:fallback:LongPoll", true);

Long poll worked just fine in our staging environment, and I couldn’t reproduce it locally either. I went through the network requests using the developer tools in Safari, Firefox, and Chrome. In all browsers, the request was just canceled. No errors, no reason, nothing.

Production and staging are essentially identical. At first, we did wonder if there were some infrastructure differences that could cause odd behavior, but nothing looked out of the ordinary.

It was always the same three requests, with the third being canceled. The difference between production and staging was that in staging the second request returned a JSON with {"status": 200} while in production it was {"status": 410}.

Enabling debugging with liveSocket.enableDebug() showed it was a timeout error:

[Log] phx-GAIBpoiUGJCzqWuC error: unable to join -  – {reason: "timeout"} (app-500773fd7d20466d8a29a0f19ba374a4.js, line 2)

Cloudflare confirmed that it is the browser closing the connection as it logged the request with 499 Client Closed Request HTTP status.

The issue is in Phoenix.

Phoenix.Transports.LongPoll

I looked into what could return a 410 status code. It happens when a new session is set up:

defp new_session(conn, endpoint, handler, opts) do
  # ...

  case DynamicSupervisor.start_child(Phoenix.Transports.LongPoll.Supervisor, spec) do
    :ignore ->
      conn |> put_status(:forbidden) |> status_json()

    {:ok, server_pid} ->
      data = {:v1, endpoint.config(:endpoint_id), server_pid, priv_topic}
      token = sign_token(endpoint, data, opts)
      conn |> put_status(:gone) |> status_token_messages_json(token, [])
  end
end

If a previous session can’t be resumed, a new session is set up:

defp dispatch(%{method: "GET"} = conn, endpoint, handler, opts) do
  case resume_session(conn, conn.params, endpoint, opts) do
    {:ok, new_conn, server_ref} ->
      listen(new_conn, server_ref, endpoint, opts)

    :error ->
      new_session(conn, endpoint, handler, opts)
  end
end

To resume an existing session, Phoenix.Transports.LongPoll will try to get the server reference from the PID stored in the token and then broadcast a subscribe message on that server reference:

defp resume_session(%Plug.Conn{} = conn, %{"token" => token}, endpoint, opts) do
  case verify_token(endpoint, token, opts) do
    {:ok, {:v1, id, pid, priv_topic}} ->
      server_ref = server_ref(endpoint.config(:endpoint_id), id, pid, priv_topic)

      new_conn =
        Plug.Conn.register_before_send(conn, fn conn ->
          unsubscribe(endpoint, server_ref)
          conn
        end)

      ref = make_ref()
      :ok = subscribe(endpoint, server_ref)
      broadcast_from!(endpoint, server_ref, {:subscribe, client_ref(server_ref), ref})

      receive do
        {:subscribe, ^ref} -> {:ok, new_conn, server_ref}
      after
        opts[:pubsub_timeout_ms] -> :error
      end

    _ ->
      :error
  end
end

That’s it!

The difference between WebSocket and LongPoll transports is that LongPoll requires all nodes to be connected, since to resume a session it needs to broadcast from the PID that the session was initiated on.

The difference between staging and production is that staging had just one server running, while production had two. The AWS load balancer was consistently flipping between the two servers for each long poll request.

So the solution is to enable distribution for LongPoll fallback to work.

Conclusion

While I’m not sure of the exact reasons why LongPoll requires distribution, I guess it is because WebSocket’s persistent connection makes it very simple to keep process state, while LongPoll has to tie multiple isolated REST requests together to achieve the same process state. In any case, you must enable distribution for your Phoenix app to function correctly with PubSub messages across all nodes.

I hope this post can save some time for others in a similar situation.

Development helpers in Elixir

2024-03-24T00:00:00Z

How do you efficiently set up development-only logic in Elixir? In one directory: _dev.

This directory will contain all your development-only logic. Combined with config/dev.exs configuration, this makes a clear and safe separation from your production logic. There will be no need for awkward Mix.env() == :dev blocks around inline code (apart from two Phoenix files). The following example will be based on a Phoenix project, but you can use this pattern for any Elixir project.

Let us get started!

`mix.exs` and `.formatter.exs`

First, we want to include _dev files in our dev and test environments by updating elixirc_paths in mix.exs:

defmodule MyApp.MixProject do
  use Mix.Project

  # ...

  # Specifies which paths to compile per environment.
  # We include `_dev` directory in the test/dev env to
  # enable the dev tools
  defp elixirc_paths(:test), do: ["lib", "test/support", "_dev", "test/_dev/support"]
  defp elixirc_paths(:dev), do: ["lib", "_dev"]
  defp elixirc_paths(_), do: ["lib"]

  # ...
end

We also want mix format to pick up on it, so update the .formatter.exs:

[
  import_deps: [:phoenix],
  plugins: [Phoenix.LiveView.HTMLFormatter],
  inputs: ["*.{heex,ex,exs}", "{_dev,config,lib,test}/**/*.{heex,ex,exs}"]
]

That is it. Now you can put all your dev-only files in the _dev directory and it will only ever get included in the dev and test environments. We can test our dev-only logic and add dev-only test support files to test/_dev/support.

Example: Seed tool

Let us set up a development helper to manually seed a variety of scenarios in our environment.

First, we may want to add some helpers in _dev/my_app, for example _dev/my_app/seed_helper.exs:

defmodule MyApp.Dev.SeedHelper do
  # ...
end

I’m namespacing the module names with Dev. Though it’s just a dev-only tool we still need to test it, so we’ll create the test file in test/_dev/my_app/seed_helper_test.exs:

defmodule MyApp.Dev.SeedHelperTest do
  use MyApp.DataCase

  # ...
end

For Phoenix, we’ll have to update our router to enable our dev-only routes. This along with navigation in the UI will be the only parts that can’t be completely separated from anything that touches production code.

defmodule MyAppWeb.Router do
  use MyAppWeb, :router

  # ...

  # Enable dev tools for dev and test env
  if Application.compile_env(:my_app_web, :dev_tools) do
    scope "/dev" do
      pipe_through :browser

      scope "/tools", MyAppWeb.Dev.Tools do
        scope "/seeds" do
          live "/", SeedLive, :index
          live "/:id/run", SeedLive, :run
        end
      end
    end
  end
end

In this case, I set a config to enable the dev tools in config/config.exs as:

config :my_app_web, :dev_tools, Mix.env() in [:dev, :test]

You could also just use Mix.env() straight in the router, but I found the config pattern more comfortable.

We should add a link in our template as well:

<%= if Application.get_env(:my_app_web, :dev_tools) do %>
  <.link navigate={"/dev/tools/seed"}>Seed</.link>
<% end %>

Now we’ll set up our dev tools, it could be in _dev/my_app_web/tools/live/seed_live.ex:

defmodule MyAppWeb.Dev.Tools.SeedLive do
  use MyAppWeb, :live_view

  @impl true
  def mount(_params, _session, socket) do
    {:ok, stream(socket, :seeds, list_seed_files())}
  end

  # ...
end

A test for the above should be in test/_dev/my_app_web/tools/seed_live_test.exs.

defmodule MyAppWeb.Dev.Tools.SeedLiveTest do
  use MyAppWeb.ConnCase

  test "lists and run seeds", %{conn: conn} do
    # ...
  end
end

This has worked very well for my team, and I highly recommend this pattern.

JavaScript source maps with Phoenix 1.7

2024-03-13T00:00:00Z

If you are setting up error tracking for the JavaScript assets in your Phoenix app, you’ll need a source map to enable readable stack traces. In the following, I’m setting this up with Sentry for error tracking and GitHub Actions for CI/CD, but you can easily adapt this to any vendor.

Generating the source maps

With Phoenix 1.7 you generate the source maps by setting the --sourcemap flag for esbuild in config/config.exs:

config :esbuild,
  version: "0.17.11",
  my_app_web: [
    args:
      ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/* --sourcemap=external),
    cd: Path.expand("../assets", __DIR__),
    env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}
  ]

In the above, I use --sourcemap=external flag to generate the source map but importantly not include the reference in our .js file. If you are fine with exposing the source map to the public you can just use --sourcemap and be done with it; the browser can show the full stack trace and your error tracker might be able to automatically pick it up.

In my case, I want to keep the source map to myself, so I have to move it out of the assets directory in my build stage so it will not be included in my release build.

Add the following commands in the Dockerfile right after the assets are compiled with mix assets.deploy:

# ...

# Compile assets
RUN mix assets.deploy

# Move source maps out of assets
RUN mkdir ./sourcemaps
RUN mv apps/my_app_web/priv/static/assets/*.js.map ./sourcemaps
# ...

With these commands I’m moving the source map files out of priv/static/assets to a place that will not be included in my release.

Upload the source maps

I’m using Github Actions for my CI/CD and will be using the Sentry Release Github Action task to upload the source map. This means that I first need to extract the source maps from the build.

Here we add a new stage named sourcemaps to our Dockerfile just before our final release stage:

# ...
# sourcemaps is a target used as remote context to push source maps to sentry
FROM scratch AS sourcemaps
COPY --from=builder /app/sourcemaps/ /

# Start a new build stage so that the final image will only contain
# the compiled release and other runtime necessities
FROM ${RUNNER_IMAGE}
# ...

The only thing we do in this stage is to copy the generated source maps so we can pull them out of the build to the host. We can pull them out of the build by running docker build . --output ./sourcemaps --target sourcemaps.

Now we can set up Github Actions to upload the source maps when we build the release:

# ...
- name: Build and push image
  id: docker-build
  uses: docker/build-push-action@v4
  with:
    cache-from: type=gha
    cache-to: type=gha,mode=max
    # push: true
    # tags: ...
    # ...

- name: Output source maps
  uses: docker/build-push-action@v4
  with:
    target: sourcemaps
    cache-from: type=gha
    cache-to: type=gha,mode=max
    outputs: type=local,dest=./sourcemaps

- name: Create Sentry release
  uses: getsentry/action-release@v1
  env:
    SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
    SENTRY_ORG: ${{ vars.SENTRY_ORG }}
    SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
  with:
    version: ${{ github.sha }}
    sourcemaps: ./sourcemaps

We are using the Build and Push docker images GHA task to first build and push our release image. After that, we output the source maps to the ./sourcemaps directory on our GHA host. Now the Sentry Release Github Actions task can pick them up and upload them.

Note that we are caching the layers so we won’t rebuild the image when running the ouput source maps task after the build and push.

Content Security Policy header with Phoenix LiveView

2024-03-07T00:00:00Z

If you use sobelow you will get a Missing Content-Security-Policy error with a default Phoenix setup. Managing Content Security Policy (CSP) can be tedious, so you may just want to opt for a catch-all 'unsafe-inline' approach to bypass it all. We should do better than that.

In this blog post, I’ll build a module with helpers to integrate CSP handling into Phoenix and Phoenix LiveView. I think the code is too minimal to be a dependency, but some of it could maybe be added upstream in the put_secure_browser_headers plug in Phoenix.

Writing our first CSP header

CSP headers consist of directives with sources. The recommendation for a default policy is default-src 'self';. This mitigates most typical cross-site scripting (XSS) attacks. 'self' means that only resources of the same scheme, host, and port, as the page itself can be executed.

With Phoenix we have a bunch of data: blobs from the hero icons. These can’t be loaded with the above policy so we should add img-src: 'self' data:; to our policy. To add a second source, like a CDN, we just add it to the sources list: default-src 'self' https://cdn.example.com;.

This is straightforward, but now comes the tricky part. What if we have inline styles or scripts?

If we can’t move it out of the DOM we should use CSP hashing. The tag will only be executed if the hashed value of the content matches the CSP source hash. As an example, we can set up the sources list with script-src 'sha256-RFWPLDbv2BY+rCkDzsE+0fr8ylGr2R2faWMhq4lfEQc='; for the following inline script to execute:

What if we have styles or scripts with dynamic content? If we are only dealing with REST this would be trivial as we can just produce the hash on each request, but we got our LiveView socket pushing diffs while the CSP header has already been sent. If possible we could calculate all the possible hash values and add them to the CSP source list, but in my case, there were way too many variants.

We shouldn’t opt for the obviously unsafe 'unsafe-inline'. Instead, we will use a nonce.

On the initial request, we will generate the nonce and then ensure we use the same nonce throughout the LiveView session.

An inline style example

To use inline styles with nonces we must use """ require Logger import Plug.Conn @doc """ Sets a content security policy header. By default the policy is `default-src 'self'`. `'nonce'` source will be expanded with an auto-generated nonce that is persisted in the process dictionary. The options can be a function or a keyword list. Sources can be a binary or list of binaries. Duplicate directives will be merged together. ## Example plug :put_content_security_policy, img_src: "'self' data:`, style_src: "'self' 'nonce'" plug :put_content_security_policy, img_src: [ "'self'", "data:" ] plug :put_content_security_policy, &MyAppWeb.CSPPolicy.opts/1 """ def put_content_security_policy(conn, fun) when is_function(fun, 1) do put_content_security_policy(conn, fun.(conn)) end def put_content_security_policy(conn, opts) when is_list(opts) do csp = opts |> Keyword.has_key?(:default_src) |> case do false -> [default_src: "'self'"] ++ opts true -> opts end |> Enum.reduce([], fn{name, sources}, acc -> sources = List.wrap(sources) Keyword.update(acc, name, sources, & &1 ++ sources) end) |> Enum.reduce("", fn {name, sources}, acc -> name = String.replace(to_string(name), "_", "-") sources = sources |> Enum.uniq() |> Enum.join(" ") |> String.replace("'nonce'", "'nonce-#{get_csp_nonce()}'") "#{acc}#{name} #{sources};" end) put_resp_header(conn, "content-security-policy", csp) end @doc """ Gets the CSP nonce. Generates a nonce and stores it in the process dictionary if one does not exist. """ def get_csp_nonce do if nonce = Process.get(:plug_csp_nonce) do nonce else nonce = csp_nonce() Process.put(:plug_csp_nonce, nonce) nonce end end defp csp_nonce do 24 |> :crypto.strong_rand_bytes() |> Base.encode64(padding: false) end @doc """ Loads the CSP nonce into the LiveView process. """ def on_mount(:default, _params, _session, %{private: %{connect_params: %{"_csp_nonce" => nonce}}} = socket) do Process.put(:plug_csp_nonce, nonce) {:cont, socket} end def on_mount(:default, _params, _session, socket) do unless Process.get(:plug_csp_nonce) do Logger.debug(""" LiveView session was misconfigured. 1) Ensure the `put_content_security_policy` plug is in your router pipeline: plug :put_content_security_policy 2) Define the CSRF meta tag inside the `` tag in your layout: 3) Pass it forward in your app.js: let csrfToken = document.querySelector("meta[name='csp-nonce']").getAttribute("content"); let liveSocket = new LiveSocket("/live", Socket, {params: {_csp_nonce: cspNonce}}); """) end {:cont, socket} end end

The module documents how to install the CSP helpers in your Phoenix app. This is how I’m using the plug helper in my router pipeline:

pipeline :browser do
  plug :accepts, ["html"]
  plug :fetch_session
  plug :fetch_live_flash
  plug :put_root_layout, html: {MyAppWeb.Layouts, :root}
  plug :protect_from_forgery
  plug :put_secure_browser_headers

  plug :put_content_security_policy,
    img_src: "'self' data:",
    style_src: "'self' 'nonce'"

  plug :load_current_user
end

Unfortunately, sobelow can’t skip plugs with inline skip comment so we must run mix sobelow --mark-skip-all to generate a .sobelow-skips file to skip the error in subsequent runs.

If you need to dynamically set CSP headers you can pass in a function:

plug :put_content_security_policy, &MyAppWeb.CSPPolicy.policy/1

defmodule MyAppWeb.CSPPolicy do
  @moduledoc false

  def policy(_conn) do
    if dsn = Application.get_env(:sentry, :dsn) do
      dsn = URI.parse(dsn)

      [
        script_src: [
          "https://js.sentry-cdn.com/",
          "https://browser.sentry-cdn.com/",
          "blob:"
        ],
        connect_src: [
          URI.to_string(%{dsn | userinfo: nil, path: "/api#{dsn.path}/"})
        ]
      ]
    else
      []
    end
  end
end

Considerations

There is a potential security risk using nonce here; an attacker may find a way to inject payload through the LiveView socket with the DOM manipulation. That could be in unescaped user content pushed to a