Distributed Erlang without DNS on AWS ECS

You must allow for internode communication in the cluster. Add the following security group to your ECS tasks:

resource "aws_security_group" "erlang_distribution" {
  name        = "${data.aws_ecs_cluster.this.cluster_name}-erlang-distribution"
  description = "Communication between Elixir nodes in the cluster"
  vpc_id      = data.aws_vpc.this.id

  ingress {
    description = "Allow distributed Erlang cluster communication"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    self        = true
  }
}

Set up an entrypoint script to pull the IP and set the release node name. I do this in the GitHub workflow that packages the Docker file:

# ...

jobs:
  package:
    # ...

    # It is necessary to set up an entrypoint script that sets the release node name
    # from the ECS container metadata, to enable distributed Erlang.
    - name: Set up AWS ECS bootstrap entrypoint script for distributed Erlang
      run: |
        cat >> .release/rel/overlays/bin/entrypoint <<'SH'
        #!/bin/sh
        set -eu
        export RELEASE_DISTRIBUTION=name
        export RELEASE_NODE=my_app-${{ github.sha }}@`curl -s $ECS_CONTAINER_METADATA_URI_V4 | jq -r ".Networks[0].IPv4Addresses[0]"`
        /app/bin/$1 ${2:-}
        SH

        chmod +x .release/rel/overlays/bin/entrypoint

Ensure that curl and jq is installed in the Docker image:

# Start a new build stage so that the final image will only contain
# the compiled release and other runtime necessities
FROM ${RUNNER_IMAGE}
RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales \
  # For distributed Erlang it is necessary to set up an entrypoint that will pull
  # the ECS instance IP. We need curl and jq to fetch the ECS instance info.
  && apt-get install -y curl jq \
  && apt-get clean && rm -f /var/lib/apt/lists/*_*

The following test uses TestServer to mock the AWS API.

Before you can run these tests you’ll need to update the aws_client function in DNSCluster.ECSResolver. I recommend setting up a helper module that can be used to generate the AWS client:

defmodule AWS.MyApp do
  @moduledoc """
  Module that contains MyApp specific helpers.
  """

  @doc """
  Creates AWS client struct.
  """
  def aws_client(region) do
    aws_credentials = aws_credentials()
    access_key = Map.fetch!(aws_credentials, :access_key_id)
    secret_key = Map.fetch!(aws_credentials, :secret_access_key)
    session_token = Map.get(aws_credentials, :token)

    access_key
    |> AWS.Client.create(secret_key, session_token, region)
    |> put_endpoint()
    |> Map.merge(%{proto: aws_proto()})
  end

  # In tests we don't want to require :aws_credentials to be running
  if Mix.env() == :test do
    defp aws_credentials do
      Application.get_env(:aws, :credentials, %{access_key_id: "test", secret_access_key: "test", token: "test"})
    end
  else
    defp aws_credentials, do: :aws_credentials.get_credentials()
  end

  # In tests we want to replace the whole endpoint while in prod/dev we just want to replace the root
  if Mix.env() == :test do
    defp put_endpoint(client), do: AWS.Client.put_endpoint(client, aws_endpoint())
  else
    defp put_endpoint(client), do: AWS.Client.put_endpoint(client, {:keep_prefixes, aws_endpoint()})
  end

  defp aws_endpoint, do: Application.get_env(:aws, :endpoint, AWS.Client.default_endpoint())

  defp aws_proto, do: Application.get_env(:aws, :proto, "https")
end

We don’t want to start :aws_credentials in tests:

# mix.exs
defmodule MyApp.MixProject do
  use Mix.Project

  # ...

  def deps do
    [
      # ...
      {:aws, "~> 1.0.2"},
      # We don't want `aws_credentials` to start in dev and test since
      # the startup will fail if credentials are not available.
      {:aws_credentials, "~> 0.3.1", runtime: Mix.env() == :prod}
    ]
  end

  #...
end

We can now test the ECSResolver (and any other functions you got that is using the aws_client helper function):

defmodule DNSCluster.ECSResolverTest do
  use MyApp.DataCase

  alias DNSCluster.ECSResolver
  alias ExUnit.CaptureLog

  @params """
    {
      "cluster": "default",
      "region": "us-east-1",
      "params": {
        "TagFilters": [
          {
            "Key": "Environment",
            "Values": ["test"]
          },
          {
            "Key": "Project",
            "Values": ["my-project"]
          }
        ]
      }
    }
    """

  @task_arns ~w(arn:aws:ecs:us-east-1:012345678910:task/1 arn:aws:ecs:us-east-1:012345678910:task/2 arn:aws:ecs:us-east-1:012345678910:task/3)

  describe "lookup/2" do
    test "with `:aaaa` type" do
      assert ECSResolver.lookup(@params, :aaaa) == []
    end

    test "with `:a` type when fetching resource arns fails" do
      start_aws_test_server()
      expect_aws_post_request(403, aws_api_error_fixture())

      assert CaptureLog.capture_log(fn ->
        assert ECSResolver.lookup(@params, :a) == []
      end) =~ "Error looking up resources by tags"
    end

    test "with `:a` type when describing tasks fails" do
      start_aws_test_server()
      expect_aws_post_request(200, aws_resources_fixture(@task_arns))
      expect_aws_post_request(403, aws_api_error_fixture())

      assert CaptureLog.capture_log(fn ->
        assert ECSResolver.lookup(@params, :a) == []
      end) =~ "Error looking up task IPs"
    end

    test "with `:a` type" do
      start_aws_test_server()

      aws_resources = aws_resources_fixture(@task_arns)

      expect_aws_post_request(200, fn params ->
        assert params["ResourceTypeFilters"] == ["ecs:task"]
        assert params["TagFilters"] == Jason.decode!(@params)["params"]["TagFilters"]

        aws_resources
      end)

      expect_aws_post_request(200, fn params ->
        assert params["cluster"] == Jason.decode!(@params)["cluster"]
        assert params["tasks"] == @task_arns

        aws_tasks_fixture(@task_arns)
      end)

      assert ECSResolver.lookup(@params, :a) == [{10, 0, 0, 0}, {10, 0, 0, 1}, {10, 0, 0, 2}]
    end

    test "with `:a` type when has nextPaginationToken" do
      {task_arns_1, task_arns_2} = Enum.split(@task_arns, 2)

      start_aws_test_server()
      expect_aws_post_request(200, %{aws_resources_fixture(task_arns_1) | "PaginationToken" => "nextPaginationToken"})
      expect_aws_post_request(200, aws_tasks_fixture(task_arns_1))

      expect_aws_post_request(200, fn params ->
        assert params["PaginationToken"] == "nextPaginationToken"

        aws_resources_fixture(task_arns_2)
      end)

      expect_aws_post_request(200, aws_tasks_fixture(task_arns_2, ip_start_at: 3))

      assert ECSResolver.lookup(@params, :a) == [{10, 0, 0, 0}, {10, 0, 0, 1}, {10, 0, 0, 3}]
    end

    defp start_aws_test_server do
      TestServer.start(http_server: {TestServer.HTTPServer.Bandit, http_options: [compress: false]})

      uri = URI.parse(TestServer.url())

      Application.put_env(:aws, :endpoint, "#{uri.host}:#{uri.port}")
      Application.put_env(:aws, :proto, uri.scheme)

      on_exit(fn ->
        Application.delete_env(:aws, :endpoint)
        Application.delete_env(:aws, :proto)
      end)
    end

    defp expect_aws_post_request(status, body_or_function) do
      TestServer.add("/", via: :post, to: fn conn ->
        {:ok, body, _} = Plug.Conn.read_body(conn)
        params = Jason.decode!(body)
        body = is_function(body_or_function) && body_or_function.(params) || body_or_function

        conn
        |> Plug.Conn.put_resp_header("Content-Type", "application/x-amz-json-1.1")
        |> Plug.Conn.send_resp(status, Jason.encode!(body))
      end)
    end

    defp aws_api_error_fixture do
      %{
        "error" => %{
          "code" => "ExpiredToken",
          "message" => "",
          "requestId" => ""
        }
      }
    end

    defp aws_resources_fixture(resource_arns) do
      %{
        "PaginationToken" => "",
        "ResourceTagMappingList" => (
          for resource_arn <- resource_arns do
            %{
                "ComplianceDetails" => %{
                    "ComplianceStatus" => true,
                    "KeysWithNoncompliantValues" => [],
                    "NoncompliantKeys" => []
                },
                "ResourceARN" => resource_arn,
                "Tags" => []
            }
          end
        )
      }
    end

    defp aws_tasks_fixture(resource_arns, opts \\ []) do
      %{
        "failures" => [],
        "tasks" => (
          for {resource_arn, n} <- Enum.with_index(resource_arns) do
            %{
              "taskArn" => resource_arn,
              "overrides" => %{
                "containerOverrides" => [
                  %{
                    "name" => "simple-app"
                  },
                  %{
                    "name" => "busybox"
                  }
                ]
              },
              "lastStatus" => "RUNNING",
              "containerInstanceArn" => "arn:aws:ecs:us-east-1:012345678910:container-instance/default/5991d8da-1d59-49d2-a31f-4230f9e73140",
              "createdAt" => 1_476_822_811.295,
              "version" => 0,
              "clusterArn" => "arn:aws:ecs:us-east-1:012345678910:cluster/default",
              "startedAt" => 1_476_822_833.998,
              "desiredStatus" => "RUNNING",
              "taskDefinitionArn" => "arn:aws:ecs:us-east-1:012345678910:task-definition/console-sample-app-dynamic-ports:1",
              "startedBy" => "ecs-svc/9223370560032507596",
              "containers" => [
                %{
                  "containerArn" => "arn:aws:ecs:us-east-1:012345678910:container/4df26bb4-f057-467b-a079-961675296e64",
                  "taskArn" => "arn:aws:ecs:us-east-1:012345678910:task/default/1dc5c17a-422b-4dc4-b493-371970c6c4d6",
                  "lastStatus" => "RUNNING",
                  "name" => "simple-app",
                  "networkBindings" => [
                    %{
                      "protocol" => "tcp",
                      "bindIP" => "0.0.0.0",
                      "containerPort" => 80,
                      "hostPort" => 32_774
                    }
                  ],
                  "networkInterfaces" => [
                      %{
                        "attachmentId" => "",
                        "ipv6Address" => "",
                        "privateIpv4Address" => "10.0.0.#{Keyword.get(opts, :ip_start_at, 0) + n}"
                      }
                  ]
                },
                %{
                  "containerArn" => "arn:aws:ecs:us-east-1:012345678910:container/e09064f7-7361-4c87-8ab9-8d073bbdbcb9",
                  "taskArn" => "arn:aws:ecs:us-east-1:012345678910:task/default/1dc5c17a-422b-4dc4-b493-371970c6c4d6",
                  "lastStatus" => "RUNNING",
                  "name" => "busybox",
                  "networkBindings" => []
                }
              ]
            }
          end
        )
      }
    end
  end
end