wip - e2e tests for distributed tracing

Author: solnic

.gitignore

@@ -17,3 +17,4 @@ test_integrations/phoenix_app/db
 
 test_integrations/*/_build
 test_integrations/*/deps
+test_integrations/*/test-results/

CONTRIBUTING.md

@@ -55,6 +55,27 @@ mix dialyzer
 
 Once all checks are passing locally, you are ready to [open a pull request](https://help.github.com/articles/using-pull-requests/).
 
+## Manual Testing with Integration Apps
+
+For manual testing of integrations, you can use the `mix test.apps.start` task to start the Phoenix test application with a custom Sentry DSN:
+
+```bash
+# Start the Phoenix app with your DSN
+mix test.apps.start --dsn https://your-public-key@o123456.ingest.sentry.io/123456
+
+# Or set the DSN via environment variable
+export SENTRY_DSN=https://your-public-key@o123456.ingest.sentry.io/123456
+mix test.apps.start
+```
+
+The Phoenix app will start on `http://localhost:4000` and includes:
+- LiveView pages for testing UI interactions
+- Oban for background job processing
+- OpenTelemetry integration for distributed tracing
+- Various error scenarios for testing error reporting
+
+This is useful for testing changes to integrations like Phoenix, Plug, Oban, or OpenTelemetry before submitting a pull request.
+
 That's it. You should be ready to make changes, run tests, and make commits! If you experience any problems, please don't hesitate to ping us in our [Discord Community](https://discord.com/invite/Ww9hbqr).
 
 ## Releasing a New Version

lib/mix/tasks/test.apps.start.ex

@@ -0,0 +1,136 @@
+defmodule Mix.Tasks.Test.Apps.Start do
+  use Mix.Task
+
+  @shortdoc "Start an integration test app for manual testing"
+
+  @moduledoc """
+  Starts an integration test application for manual testing with a custom DSN.
+
+  ## Usage
+
+      $ mix test.apps.start --app phoenix_app --dsn YOUR_DSN
+
+  ## Options
+
+    * `--app` - The integration app to start. Available apps:
+      * `phoenix_app` (default) - Phoenix LiveView application with Oban and OpenTelemetry
+
+    * `--dsn` - The Sentry DSN to use (required). Can be a full DSN URL or
+      omitted to use the DSN from environment variables.
+
+    * `--environment` - The environment name to report to Sentry (default: "manual-test")
+
+  ## Examples
+
+      # Start phoenix_app with a custom DSN
+      $ mix test.apps.start --dsn https://public@sentry.io/123
+
+      # Use DSN from SENTRY_DSN environment variable
+      $ export SENTRY_DSN=https://public@sentry.io/123
+      $ mix test.apps.start
+
+  """
+
+  @switches [
+    app: :string,
+    dsn: :string,
+    environment: :string
+  ]
+
+  @available_apps ["phoenix_app"]
+
+  @impl true
+  def run(args) when is_list(args) do
+    {opts, _args} = OptionParser.parse!(args, strict: @switches)
+
+    app = Keyword.get(opts, :app, "phoenix_app")
+    dsn = Keyword.get(opts, :dsn) || System.get_env("SENTRY_DSN")
+    environment = Keyword.get(opts, :environment, "manual-test")
+
+    unless app in @available_apps do
+      Mix.raise("""
+      Invalid app: #{app}
+
+      Available apps:
+      #{Enum.map_join(@available_apps, "\n", &"  - #{&1}")}
+      """)
+    end
+
+    unless dsn do
+      Mix.raise("""
+      No DSN provided. Please provide a DSN via:
+        --dsn flag: mix test.apps.start --dsn YOUR_DSN
+        Or set SENTRY_DSN environment variable
+      """)
+    end
+
+    app_path = Path.join("test_integrations", app)
+
+    unless File.dir?(app_path) do
+      Mix.raise("Integration app not found: #{app_path}")
+    end
+
+    Mix.shell().info([
+      :cyan,
+      :bright,
+      "\n==> Starting integration app: #{app}",
+      :reset
+    ])
+
+    Mix.shell().info("DSN: #{mask_dsn(dsn)}")
+    Mix.shell().info("Environment: #{environment}\n")
+
+    # Set up dependencies
+    Mix.shell().info("Installing dependencies...")
+
+    case System.cmd("mix", ["deps.get"], cd: app_path, into: IO.stream(:stdio, :line)) do
+      {_, 0} -> :ok
+      {_, status} -> Mix.raise("Failed to install dependencies (exit status: #{status})")
+    end
+
+    # Check if overmind is available
+    case System.cmd("which", ["overmind"], stderr_to_stdout: true) do
+      {_, 0} ->
+        # Set environment variables
+        env = [
+          {"SENTRY_DSN", dsn},
+          {"SENTRY_ENVIRONMENT", environment}
+        ]
+
+        # Start the application
+        Mix.shell().info([
+          :green,
+          :bright,
+          "\n==> Starting #{app} with Overmind...",
+          :reset,
+          "\n"
+        ])
+
+        System.cmd("overmind", ["start"],
+          cd: app_path,
+          into: IO.stream(:stdio, :line),
+          env: env
+        )
+
+      _ ->
+        Mix.raise("""
+        Overmind is not installed. Please install it:
+
+        macOS: brew install overmind tmux
+        Linux: go install github.com/DarthSim/overmind/v2@latest
+
+        Then add to PATH: export PATH=$PATH:$(go env GOPATH)/bin
+        """)
+    end
+  end
+
+  defp mask_dsn(dsn) do
+    case URI.parse(dsn) do
+      %URI{userinfo: userinfo} when is_binary(userinfo) ->
+        String.replace(dsn, userinfo, "***")
+
+      _ ->
+        dsn
+    end
+  end
+end

mix.exs

@@ -22,11 +22,8 @@ defmodule Sentry.Mixfile do
         plt_add_apps: [:mix, :ex_unit]
       ],
       test_coverage: [tool: ExCoveralls],
-      preferred_cli_env: [
-        "coveralls.html": :test,
-        "test.integrations": :test
-      ],
       name: "Sentry",
+      cli: cli(),
       docs: [
         extra_section: "Guides",
         extras: [
@@ -93,6 +90,16 @@ defmodule Sentry.Mixfile do
   defp test_paths(nil), do: ["test"]
   defp test_paths(integration), do: ["test_integrations/#{integration}/test"]
 
+  defp cli do
+    [
+      preferred_envs: [
+        "coveralls.html": :test,
+        "test.integrations": :test,
+        "test.apps.start": :dev
+      ]
+    ]
+  end
+
   defp deps do
     [
       {:nimble_options, "~> 1.0"},

test_integrations/phoenix_app/Procfile

@@ -0,0 +1 @@
+phoenix: mix phx.server

test_integrations/phoenix_app/config/config.exs

@@ -65,6 +65,14 @@ config :opentelemetry, span_processor: {Sentry.OpenTelemetry.SpanProcessor, []}
 config :opentelemetry,
   sampler: {Sentry.OpenTelemetry.Sampler, [drop: ["Elixir.Oban.Stager process"]]}
 
+# Configure OpenTelemetry to use Sentry propagator for distributed tracing
+config :opentelemetry,
+  text_map_propagators: [
+    :trace_context,
+    :baggage,
+    Sentry.OpenTelemetry.Propagator
+  ]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"

test_integrations/phoenix_app/config/dev.exs

@@ -84,12 +84,19 @@ dsn =
     do: System.get_env("SENTRY_DSN_LOCAL"),
     else: System.get_env("SENTRY_DSN")
 
+# For e2e tracing tests, use the TestClient to log events to a file
+client =
+  if System.get_env("SENTRY_E2E_TEST_MODE") == "true",
+    do: PhoenixApp.TestClient,
+    else: Sentry.HackneyClient
+
 config :sentry,
   dsn: dsn,
   environment_name: :dev,
   enable_source_code_context: true,
   send_result: :sync,
-  traces_sample_rate: 1.0
+  traces_sample_rate: 1.0,
+  client: client
 
 config :phoenix_app, Oban,
   repo: PhoenixApp.Repo,

test_integrations/phoenix_app/config/runtime.exs

@@ -20,6 +20,13 @@ if System.get_env("PHX_SERVER") do
   config :phoenix_app, PhoenixAppWeb.Endpoint, server: true
 end
 
+# Allow runtime configuration of Sentry DSN and environment
+if dsn = System.get_env("SENTRY_DSN") do
+  config :sentry,
+    dsn: dsn,
+    environment_name: System.get_env("SENTRY_ENVIRONMENT") || config_env()
+end
+
 if config_env() == :prod do
   # database_url =
   #   System.get_env("DATABASE_URL") ||

test_integrations/phoenix_app/lib/phoenix_app/test_client.ex

@@ -0,0 +1,86 @@
+defmodule PhoenixApp.TestClient do
+  @moduledoc """
+  A test Sentry client that logs envelopes to a file for e2e test validation.
+
+  This client mimics the behavior of Sentry::DebugTransport in sentry-ruby,
+  logging all envelopes to a file that can be read by Playwright tests.
+  """
+
+  require Logger
+
+  @behaviour Sentry.HTTPClient
+
+  @impl true
+  def post(_url, _headers, body) do
+    log_envelope(body)
+
+    # Return success response
+    {:ok, 200, [], ~s({"id":"test-event-id"})}
+  end
+
+  defp log_envelope(body) when is_binary(body) do
+    log_file = Path.join([File.cwd!(), "tmp", "sentry_debug_events.log"])
+
+    # Ensure the tmp directory exists
+    log_dir = Path.dirname(log_file)
+    File.mkdir_p!(log_dir)
+
+    # Parse the envelope binary to extract events and headers
+    case parse_envelope(body) do
+      {:ok, envelope_data} ->
+        # Write the envelope data as JSON
+        json = Jason.encode!(envelope_data)
+        File.write!(log_file, json <> "\n", [:append])
+
+      {:error, reason} ->
+        Logger.warning("Failed to parse envelope for logging: #{inspect(reason)}")
+    end
+  rescue
+    error ->
+      Logger.warning("Failed to log envelope: #{inspect(error)}")
+  end
+
+  defp parse_envelope(body) when is_binary(body) do
+    # Envelope format: header\nitem_header\nitem_payload[\nitem_header\nitem_payload...]
+    # See: https://develop.sentry.dev/sdk/envelopes/
+
+    lines = String.split(body, "\n")
+
+    with {:ok, header_line, rest} <- get_first_line(lines),
+         {:ok, envelope_headers} <- Jason.decode(header_line),
+         {:ok, items} <- parse_items(rest) do
+
+      envelope = %{
+        headers: envelope_headers,
+        items: items
+      }
+
+      {:ok, envelope}
+    else
+      error -> {:error, error}
+    end
+  end
+
+  defp get_first_line([first | rest]), do: {:ok, first, rest}
+  defp get_first_line([]), do: {:error, :empty_envelope}
+
+  defp parse_items(lines), do: parse_items(lines, [])
+
+  defp parse_items([], acc), do: {:ok, Enum.reverse(acc)}
+
+  defp parse_items([item_header_line, payload_line | rest], acc) do
+    with {:ok, _item_header} <- Jason.decode(item_header_line),
+         {:ok, payload} <- Jason.decode(payload_line) do
+      parse_items(rest, [payload | acc])
+    else
+      _error ->
+        # Skip malformed items
+        parse_items(rest, acc)
+    end
+  end
+
+  defp parse_items([_single_line], acc) do
+    # Handle trailing empty line
+    {:ok, Enum.reverse(acc)}
+  end
+end

test_integrations/phoenix_app/lib/phoenix_app_web/controllers/page_controller.ex

@@ -48,4 +48,37 @@ defmodule PhoenixAppWeb.PageController do
 
     render(conn, :home, layout: false)
   end
+
+  # E2E tracing test endpoints
+  def api_error(_conn, _params) do
+    raise ArithmeticError, "bad argument in arithmetic expression"
+  end
+
+  def health(conn, _params) do
+    json(conn, %{status: "ok"})
+  end
+
+  def api_data(conn, _params) do
+    # Fetch actual data from the database
+    Tracer.with_span "fetch_data" do
+      users = Repo.all(User)
+
+      Tracer.with_span "process_data" do
+        # Do some processing
+        user_count = length(users)
+
+        # Make another query to demonstrate nested DB operations
+        first_user = Repo.get(User, 1)
+
+        json(conn, %{
+          message: "Data fetched successfully",
+          data: %{
+            user_count: user_count,
+            first_user: if(first_user, do: first_user.name, else: nil),
+            timestamp: DateTime.utc_now() |> DateTime.to_iso8601()
+          }
+        })
+      end
+    end
+  end
 end