Skip to content
23 changes: 14 additions & 9 deletions e2e/framework/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -234,20 +234,21 @@ func (f *Framework) DevsySSHOnce(
workspace string,
command string,
) (string, error) {
out, stderr, err := f.ExecCommandCapture(ctx, []string{
out, stderr, err := f.ExecCommandDiagnosticCapture(ctx, []string{
cmdWorkspace, cmdSSH, workspace, flagCommand, command, flagDebug,
})
if err != nil {
diagnostic := externalCommandDiagnostic(stderr)
if strings.Contains(stderr, "AGENT_INJECTION_RUNTIME_UNAVAILABLE") ||
strings.Contains(stderr, "PODMAN_ROOTFUL_RUNTIME_POISONED") {
return "", fmt.Errorf(
"SSH_RUNTIME_UNAVAILABLE: %w (stderr: %s)",
errors.Join(docker.ErrRuntimeUnavailable, err),
stderr,
diagnostic,
)
}
if stderr != "" {
return "", fmt.Errorf("devsy ssh failed: %w (stderr: %s)", err, stderr)
return "", fmt.Errorf("devsy ssh failed: %w (stderr: %s)", err, diagnostic)
}
return "", fmt.Errorf("devsy ssh failed: %w", err)
}
Expand Down Expand Up @@ -406,10 +407,10 @@ func providerNameFromAddArgs(args []string) string {
func (f *Framework) DevsyProviderDelete(ctx context.Context, args ...string) error {
baseArgs := []string{cmdProvider, cmdDelete}
baseArgs = append(baseArgs, args...)
_, stderr, err := f.ExecCommandCapture(ctx, baseArgs)
_, stderr, err := f.ExecCommandDiagnosticCapture(ctx, baseArgs)
if err != nil {
if strings.Contains(stderr, "because workspace") {
return fmt.Errorf("%w: %s", errProviderInUse, stderr)
return fmt.Errorf("%w: %s", errProviderInUse, externalCommandDiagnostic(stderr))
}
return err
}
Expand Down Expand Up @@ -603,10 +604,14 @@ func ImportGpgKey(privateKeyPath string) error {
// #nosec G204 -- test imports a controlled key path
out, err := exec.Command("gpg", "--batch", "--import", privateKeyPath).CombinedOutput()
if err != nil {
return fmt.Errorf("import gpg private key: %s: %w", out, err)
return fmt.Errorf(
"import gpg private key: %s: %w",
externalCommandDiagnostic(string(out)),
err,
)
}
if out, err := exec.Command("gpg-connect-agent", "/bye").CombinedOutput(); err != nil {
return fmt.Errorf("start gpg-agent: %s: %w", out, err)
return fmt.Errorf("start gpg-agent: %s: %w", externalCommandDiagnostic(string(out)), err)
}
return nil
}
Expand All @@ -615,7 +620,7 @@ func (f *Framework) DevsySSHGpgSecretKeyForwarded(
ctx context.Context,
workspace, expectedFingerprint string,
) error {
stdout, _, err := f.ExecCommandCapture(ctx, []string{
stdout, _, err := f.ExecCommandDiagnosticCapture(ctx, []string{
cmdWorkspace,
cmdSSH,
names.Flag(names.AgentForwarding),
Expand All @@ -631,7 +636,7 @@ func (f *Framework) DevsySSHGpgSecretKeyForwarded(
return fmt.Errorf(
"gpg agent forwarding failed: secret key %s not reachable in container, got: %q",
expectedFingerprint,
stdout,
externalCommandDiagnostic(stdout),
)
}
return nil
Expand Down
181 changes: 181 additions & 0 deletions e2e/framework/command_diagnostics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
package framework

import (
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"

"github.com/devsy-org/devsy/pkg/docker"
"github.com/stretchr/testify/require"
)

const (
diagnosticFixtureExecutable = "devsy"
diagnosticFixtureValue = "external-tool-fixture"
diagnosticFixtureText = "public command detail"
diagnosticFixture = "Authorization:\n Bearer " + diagnosticFixtureValue + "\n" + diagnosticFixtureText
fixtureCommandFailure = "printf '%s' \"$DEVSY_DIAGNOSTIC_FIXTURE\"\nexit 7\n"
fixtureCommandSuccess = "exit 0\n"
)

func TestGPGCommandFailuresMaskFoldedDiagnostics(t *testing.T) {
for _, tc := range []struct {
name string
importCommand string
agentCommand string
detail string
}{
{"import", fixtureCommandFailure, fixtureCommandSuccess, "import gpg private key"},
{"connect agent", fixtureCommandSuccess, fixtureCommandFailure, "start gpg-agent"},
} {
t.Run(tc.name, func(t *testing.T) {
bin := externalCommandFixture(t)
writeDiagnosticFixture(t, bin, "gpg", tc.importCommand)
writeDiagnosticFixture(t, bin, "gpg-connect-agent", tc.agentCommand)
assertMaskedExternalFailure(
t,
ImportGpgKey(filepath.Join(bin, "fixture.key")),
tc.detail,
)
})
}
}

func TestSSHAgentCommandFailuresMaskFoldedDiagnostics(t *testing.T) {
bin := externalCommandFixture(t)
writeDiagnosticFixture(t, bin, "ssh-keygen", fixtureCommandFailure)
writeDiagnosticFixture(t, bin, "ssh-add", fixtureCommandFailure)
key := filepath.Join(bin, "fixture.key")
for _, tc := range []struct {
name string
run func() error
}{
{"ssh-keygen", func() error { return generateAgentKey(key) }},
{"ssh-add", func() error { return addKeyToAgent(filepath.Join(bin, "socket"), key) }},
} {
t.Run(tc.name, func(t *testing.T) {
assertMaskedExternalFailure(t, tc.run(), tc.name)
})
}
}

func TestForwardedGPGKeyMismatchMasksFoldedDiagnostics(t *testing.T) {
bin := externalCommandFixture(t)
writeDiagnosticFixture(
t,
bin,
diagnosticFixtureExecutable,
"printf '%s' \"$DEVSY_DIAGNOSTIC_FIXTURE\"\n",
)
f := &Framework{DevsyBinDir: bin, DevsyBinName: diagnosticFixtureExecutable}
err := f.DevsySSHGpgSecretKeyForwarded(t.Context(), "fixture-workspace", "expected-fingerprint")
require.ErrorContains(t, err, "expected-fingerprint not reachable")
require.ErrorContains(t, err, diagnosticFixtureText)
require.NotContains(t, err.Error(), diagnosticFixtureValue)
require.Contains(t, err.Error(), "***")
}

func TestFrameworkCommandFailuresMaskDiagnosticCopies(t *testing.T) {
bin := externalCommandFixture(t)
writeDiagnosticFixture(t, bin, diagnosticFixtureExecutable, "printf '%s' 'functional stdout'\n"+
"printf '%s\\n' \"$DEVSY_DIAGNOSTIC_DETAIL\" \"$DEVSY_DIAGNOSTIC_FIXTURE\" >&2\nexit 7\n")
f := &Framework{DevsyBinDir: bin, DevsyBinName: diagnosticFixtureExecutable}
for _, tc := range []struct {
name string
detail string
cause error
run func(*testing.T, *Framework) error
}{
{"ssh", "ordinary SSH failure", nil, fixtureSSHOnce},
{"runtime unavailable", "AGENT_INJECTION_RUNTIME_UNAVAILABLE", docker.ErrRuntimeUnavailable, fixtureSSHOnce},
{"provider in use", "because workspace exists", errProviderInUse, fixtureProviderDelete},
{"JSON result", "JSON command failure", nil, fixtureCommandJSON},
{"setup logs", "setup logs failure", nil, fixtureSetupLogs},
{"workspace edit", "workspace edit failure", nil, fixtureWorkspaceEdit},
} {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("DEVSY_DIAGNOSTIC_DETAIL", tc.detail)
err := tc.run(t, f)
require.ErrorContains(t, err, tc.detail)
require.ErrorContains(t, err, diagnosticFixtureText)
require.NotContains(t, err.Error(), diagnosticFixtureValue)
if tc.cause != nil {
require.ErrorIs(t, err, tc.cause)
}
})
}
}

func fixtureSSHOnce(t *testing.T, f *Framework) error {
_, err := f.DevsySSHOnce(t.Context(), "fixture-workspace", "true")
return err
}

func fixtureProviderDelete(t *testing.T, f *Framework) error {
return f.DevsyProviderDelete(t.Context(), "fixture-provider")
}

func fixtureCommandJSON(t *testing.T, f *Framework) error {
_, err := f.ExecCommandJSON(t.Context(), []string{"fixture"})
return err
}

func fixtureSetupLogs(t *testing.T, f *Framework) error {
t.Helper()
stdout, stderr, err := f.DevsySetupLogs(t.Context())
require.Equal(t, "functional stdout", stdout)
require.Contains(t, stderr, diagnosticFixture)
return err
}

func fixtureWorkspaceEdit(t *testing.T, f *Framework) error {
_, err := f.DevsyWorkspaceEditJSON(t.Context(), "fixture-workspace")
return err
}

func TestWorkspaceEditDecodeErrorMasksDiagnosticCopy(t *testing.T) {
bin := externalCommandFixture(t)
writeDiagnosticFixture(
t,
bin,
diagnosticFixtureExecutable,
"printf '%s' \"$DEVSY_DIAGNOSTIC_FIXTURE\"\n",
)
f := &Framework{DevsyBinDir: bin, DevsyBinName: diagnosticFixtureExecutable}
_, err := f.DevsyWorkspaceEditResult(t.Context(), "fixture-workspace")
require.ErrorContains(t, err, "decode workspace edit result")
require.ErrorContains(t, err, diagnosticFixtureText)
require.NotContains(t, err.Error(), diagnosticFixtureValue)
}

func externalCommandFixture(t *testing.T) string {
t.Helper()
if runtime.GOOS == "windows" {
t.Skip("requires POSIX executable fixtures")
}
bin := t.TempDir()
t.Setenv("PATH", bin)
t.Setenv("DEVSY_DIAGNOSTIC_FIXTURE", diagnosticFixture)
return bin
}

func writeDiagnosticFixture(t *testing.T, bin, name, script string) {
t.Helper()
root, err := os.OpenRoot(bin)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, root.Close()) })
require.NoError(t, root.WriteFile(name, []byte("#!/bin/sh\n"+script), 0o700))
}

func assertMaskedExternalFailure(t *testing.T, err error, detail string) {
t.Helper()
require.ErrorContains(t, err, detail)
var exit *exec.ExitError
require.ErrorAs(t, err, &exit)
require.Equal(t, 7, exit.ExitCode())
require.ErrorContains(t, err, diagnosticFixtureText)
require.NotContains(t, err.Error(), diagnosticFixtureValue)
require.Contains(t, err.Error(), "***")
}
4 changes: 3 additions & 1 deletion e2e/framework/dockerwsl/setup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
wantEndpoint = "ssh://devsyci@" + testAlias
envLineSSHHost = envSSHDockerHost + "=" + wantEndpoint + "\n"
envLineDaemon = envWSLDaemonID + "=" + testDaemonID + "\n"
envLineSSHHostLog = envSSHDockerHost + "=ssh://[REDACTED]@" + testAlias + "\n"
)

type DockerWSLSetupSuite struct {
Expand Down Expand Up @@ -185,7 +186,8 @@ func (s *DockerWSLSetupSuite) TestLocalModeNeedsNoGitHubFiles() {
s.Require().
NoError(s.rig.controller(func(o *dockerWSLOptions) { o.RunID = testRunID }).setup(context.Background()))

s.Contains(s.rig.out.String(), "local env: "+envLineSSHHost)
s.Contains(s.rig.out.String(), "local env: "+envLineSSHHostLog)
s.NotContains(s.rig.out.String(), "ssh://devsyci@")
s.Contains(s.rig.out.String(), "local path: "+fixtureFile("docker-cli", "docker"))
s.Empty(s.rig.files.Text(testGHEnv))
}
Expand Down
63 changes: 55 additions & 8 deletions e2e/framework/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/devsy-org/devsy/internal/diagnosticredact"
"github.com/devsy-org/devsy/logging/redact"
"github.com/devsy-org/devsy/pkg/docker"
"github.com/devsy-org/devsy/pkg/livetee"
)
Expand Down Expand Up @@ -96,32 +99,76 @@ func (f *Framework) ExecCommandContaining(
// ExecCommandCapture executes the command string with the devsy test binary, and returns
// stdout, stderr, and any error that occurred.
func (f *Framework) ExecCommandCapture(ctx context.Context, args []string) (string, string, error) {
return f.execCommandCapture(ctx, args, false)
}

func (f *Framework) ExecCommandDiagnosticCapture(
ctx context.Context,
args []string,
) (string, string, error) {
return f.execCommandCapture(ctx, args, true)
}

func (f *Framework) execCommandCapture(
ctx context.Context,
args []string,
diagnostic bool,
) (string, string, error) {
var execOut bytes.Buffer
var execErr bytes.Buffer

//nolint:gosec // e2e harness runs its own test commands by design
cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...)
docker.PrepareForGroupCancellation(cmd)
cmd.WaitDelay = execWaitDelay
outTee := livetee.New(&execOut, os.Stdout)
cmd.Stdout = outTee
errTee := livetee.New(&execErr, os.Stderr)
cmd.Stderr = errTee
var redactor *redact.Redactor
if diagnostic {
redactor = redact.New(redact.Options{Environment: cmd.Environ()})
}
stdout, closeStdout := captureCommandOutput(&execOut, os.Stdout, redactor)
stderr, closeStderr := captureCommandOutput(&execErr, os.Stderr, redactor)
cmd.Stdout = stdout
cmd.Stderr = stderr

err := cmd.Run()
f.liveStdoutDropped.Add(outTee.Close())
f.liveStderrDropped.Add(errTee.Close())
f.liveStdoutDropped.Add(closeStdout())
f.liveStderrDropped.Add(closeStderr())
if err != nil && ctx.Err() != nil {
err = ctx.Err()
}
return execOut.String(), execErr.String(), err
}

func captureCommandOutput(
capture, live io.Writer,
redactor *redact.Redactor,
) (io.Writer, func() int64) {
if redactor == nil {
tee := livetee.New(capture, live)
return tee, tee.Close
}
tee := livetee.New(io.Discard, live)
diagnostic := newLiveDiagnosticCapture(tee, redactor)
return io.MultiWriter(capture, diagnostic), diagnostic.close
}

func externalCommandDiagnostic(value string) string {
redactor := redact.New(redact.Options{Environment: os.Environ()})
return diagnosticredact.Record(redactor, maskRecordText(redactor, value))
}

// ExecCommandJSON runs the devsy test binary and decodes its stdout as a JSON object.
func (f *Framework) ExecCommandJSON(ctx context.Context, args []string) (map[string]any, error) {
stdout, stderr, err := f.ExecCommandCapture(ctx, args)
stdout, stderr, err := f.ExecCommandDiagnosticCapture(ctx, args)
if err != nil {
return nil, fmt.Errorf("devsy %s: %w (stderr: %s)", strings.Join(args, " "), err, stderr)
return nil, fmt.Errorf(
"devsy %s: %w (stderr: %s)",
externalCommandDiagnostic(
strings.Join(args, " "),
),
err,
externalCommandDiagnostic(stderr),
)
}

var result map[string]any
Expand Down
Loading
Loading