From 40ab524072a4d1a77d14f32d57f3a5ae885c5b94 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 06:50:28 -0600 Subject: [PATCH 01/11] fix(command): mask subprocess diagnostics --- go.mod | 1 + go.sum | 2 + internal/diagnosticredact/redact.go | 96 ++++++++++++++++++++++ internal/diagnosticredact/redact_test.go | 48 +++++++++++ pkg/clierr/errors.go | 11 ++- pkg/clierr/errors_test.go | 11 +++ pkg/command/command.go | 44 +++++++++- pkg/command/diagnostic_test.go | 100 +++++++++++++++++++++++ pkg/command/process_windows.go | 22 +---- pkg/docker/helper.go | 6 +- pkg/docker/image_platform.go | 7 +- pkg/docker/image_platform_test.go | 12 +++ 12 files changed, 331 insertions(+), 29 deletions(-) create mode 100644 internal/diagnosticredact/redact.go create mode 100644 internal/diagnosticredact/redact_test.go create mode 100644 pkg/command/diagnostic_test.go diff --git a/go.mod b/go.mod index f3bee37ce8..bea4e6ac84 100644 --- a/go.mod +++ b/go.mod @@ -229,6 +229,7 @@ require ( github.com/davidmz/go-pageant v1.0.2 // indirect github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect github.com/devsy-org/admin-apis v1.2.0 // indirect + github.com/devsy-org/devsy/logging v0.0.0-20261121080943-b1a01fb92a98 github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb // indirect github.com/dghubble/oauth1 v0.7.3 // indirect github.com/dghubble/sling v1.4.0 // indirect diff --git a/go.sum b/go.sum index 9311f9f120..6baf6e1af8 100644 --- a/go.sum +++ b/go.sum @@ -418,6 +418,8 @@ github.com/devsy-org/apiserver v1.5.4 h1:/bEPrSRSlfii2QHxc9qAiXlQawNJfIaPLraR3bY github.com/devsy-org/apiserver v1.5.4/go.mod h1:sDFCTjCN13wAHhno4KX0Z756gCI8ttW/c7NwTzo16K0= github.com/devsy-org/devsy-runtime-sdk v1.6.0 h1:iEV4Y0Sc3LJNkTLadJSKcWcAf/oCXvC/AkHilrg/KyM= github.com/devsy-org/devsy-runtime-sdk v1.6.0/go.mod h1:MiBP/fiY83DAS0TueEiJZSHYZ1vq85UR4KJTMfHIM+E= +github.com/devsy-org/devsy/logging v0.0.0-20261121080943-b1a01fb92a98 h1:Z+uG77rbf+Msu7LSoWnAe4MylASfsr8gin37+DA4ikk= +github.com/devsy-org/devsy/logging v0.0.0-20261121080943-b1a01fb92a98/go.mod h1:kpwoMhSdrhnJfTJ014ht04BLw0DS7FyF33RqIIe8CKs= github.com/devsy-org/ssh v1.2.10 h1:fCxyqx8ti6qzKhVts55q0zDpx72q1YHNM7VK6TGkAps= github.com/devsy-org/ssh v1.2.10/go.mod h1:IiIPdjt/ilk08Z+I6ofwUaH52I/1PmVnp+0pOVQXlwI= github.com/devsy-org/tailscale v1.102.2 h1:9SB6htvO+HmG8alal8WGCshHapfHR7dUFRSMYiFIzIM= diff --git a/internal/diagnosticredact/redact.go b/internal/diagnosticredact/redact.go new file mode 100644 index 0000000000..cc303d443f --- /dev/null +++ b/internal/diagnosticredact/redact.go @@ -0,0 +1,96 @@ +package diagnosticredact + +import ( + "bytes" + "encoding/json" + "os" + "strings" + + "github.com/devsy-org/devsy/logging/redact" +) + +func EnvironmentRecord(value string) string { + return RecordWithEnvironment(os.Environ(), value) +} + +func RecordWithEnvironment(environment []string, value string) string { + return Record(redact.New(redact.Options{Environment: environment}), value) +} + +func Record(redactor *redact.Redactor, value string) string { + if redactor == nil { + return value + } + conservative := redactor.Redact(value) + folded := redactor.RedactDiagnostic(value) + switch { + case conservative == value || conservative == folded: + return folded + case folded == value: + return conservative + case json.Valid([]byte(value)): + return reconcileObjects(redactor, conservative, folded) + default: + return reconcileText(redactor, conservative, folded) + } +} + +func reconcileText(redactor *redact.Redactor, left, right string) string { + prefix, suffix := sharedEdges(left, right) + start, end := left[:prefix], left[len(left)-suffix:] + marker := redactor.Mask() + if strings.HasSuffix(start, marker) || strings.HasPrefix(end, marker) { + marker = "" + } + masked := strings.ToValidUTF8(start+marker+end, "\uFFFD") + if completeMaskIsStable(redactor, masked) { + return masked + } + return redactor.Mask() +} + +func sharedEdges(left, right string) (int, int) { + limit := min(len(left), len(right)) + prefix := 0 + for prefix < limit && left[prefix] == right[prefix] { + prefix++ + } + suffix := 0 + for suffix < limit-prefix && left[len(left)-1-suffix] == right[len(right)-1-suffix] { + suffix++ + } + return prefix, suffix +} + +func reconcileObjects(redactor *redact.Redactor, conservative, folded string) string { + mask, _ := json.Marshal(redactor.Mask()) + left, right := decodeObject(conservative), decodeObject(folded) + if left == nil || right == nil { + return string(mask) + } + for key, value := range left { + next, present := right[key] + if !present { + delete(left, key) + } else if !bytes.Equal(value, next) { + left[key] = mask + } + } + merged, err := json.Marshal(left) + if err != nil || !completeMaskIsStable(redactor, string(merged)) { + return string(mask) + } + return string(merged) +} + +func completeMaskIsStable(redactor *redact.Redactor, value string) bool { + return redactor.Redact(value) == value && redactor.RedactDiagnostic(value) == value +} + +func decodeObject(value string) map[string]json.RawMessage { + var result map[string]json.RawMessage + if json.Unmarshal([]byte(value), &result) != nil { + return nil + } + return result +} diff --git a/internal/diagnosticredact/redact_test.go b/internal/diagnosticredact/redact_test.go new file mode 100644 index 0000000000..4854ae051a --- /dev/null +++ b/internal/diagnosticredact/redact_test.go @@ -0,0 +1,48 @@ +package diagnosticredact + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/devsy-org/devsy/logging/redact" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRecordKeepsConservativeURLMasking(t *testing.T) { + redactor := redact.FromEntries(nil) + const privateURL = "https://user:synthetic/path@host.test/" + const publicMessage = "ordinary client detail" + const diagnostic = "Authorization:\n Bearer synthetic-folded\n\n" + publicMessage + assert.Equal(t, "https://***@host.test/", Record(redactor, privateURL)) + context := strings.Repeat("a", 17*1024) + raw, err := json.Marshal(map[string]string{ + "msg": diagnostic, "url": privateURL, "level": "error", "context": context, + }) + require.NoError(t, err) + var cleaned map[string]string + require.NoError(t, json.Unmarshal([]byte(Record(redactor, string(raw))), &cleaned)) + assert.Equal(t, "error", cleaned["level"]) + assert.Equal(t, context, cleaned["context"]) + assert.Equal(t, redactor.Mask(), cleaned["url"]) + assert.Equal(t, redactor.Mask(), cleaned["msg"]) +} + +func TestRecordKeepsCompleteInputMaskingAcrossMixedRecords(t *testing.T) { + const canary = "synthetic-mixed-record-value" + const detail = "ordinary mixed record detail" + input := "\"Authorization:\"\n Bearer " + canary + "\n\n" + detail + output := Record(redact.FromEntries(nil), input) + assert.NotContains(t, output, canary) + assert.Contains(t, output, detail) +} + +func TestRecordKeepsOrdinaryPrefixAtTerminalFold(t *testing.T) { + const canary = "synthetic-terminal-fold-value" + const detail = "ordinary detail before terminal fold" + input := detail + "\nAuthorization: Bearer synthetic-initial-value\n " + canary + output := Record(redact.FromEntries(nil), input) + assert.NotContains(t, output, canary) + assert.Contains(t, output, detail) +} diff --git a/pkg/clierr/errors.go b/pkg/clierr/errors.go index 3e2c3b3703..78cd411623 100644 --- a/pkg/clierr/errors.go +++ b/pkg/clierr/errors.go @@ -9,6 +9,7 @@ import ( "maps" "strings" + "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/secrets" "go.uber.org/zap/zapcore" ) @@ -191,9 +192,7 @@ func classifyContextError(err error) *CLIError { } func classifyDockerDaemonError(err error) *CLIError { - message := strings.ToLower(err.Error()) - if !strings.Contains(message, "cannot connect to the docker daemon") && - !strings.Contains(message, "is the docker daemon running") { + if !command.MatchErrorOutput(err, matchesDockerDaemonError) { return nil } return &CLIError{ @@ -204,6 +203,12 @@ func classifyDockerDaemonError(err error) *CLIError { } } +func matchesDockerDaemonError(message string) bool { + message = strings.ToLower(message) + return strings.Contains(message, "cannot connect to the docker daemon") || + strings.Contains(message, "is the docker daemon running") +} + func classifySecretError(err error) *CLIError { for _, state := range []struct { target error diff --git a/pkg/clierr/errors_test.go b/pkg/clierr/errors_test.go index 487a9e6c17..3466599352 100644 --- a/pkg/clierr/errors_test.go +++ b/pkg/clierr/errors_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + "github.com/devsy-org/devsy/pkg/command" "go.uber.org/zap/zapcore" ) @@ -35,6 +36,16 @@ func TestClassifyCancellation(t *testing.T) { const testRateLimited = "rate limited" +func TestClassifyDockerDaemonSurvivesOutputMasking(t *testing.T) { + const marker = "cannot connect to the docker daemon" + t.Setenv("DEVSY_TEST_DOCKER_TOKEN", marker) + err := command.WrapCommandError([]byte(marker), stderrs.New("inspect failed")) + got := Classify(fmt.Errorf("command failed: %w", err)) + if got.Code != CodeDockerDaemonUnreachable || got.Message != dockerDaemonUnavailableMessage { + t.Fatalf("Classify masked Docker output = %+v", got) + } +} + func TestClassify_NilReturnsNil(t *testing.T) { if got := Classify(nil); got != nil { t.Fatalf("Classify(nil) = %v, want nil", got) diff --git a/pkg/command/command.go b/pkg/command/command.go index 6262202b75..f2daf2ea6a 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -2,7 +2,11 @@ package command import ( "errors" + "fmt" "os/exec" + "slices" + + "github.com/devsy-org/devsy/internal/diagnosticredact" ) func WrapCommandError(stdout []byte, err error) error { @@ -22,6 +26,33 @@ type Error struct { } func (e *Error) Error() string { + return diagnosticredact.EnvironmentRecord(e.output()) +} + +func (e *Error) Unwrap() error { + return e.err +} + +func MatchErrorOutput(err error, match func(string) bool) bool { + return err != nil && (match(err.Error()) || matchOriginalCommandOutput(err, match)) +} + +func matchOriginalCommandOutput(err error, match func(string) bool) bool { + switch err := err.(type) { + case nil: + return false + case *Error: + return match(err.output()) || matchOriginalCommandOutput(err.err, match) + case interface{ Unwrap() []error }: + return slices.ContainsFunc(err.Unwrap(), func(cause error) bool { + return matchOriginalCommandOutput(cause, match) + }) + default: + return matchOriginalCommandOutput(errors.Unwrap(err), match) + } +} + +func (e *Error) output() string { message := "" if len(e.stdout) > 0 { message += string(e.stdout) + "\n" @@ -35,8 +66,17 @@ func (e *Error) Error() string { return message + e.err.Error() } -func (e *Error) Unwrap() error { - return e.err +func formatTaskkillDiagnostic(pid int, err error, output []byte) error { + detail := truncateOutput(diagnosticredact.EnvironmentRecord(string(output))) + return fmt.Errorf("taskkill /PID %d /T /F: %w: %s", pid, err, detail) +} + +func truncateOutput(output string) string { + const maxOutput = 256 + if len(output) > maxOutput { + return output[:maxOutput] + "..." + } + return output } func Exists(cmd string) bool { diff --git a/pkg/command/diagnostic_test.go b/pkg/command/diagnostic_test.go new file mode 100644 index 0000000000..c564b295aa --- /dev/null +++ b/pkg/command/diagnostic_test.go @@ -0,0 +1,100 @@ +package command + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTaskkillDiagnosticMasksFoldedOutput(t *testing.T) { + const fixture = "synthetic-taskkill-continuation" + cause := errors.New("operation failed") + err := formatTaskkillDiagnostic( + 42, + cause, + []byte("public diagnostic\nAuthorization: Bearer\n "+fixture), + ) + require.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), "/PID 42") + assert.Contains(t, err.Error(), "public diagnostic") + assert.NotContains(t, err.Error(), fixture) +} + +func TestTaskkillDiagnosticMasksBeforeTruncating(t *testing.T) { + const privatePrefix = "synthetic-long-command-value" + value := privatePrefix + strings.Repeat("z", 96) + t.Setenv("DEVSY_TEST_TASKKILL_TOKEN", value) + output := strings.Repeat("public ", 30) + value + strings.Repeat(" ordinary", 32) + err := formatTaskkillDiagnostic(42, errors.New("fixture failure"), []byte(output)) + assert.NotContains(t, err.Error(), privatePrefix) + assert.Contains(t, err.Error(), "***") + assert.Contains(t, err.Error(), "ordinary") + assert.True(t, strings.HasSuffix(err.Error(), "...")) +} + +func TestCommandErrorMasksFoldedOutput(t *testing.T) { + const fixture = "synthetic-command-continuation" + exit := &exec.ExitError{ + Stderr: []byte("Authorization: Bearer\r\n " + fixture + "\r\npublic stderr"), + } + err := WrapCommandError([]byte("public stdout"), exit) + require.ErrorIs(t, err, exit) + assert.NotContains(t, err.Error(), fixture) + assert.Contains(t, err.Error(), "public stdout") + assert.Contains(t, err.Error(), "public stderr") +} + +func TestWindowsCapturedOutputUsesDiagnosticFormatter(t *testing.T) { + file, err := parser.ParseFile( + token.NewFileSet(), + "process_windows.go", + nil, + parser.SkipObjectResolution, + ) + require.NoError(t, err) + var paths int + for _, decl := range file.Decls { + function, ok := decl.(*ast.FuncDecl) + if !ok || function.Body == nil { + continue + } + captured, formatted := countWindowsDiagnosticCalls(function.Body) + assert.Equal( + t, + captured, + formatted, + "%s must mask captures before formatting errors", + function.Name.Name, + ) + paths += captured + } + assert.GreaterOrEqual(t, paths, 2) +} + +func countWindowsDiagnosticCalls(body *ast.BlockStmt) (captured, formatted int) { + ast.Inspect(body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + switch target := call.Fun.(type) { + case *ast.SelectorExpr: + if target.Sel.Name == "CombinedOutput" { + captured++ + } + case *ast.Ident: + if target.Name == "formatTaskkillDiagnostic" { + formatted++ + } + } + return true + }) + return captured, formatted +} diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index ef2e2ad968..22f747448a 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -154,10 +154,7 @@ func terminateLegacyRefWithoutIdentity(ref ProcessRef) error { cmd := exec.Command("taskkill", "/PID", strconv.Itoa(ref.PID), "/T", "/F") output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf( - "taskkill /PID %d /T /F: %w: %s", - ref.PID, err, truncateOutput(string(output)), - ) + return formatTaskkillDiagnostic(ref.PID, err, output) } return verifyTerminated(ref.PID) } @@ -270,12 +267,7 @@ func killTree(pid, treeName string) error { if checkErr == nil && !stillRunning { return nil } - return fmt.Errorf( - "taskkill /PID %d /T /F: %w: %s", - parsed, - runErr, - truncateOutput(string(output)), - ) + return formatTaskkillDiagnostic(parsed, runErr, output) } if jobErr != nil { @@ -397,16 +389,6 @@ func parsePID(pid string) (int, error) { return parsed, nil } -// truncateOutput bounds captured taskkill output, which is localized and -// never parsed. -func truncateOutput(output string) string { - const maxOutput = 256 - if len(output) > maxOutput { - return output[:maxOutput] + "..." - } - return output -} - func processRefExited(ref ProcessRef) (bool, error) { running, err := savedWorkerRunning(ref) if err != nil { diff --git a/pkg/docker/helper.go b/pkg/docker/helper.go index 90f1b1720e..9150e3e3cc 100644 --- a/pkg/docker/helper.go +++ b/pkg/docker/helper.go @@ -68,7 +68,11 @@ var imageNotFoundMarkers = []string{ } func isImageNotFoundError(err error) bool { - msg := strings.ToLower(err.Error()) + return command.MatchErrorOutput(err, imageNotFoundMessage) +} + +func imageNotFoundMessage(message string) bool { + msg := strings.ToLower(message) for _, marker := range imageNotFoundMarkers { if strings.Contains(msg, marker) { return true diff --git a/pkg/docker/image_platform.go b/pkg/docker/image_platform.go index 7465585bf5..19b1bbb6b7 100644 --- a/pkg/docker/image_platform.go +++ b/pkg/docker/image_platform.go @@ -70,13 +70,12 @@ func (r *DockerHelper) inspectPlatformOrFallback( return nil, fmt.Errorf("inspect image platform: %w", ctx.Err()) } wrapped := command.WrapCommandError(nil, err) - message := strings.ToLower(wrapped.Error()) - if platformInspectionUnsupported(message) { + if command.MatchErrorOutput(wrapped, platformInspectionUnsupported) { // Older engines and compatible runtimes have a single-platform local // image store. The caller still verifies that cached configuration. return base, nil } - if platformInspectionMiss(message) { + if command.MatchErrorOutput(wrapped, platformInspectionMiss) { return nil, fmt.Errorf("%s: %w", image, ErrImageNotFound) } return nil, classifyImageError(image, fmt.Errorf("inspect image platform: %w", wrapped)) @@ -92,11 +91,13 @@ func (r *DockerHelper) inspectPlatformOrFallback( } func platformInspectionUnsupported(message string) bool { + message = strings.ToLower(message) return strings.Contains(message, "unknown flag: --platform") || (strings.Contains(message, "--platform") && strings.Contains(message, "requires api version")) } func platformInspectionMiss(message string) bool { + message = strings.ToLower(message) return strings.Contains(message, "does not match the specified platform") || strings.Contains(message, "no matching manifest for") || strings.Contains(message, "no image found for platform") diff --git a/pkg/docker/image_platform_test.go b/pkg/docker/image_platform_test.go index 8eac158052..caaa63e980 100644 --- a/pkg/docker/image_platform_test.go +++ b/pkg/docker/image_platform_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" @@ -57,6 +58,17 @@ esac require.NoError(t, err) _, err = helper.InspectImageForPlatform(context.Background(), "test:base", "linux/arm64") require.ErrorIs(t, err, ErrImageNotFound) + t.Setenv("DEVSY_TEST_PLATFORM_TOKEN", "unknown flag: --platform") + _, err = helper.InspectImageForPlatform(t.Context(), "test:base", "linux/amd64") + require.NoError(t, err) +} + +func TestImageMissClassificationSurvivesOutputMasking(t *testing.T) { + const marker = "no such image" + t.Setenv("DEVSY_TEST_IMAGE_TOKEN", marker) + err := command.WrapCommandError([]byte(marker), errors.New("inspect failed")) + require.NotContains(t, err.Error(), marker) + require.ErrorIs(t, classifyImageError("fixture", err), ErrImageNotFound) } func TestValidateImagePlatform(t *testing.T) { From 9a91af684f129fcc59f5129b479476513aafb4ff Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 07:02:24 -0600 Subject: [PATCH 02/11] fix(logging): mask operating system tool diagnostics --- cmd/platform/start.go | 36 +++- cmd/platform/start_docker.go | 8 +- cmd/platform/start_docker_diagnostic_test.go | 102 ++++++++++ cmd/platform/start_uninstall.go | 13 +- pkg/apple/helper.go | 41 ++-- pkg/apple/helper_test.go | 35 ++++ pkg/command/command.go | 28 ++- pkg/docker/helper.go | 110 +++++++---- pkg/docker/stdio_transport_test.go | 195 ++++++++++++++++++- pkg/driver/microsandbox/cliclient.go | 19 +- pkg/driver/microsandbox/cliclient_test.go | 49 +++++ pkg/driver/microsandbox/recreate_cli_test.go | 27 ++- pkg/git/lfs.go | 7 +- pkg/git/lfs_test.go | 25 +++ pkg/gitsshsigning/server.go | 13 +- pkg/gitsshsigning/server_test.go | 48 +++++ pkg/gpg/gpg_forwarding.go | 7 +- pkg/gpg/gpg_import_test.go | 53 +++++ pkg/ide/jetbrains/generic.go | 12 +- pkg/ide/jetbrains/generic_test.go | 26 +++ pkg/ide/rstudio/rstudio.go | 43 ++-- pkg/ide/rstudio/rstudio_test.go | 48 +++++ pkg/shell/shell.go | 12 +- pkg/shell/shell_diagnostic_test.go | 19 ++ 24 files changed, 865 insertions(+), 111 deletions(-) create mode 100644 cmd/platform/start_docker_diagnostic_test.go create mode 100644 pkg/gpg/gpg_import_test.go create mode 100644 pkg/ide/rstudio/rstudio_test.go create mode 100644 pkg/shell/shell_diagnostic_test.go diff --git a/cmd/platform/start.go b/cmd/platform/start.go index 2b1ce5b94e..173c5e2dff 100644 --- a/cmd/platform/start.go +++ b/cmd/platform/start.go @@ -22,6 +22,7 @@ import ( "github.com/devsy-org/api/pkg/auth" devsyclientset "github.com/devsy-org/api/pkg/clientset/versioned" platformflags "github.com/devsy-org/devsy/cmd/platform/flags" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/log" devsyopen "github.com/devsy-org/devsy/pkg/open" @@ -59,6 +60,8 @@ const ( var defaultReleaseName = config.PlatformReleaseName +var platformToolCommand = exec.CommandContext + var defaultDeploymentName = "loft" // Need to update helm chart if we change this! // StartCmd holds the login cmd flags. @@ -691,9 +694,10 @@ func checkCLITools(ctx context.Context, kubeContext string) error { ) } - output, err := exec.Command("helm", "version").CombinedOutput() + output, err := platformToolCommand(ctx, "helm", "version").CombinedOutput() if err != nil { - return fmt.Errorf("seems like there are issues with your helm client: \n\n%s", output) + detail := diagnosticredact.EnvironmentRecord(string(output)) + return fmt.Errorf("seems like there are issues with your helm client: \n\n%s", detail) } _, err = exec.LookPath("kubectl") @@ -704,7 +708,7 @@ func checkCLITools(ctx context.Context, kubeContext string) error { ) } - kubectlCmd := exec.CommandContext( + kubectlCmd := platformToolCommand( ctx, "kubectl", "version", @@ -715,7 +719,7 @@ func checkCLITools(ctx context.Context, kubeContext string) error { if err != nil { return fmt.Errorf( "seems like kubectl cannot connect to your Kubernetes cluster: \n\n%s", - output, + diagnosticredact.EnvironmentRecord(string(output)), ) } @@ -1232,13 +1236,17 @@ func installNginxIngress( fmt.Fprint(os.Stderr, "\n") log.Infof("Executing command: helm %s\n", strings.Join(args, " ")) log.Info("Waiting for ingress controller deployment, this can take several minutes") - helmCmd := exec.CommandContext( + helmCmd := platformToolCommand( ctx, "helm", args...) // #nosec G204 -- helm args are constructed internally output, err := helmCmd.CombinedOutput() if err != nil { - return fmt.Errorf("error during helm command: %s (%w)", string(output), err) + return fmt.Errorf( + "error during helm command: %s (%w)", + diagnosticredact.EnvironmentRecord(string(output)), + err, + ) } list, err := kubeClient.CoreV1().Secrets(ingressNginx).List(ctx, metav1.ListOptions{ @@ -1535,7 +1543,7 @@ func upgradeRelease( fmt.Fprint(os.Stderr, "\n") log.Infof("Executing command: helm %s\n", strings.Join(args, " ")) log.Info("Waiting for helm command, this can take up to several minutes") - helmCmd := exec.CommandContext( + helmCmd := platformToolCommand( ctx, "helm", args...) // #nosec G204 -- helm args are constructed internally @@ -1544,7 +1552,11 @@ func upgradeRelease( } output, err := helmCmd.CombinedOutput() if err != nil { - return fmt.Errorf("error during helm command: %s (%w)", string(output), err) + return fmt.Errorf( + "error during helm command: %s (%w)", + diagnosticredact.EnvironmentRecord(string(output)), + err, + ) } log.Infof("Devsy Platform has been deployed to your cluster!") @@ -1564,7 +1576,7 @@ func getReleaseManifests( ) args = helmArgs(args, target, extraArgs) - helmCmd := exec.CommandContext( + helmCmd := platformToolCommand( ctx, "helm", args...) // #nosec G204 -- helm args are constructed internally @@ -1573,7 +1585,11 @@ func getReleaseManifests( } output, err := helmCmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("error during helm command: %s (%w)", string(output), err) + return "", fmt.Errorf( + "error during helm command: %s (%w)", + diagnosticredact.EnvironmentRecord(string(output)), + err, + ) } return string(output), nil } diff --git a/cmd/platform/start_docker.go b/cmd/platform/start_docker.go index 268de41fa9..1e079b331a 100644 --- a/cmd/platform/start_docker.go +++ b/cmd/platform/start_docker.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/config" devcconfig "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/hash" @@ -257,9 +258,10 @@ func (cmd *StartCmd) prepareDocker() error { ) } - output, err := exec.Command("docker", "ps").CombinedOutput() + output, err := platformToolCommand(context.Background(), "docker", "ps").CombinedOutput() if err != nil { - return fmt.Errorf("seems like there are issues with your docker cli: \n\n%s", output) + detail := diagnosticredact.EnvironmentRecord(string(output)) + return fmt.Errorf("seems like there are issues with your docker cli: \n\n%s", detail) } return nil @@ -446,7 +448,7 @@ func (e *Error) Error() string { message += string(exitError.Stderr) + "\n" } - return message + e.err.Error() + return diagnosticredact.EnvironmentRecord(message + e.err.Error()) } func (e *Error) Unwrap() error { diff --git a/cmd/platform/start_docker_diagnostic_test.go b/cmd/platform/start_docker_diagnostic_test.go new file mode 100644 index 0000000000..cb3fdb8fae --- /dev/null +++ b/cmd/platform/start_docker_diagnostic_test.go @@ -0,0 +1,102 @@ +package platform + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/devsy-org/devsy/pkg/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" +) + +const platformFoldedValue = "synthetic-platform-continuation" + +func TestDockerStartErrorMasksFoldedOutput(t *testing.T) { + const fixture = "synthetic-docker-start-continuation" + exit := &exec.ExitError{ + Stderr: []byte("Authorization: Bearer\r\n " + fixture + "\r\npublic stderr"), + } + err := WrapCommandError([]byte("public stdout"), exit) + require.ErrorIs(t, err, exit) + assert.NotContains(t, err.Error(), fixture) + assert.Contains(t, err.Error(), "public stdout") + assert.Contains(t, err.Error(), "public stderr") +} + +func TestPlatformToolFailuresMaskFoldedOutput(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + fixturePlatformTools(t, dir) + t.Setenv("PATH", dir) + target := helmTarget{chartName: "fixture"} + for _, fixture := range []struct { + name string + run func() error + }{ + {"docker", (&StartCmd{}).prepareDocker}, + {"helm version", func() error { return checkCLITools(ctx, "") }}, + {"kubectl version", func() error { return checkCLITools(ctx, "fixture") }}, + {"ingress", func() error { return installNginxIngress(ctx, nil, "") }}, + {"upgrade", func() error { return upgradeRelease(ctx, target, nil) }}, + {"manifests", func() error { + _, err := getReleaseManifests(ctx, target, nil) + return err + }}, + } { + t.Run(fixture.name, func(t *testing.T) { + t.Setenv("DEVSY_FIXTURE_TOOL_SUCCESS", fixture.name) + err := fixture.run() + require.Error(t, err) + assert.Contains(t, err.Error(), "public tool failure") + assert.NotContains(t, err.Error(), platformFoldedValue) + }) + } +} + +func TestHelmUninstallFailureMasksLoggedOutput(t *testing.T) { + fixturePlatformTools(t, t.TempDir()) + observed := log.InitTestObserved(t, zapcore.DebugLevel) + runHelmUninstallCommand(platformToolCommand(t.Context(), "helm")) + require.Positive(t, observed.FilterMessageSnippet("public tool failure").Len()) + require.Zero(t, observed.FilterMessageSnippet(platformFoldedValue).Len()) +} + +func fixturePlatformTools(t *testing.T, dir string) { + t.Helper() + self, err := os.Executable() + require.NoError(t, err) + original := platformToolCommand + t.Cleanup(func() { platformToolCommand = original }) + platformToolCommand = func(ctx context.Context, name string, _ ...string) *exec.Cmd { + command := exec.CommandContext(ctx, self, "-test.run=^TestSyntheticPlatformTool$") + command.Env = append(os.Environ(), "DEVSY_PLATFORM_DIAGNOSTIC_FIXTURE="+name) + return command + } + for _, name := range []string{"docker", "helm", "kubectl"} { + if runtime.GOOS == "windows" { + name += ".exe" + } + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, nil, 0o600)) + mode, err := os.Stat(path) + require.NoError(t, err) + require.NoError(t, os.Chmod(path, mode.Mode()|0o100)) + } +} + +func TestSyntheticPlatformTool(t *testing.T) { + tool := os.Getenv("DEVSY_PLATFORM_DIAGNOSTIC_FIXTURE") + if tool == "" || + tool == "helm" && os.Getenv("DEVSY_FIXTURE_TOOL_SUCCESS") == "kubectl version" { + return + } + _, _ = os.Stderr.WriteString( + "public tool failure\nAuthorization: Bearer\n " + platformFoldedValue + "\n", + ) + os.Exit(9) +} diff --git a/cmd/platform/start_uninstall.go b/cmd/platform/start_uninstall.go index 0df233d26a..1ced5a4fd3 100644 --- a/cmd/platform/start_uninstall.go +++ b/cmd/platform/start_uninstall.go @@ -8,6 +8,7 @@ import ( "strings" devsyclientset "github.com/devsy-org/api/pkg/clientset/versioned" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/log" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -70,12 +71,16 @@ func runHelmUninstall( } log.Infof("Executing command: helm %s", strings.Join(args, " ")) helmCmd := exec.CommandContext(ctx, "helm", args...) // #nosec G204 -- internally constructed - output, err := helmCmd.CombinedOutput() + runHelmUninstallCommand(helmCmd) + return nil +} + +func runHelmUninstallCommand(cmd *exec.Cmd) { + output, err := cmd.CombinedOutput() if err != nil { - log.Errorf("error during helm command: %s (%v)", string(output), err) + log.Errorf("error during helm command: %s (%v)", + diagnosticredact.RecordWithEnvironment(cmd.Environ(), string(output)), err) } - - return nil } func cleanupPlatformResources( diff --git a/pkg/apple/helper.go b/pkg/apple/helper.go index 01e434a4ee..5b284e392c 100644 --- a/pkg/apple/helper.go +++ b/pkg/apple/helper.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/devsy-org/devsy/internal/diagnosticredact" + "github.com/devsy-org/devsy/logging/redact" "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/image" @@ -40,11 +42,11 @@ func (h *AppleHelper) EnsureSystemRunning(ctx context.Context) error { cctx, cancel := context.WithTimeout(ctx, systemStartTimeout) defer cancel() - out, err := h.buildCmd(cctx, "system", "start").CombinedOutput() + out, err := h.diagnosticOutput(cctx, "system", "start") if err != nil { return fmt.Errorf( "start container system service (run `container system start` manually): %s: %w", - strings.TrimSpace(string(out)), err, + out, err, ) } return nil @@ -99,25 +101,25 @@ func (h *AppleHelper) Push(ctx context.Context, image string, stdout, stderr io. } func (h *AppleHelper) Tag(ctx context.Context, image, tag string) error { - out, err := h.buildCmd(ctx, "image", "tag", image, tag).CombinedOutput() + out, err := h.diagnosticOutput(ctx, "image", "tag", image, tag) if err != nil { - return fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err) + return fmt.Errorf("%s: %w", out, err) } return nil } func (h *AppleHelper) Stop(ctx context.Context, id string) error { - out, err := h.buildCmd(ctx, "stop", id).CombinedOutput() + out, err := h.diagnosticOutput(ctx, "stop", id) if err != nil { - return fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err) + return fmt.Errorf("%s: %w", out, err) } return nil } func (h *AppleHelper) Remove(ctx context.Context, id string) error { - out, err := h.buildCmd(ctx, "delete", id).CombinedOutput() + out, err := h.diagnosticOutput(ctx, "delete", id) if err != nil { - return fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err) + return fmt.Errorf("%s: %w", out, err) } return nil } @@ -143,9 +145,9 @@ func (h *AppleHelper) RunWithDir(ctx context.Context, dir string, args []string, } func (h *AppleHelper) StartContainer(ctx context.Context, id string) error { - out, err := h.buildCmd(ctx, "start", id).CombinedOutput() + out, err := h.diagnosticOutput(ctx, "start", id) if err != nil { - return fmt.Errorf("failed to start container: %s: %w", strings.TrimSpace(string(out)), err) + return fmt.Errorf("failed to start container: %s: %w", out, err) } return nil } @@ -339,14 +341,15 @@ func parseImageTag(ref string) string { // EnsureBuilderRunning starts the BuildKit builder; an already-running builder is // a no-op, other failures are returned so a build never runs against a dead builder. func (h *AppleHelper) EnsureBuilderRunning(ctx context.Context) error { - out, err := h.buildCmd(ctx, "builder", "start").CombinedOutput() + cmd := h.buildCmd(ctx, "builder", "start") + out, err := cmd.CombinedOutput() if err == nil { return nil } if strings.Contains(strings.ToLower(string(out)), "already running") { return nil } - return fmt.Errorf("start container builder: %s: %w", strings.TrimSpace(string(out)), err) + return fmt.Errorf("start container builder: %s: %w", appleCommandDiagnostic(cmd, out), err) } // SystemRunning reports whether the container system service is running. @@ -369,6 +372,20 @@ func (h *AppleHelper) buildCmd(ctx context.Context, args ...string) *exec.Cmd { return cmd } +func (h *AppleHelper) diagnosticOutput(ctx context.Context, args ...string) (string, error) { + cmd := h.buildCmd(ctx, args...) + out, err := cmd.CombinedOutput() + if err == nil { + return "", nil + } + return appleCommandDiagnostic(cmd, out), err +} + +func appleCommandDiagnostic(cmd *exec.Cmd, value []byte) string { + redactor := redact.New(redact.Options{Environment: cmd.Environ()}) + return strings.TrimSpace(diagnosticredact.Record(redactor, string(value))) +} + func (h *AppleHelper) listContainers(ctx context.Context) ([]containerInspect, error) { out, err := h.buildCmd(ctx, "list", "--all", "--format", "json").Output() if err != nil { diff --git a/pkg/apple/helper_test.go b/pkg/apple/helper_test.go index d35cda0cfc..08f2914f95 100644 --- a/pkg/apple/helper_test.go +++ b/pkg/apple/helper_test.go @@ -5,12 +5,47 @@ package apple import ( "context" "os" + "os/exec" "path/filepath" "strconv" "testing" "time" + + "github.com/stretchr/testify/require" ) +func TestAppleMasksReturnedFailureOutput(t *testing.T) { + const foldedValue = "synthetic-apple-command-folded-value" + const registeredValue = "synthetic-apple-command-registered-value" + const detail = "public Apple command detail" + const output = "Authorization:\n Bearer " + foldedValue + "\n\n" + detail + " " + registeredValue + helper := &AppleHelper{ + Command: stubContainer(t, output, 23), + Environment: []string{"DEVSY_APPLE_TEST_TOKEN=" + registeredValue}, + } + for _, tc := range []struct { + name string + run func(context.Context) error + }{ + {"system", helper.EnsureSystemRunning}, + {"builder", helper.EnsureBuilderRunning}, + {"start", func(ctx context.Context) error { return helper.StartContainer(ctx, "fixture-container") }}, + {"stop", func(ctx context.Context) error { return helper.Stop(ctx, "fixture-container") }}, + {"remove", func(ctx context.Context) error { return helper.Remove(ctx, "fixture-container") }}, + {"tag", func(ctx context.Context) error { return helper.Tag(ctx, "fixture-image", "fixture-tag") }}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.run(t.Context()) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 23, exitErr.ExitCode()) + require.Contains(t, err.Error(), detail) + require.NotContains(t, err.Error(), foldedValue) + require.NotContains(t, err.Error(), registeredValue) + }) + } +} + func TestMatchesLabels(t *testing.T) { labels := map[string]string{"dev.containers.id": "ws-1", "k": "v"} cases := []struct { diff --git a/pkg/command/command.go b/pkg/command/command.go index f2daf2ea6a..9caef94605 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -5,6 +5,7 @@ import ( "fmt" "os/exec" "slices" + "strings" "github.com/devsy-org/devsy/internal/diagnosticredact" ) @@ -20,12 +21,32 @@ func WrapCommandError(stdout []byte, err error) error { } } +func WrapCombinedOutputError(output string, err error, environment []string) error { + if err == nil || output == "" { + return err + } + masked := strings.TrimSpace(diagnosticredact.RecordWithEnvironment(environment, output)) + if masked == "" { + return err + } + return &Error{ + captured: output, + maskedCapture: masked, + err: err, + } +} + type Error struct { - stdout []byte - err error + stdout []byte + captured string + maskedCapture string + err error } func (e *Error) Error() string { + if e.captured != "" { + return e.maskedCapture + ": " + e.err.Error() + } return diagnosticredact.EnvironmentRecord(e.output()) } @@ -53,6 +74,9 @@ func matchOriginalCommandOutput(err error, match func(string) bool) bool { } func (e *Error) output() string { + if e.captured != "" { + return e.captured + ": " + e.err.Error() + } message := "" if len(e.stdout) > 0 { message += string(e.stdout) + "\n" diff --git a/pkg/docker/helper.go b/pkg/docker/helper.go index 9150e3e3cc..a162da5b87 100644 --- a/pkg/docker/helper.go +++ b/pkg/docker/helper.go @@ -16,6 +16,7 @@ import ( "sync/atomic" "time" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/image" @@ -177,16 +178,26 @@ func (r *DockerHelper) ClientVersion(ctx context.Context) string { } func runCmdCombined(ctx context.Context, cmd *exec.Cmd) error { + out, err := captureCommandOutput(ctx, cmd) + return capturedCommandError(cmd, out, err) +} + +func captureCommandOutput(ctx context.Context, cmd *exec.Cmd) (string, error) { var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out if err := runCmd(ctx, cmd); err != nil { - if msg := strings.TrimSpace(out.String()); msg != "" { - return fmt.Errorf("%s: %w", msg, err) - } - return err + return out.String(), err } - return nil + return "", nil +} + +func capturedCommandError(cmd *exec.Cmd, output string, err error) error { + return command.WrapCombinedOutputError(output, err, cmd.Environ()) +} + +func dockerCommandDiagnostic(cmd *exec.Cmd, value string) string { + return diagnosticredact.RecordWithEnvironment(cmd.Environ(), value) } const envUnset = "" @@ -242,24 +253,18 @@ func runtimeDiagnosticsPodman(r *DockerHelper, diag map[string]string) { // between docker (.ServerVersion) and podman/nerdctl, so a shared template // would falsely fail non-docker runtimes. func (r *DockerHelper) Ping(ctx context.Context) error { - cctx, cancel := context.WithTimeout(ctx, pingTimeout) - defer cancel() - - err := runCmdCombined(cctx, r.buildCmd(cctx, "info")) - if err != nil && cctx.Err() != nil { - return errors.Join(err, cctx.Err()) - } + _, err := r.ping(ctx) return err } // CheckRuntimeHealth reports only control-plane availability failures. A // responsive runtime error must not end an injection retry as a daemon outage. func (r *DockerHelper) CheckRuntimeHealth(ctx context.Context) error { - err := r.Ping(ctx) + rawFailure, err := r.ping(ctx) if err == nil { return nil } - lower := strings.ToLower(err.Error()) + lower := strings.ToLower(rawFailure) if errors.Is(err, context.DeadlineExceeded) || strings.Contains(lower, "cannot connect") || strings.Contains(lower, "connection refused") || @@ -375,29 +380,21 @@ func (r *DockerHelper) DeleteVolume(ctx context.Context, volume string) error { return nil } - out, err := r.buildCmd(ctx, "volume", "list", "-q", "--filter", "name="+volume).CombinedOutput() + out, err := r.combinedOutput(ctx, "volume", "list", "-q", "--filter", "name="+volume) if err != nil { - return fmt.Errorf("list volume %s: %s: %w", volume, strings.TrimSpace(string(out)), err) + return fmt.Errorf("list volume %s: %w", volume, err) } if len(out) == 0 { return nil } - out, err = r.buildCmd(ctx, "volume", "rm", volume).CombinedOutput() - if err != nil { - return fmt.Errorf("%s: %w", string(out), err) - } - - return nil + _, err = r.combinedOutput(ctx, "volume", "rm", volume) + return err } func (r *DockerHelper) Stop(ctx context.Context, id string) error { - out, err := r.buildCmd(ctx, "stop", id).CombinedOutput() - if err != nil { - return fmt.Errorf("%s: %w", string(out), err) - } - - return nil + _, err := r.combinedOutput(ctx, "stop", id) + return err } // PullOptions configures an image pull. @@ -442,12 +439,8 @@ func (r *DockerHelper) Pull(ctx context.Context, opts PullOptions) error { } func (r *DockerHelper) Remove(ctx context.Context, id string) error { - out, err := r.buildCmd(ctx, "rm", "-v", id).CombinedOutput() - if err != nil { - return fmt.Errorf("%s: %w", string(out), err) - } - - return nil + _, err := r.combinedOutput(ctx, "rm", "-v", id) + return err } type Streams struct { @@ -619,17 +612,16 @@ func runStreams(ctx context.Context, cmd *exec.Cmd, streams Streams) error { } func (r *DockerHelper) StartContainer(ctx context.Context, containerId string) error { - out, err := r.buildCmd(ctx, "start", containerId).CombinedOutput() + _, err := r.combinedOutput(ctx, "start", containerId) if err != nil { - stateErr, _ := r.buildCmd(ctx, "inspect", containerId, "--format", - "{{.State.Error}} (exit code: {{.State.ExitCode}})"). - CombinedOutput() - logs, _ := r.buildCmd(ctx, "logs", containerId, "--tail", "50").CombinedOutput() - details := strings.TrimSpace(string(stateErr) + "\n" + string(logs)) + stateErr := r.diagnosticOutput(ctx, "inspect", containerId, "--format", + "{{.State.Error}} (exit code: {{.State.ExitCode}})") + logs := r.diagnosticOutput(ctx, "logs", containerId, "--tail", "50") + details := strings.TrimSpace(stateErr + "\n" + logs) if details != "" { log.Errorf("container failed to start: %s", details) } - return fmt.Errorf("failed to start container: %s: %w", string(out), err) + return fmt.Errorf("failed to start container: %w", err) } container, err := r.FindContainerByID(ctx, []string{containerId}) @@ -644,9 +636,9 @@ func (r *DockerHelper) StartContainer(ctx context.Context, containerId string) e // UnpauseContainer unpauses a paused container. func (r *DockerHelper) UnpauseContainer(ctx context.Context, containerId string) error { - out, err := r.buildCmd(ctx, "unpause", containerId).CombinedOutput() + _, err := r.combinedOutput(ctx, "unpause", containerId) if err != nil { - return fmt.Errorf("failed to unpause container: %s: %w", string(out), err) + return fmt.Errorf("failed to unpause container: %w", err) } return nil @@ -979,6 +971,23 @@ func failedBootSentinel(status config.ContainerStatus, graceElapsed bool) error return nil } +func (r *DockerHelper) ping(ctx context.Context) (string, error) { + cctx, cancel := context.WithTimeout(ctx, pingTimeout) + defer cancel() + + cmd := r.buildCmd(cctx, "info") + out, err := captureCommandOutput(cctx, cmd) + if err == nil { + return "", nil + } + rawFailure := out + "\n" + err.Error() + err = capturedCommandError(cmd, out, err) + if cctx.Err() != nil { + err = errors.Join(err, cctx.Err()) + } + return rawFailure, err +} + func (r *DockerHelper) buildCmd(ctx context.Context, args ...string) *exec.Cmd { name, cmdArgs := r.DockerCommand, args if r.Elevator != nil { @@ -994,6 +1003,21 @@ func (r *DockerHelper) buildCmd(ctx context.Context, args ...string) *exec.Cmd { return cmd } +func (r *DockerHelper) combinedOutput(ctx context.Context, args ...string) ([]byte, error) { + cmd := r.buildCmd(ctx, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, capturedCommandError(cmd, string(out), err) + } + return out, nil +} + +func (r *DockerHelper) diagnosticOutput(ctx context.Context, args ...string) string { + cmd := r.buildCmd(ctx, args...) + output, _ := cmd.CombinedOutput() + return dockerCommandDiagnostic(cmd, string(output)) +} + func (r *DockerHelper) resolveDockerContext(ctx context.Context, envContext string) string { if envContext != envUnset { return envContext diff --git a/pkg/docker/stdio_transport_test.go b/pkg/docker/stdio_transport_test.go index cb9b69ce75..da7f9aabae 100644 --- a/pkg/docker/stdio_transport_test.go +++ b/pkg/docker/stdio_transport_test.go @@ -17,17 +17,31 @@ import ( "testing" "time" + "github.com/devsy-org/devsy/pkg/clierr" + "github.com/devsy-org/devsy/pkg/log" dockerclient "github.com/docker/docker/client" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "go.uber.org/zap/zapcore" ) const ( - helperModeEnv = "DEVSY_TEST_DIAL_STDIO_MODE" - helperEchoMode = "echo" - helperHangMode = "hang" - shortShutdown = 150 * time.Millisecond - boundedInterval = 5 * time.Second + helperModeEnv = "DEVSY_TEST_DIAL_STDIO_MODE" + helperEchoMode = "echo" + helperHangMode = "hang" + helperStartFailureMode = "command-failure" + helperFailureCommandEnv = "DEVSY_TEST_DOCKER_FAILURE_COMMAND" + helperDaemonFixtureEnv = "DEVSY_DOCKER_TEST_DAEMON_TOKEN" + helperExactFixtureEnv = "DEVSY_DOCKER_TEST_EXACT_TOKEN" + fixtureInfoCommand = "info" + fixtureStopCommand = "stop" + fixtureContainerName = "fixture-container" + startFoldedFixture = "synthetic-docker-start-folded-value" + startRegisteredFixture = "synthetic-docker-start-registered-value" + startPublicDetail = "public Docker start detail" + startInspectionDetail = "public Docker inspection detail" + shortShutdown = 150 * time.Millisecond + boundedInterval = 5 * time.Second ) var helperModes = map[string]func(){ @@ -49,8 +63,44 @@ var helperModes = map[string]func(){ spawnChild() time.Sleep(time.Minute) }, - "spawn-child-exit": spawnChild, - "daemon": serveFakeDaemon, + "spawn-child-exit": spawnChild, + "daemon": serveFakeDaemon, + helperStartFailureMode: failDockerFixtureCommand, +} + +func failDockerFixtureCommand() { + if len(os.Args) <= 1 { + return + } + command := os.Args[1] + if command == "volume" && len(os.Args) > 2 { + command += "/" + os.Args[2] + } + failure := os.Getenv(helperFailureCommandEnv) + if command == failure { + writeDockerFixtureFailure() + } + if failure == "start" && command == "inspect" { + _, _ = fmt.Fprintln(os.Stdout, startInspectionDetail, startRegisteredFixture) + } + if command == "volume/list" { + _, _ = fmt.Fprintln(os.Stdout, "fixture-volume-entry") + } +} + +func writeDockerFixtureFailure() { + if exact := os.Getenv(helperExactFixtureEnv); exact != "" { + _, _ = fmt.Fprint(os.Stderr, exact) + os.Exit(23) + } + if daemon := os.Getenv(helperDaemonFixtureEnv); daemon != "" { + _, _ = fmt.Fprintln(os.Stderr, daemon) + } + _, _ = fmt.Fprint( + os.Stderr, + "Authorization:\n Bearer "+startFoldedFixture+" cannot connect\n\n"+startPublicDetail+" "+startRegisteredFixture, + ) + os.Exit(23) } func TestMain(m *testing.M) { @@ -64,6 +114,137 @@ func TestMain(m *testing.M) { os.Exit(0) } +func TestDockerMasksReturnedFailureOutput(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + for _, tc := range []struct { + command string + run func(*DockerHelper, context.Context, string) error + }{ + {"start", (*DockerHelper).StartContainer}, + {fixtureStopCommand, (*DockerHelper).Stop}, + {"rm", (*DockerHelper).Remove}, + {"unpause", (*DockerHelper).UnpauseContainer}, + {"volume/list", (*DockerHelper).DeleteVolume}, + {"volume/rm", (*DockerHelper).DeleteVolume}, + {fixtureInfoCommand, func(helper *DockerHelper, ctx context.Context, _ string) error { return helper.Ping(ctx) }}, + {"machine", func(helper *DockerHelper, ctx context.Context, _ string) error { + return helper.StartPodmanMachine(ctx) + }}, + } { + t.Run(tc.command, func(t *testing.T) { + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=" + helperStartFailureMode, + helperFailureCommandEnv + "=" + tc.command, + "DEVSY_DOCKER_TEST_TOKEN=" + startRegisteredFixture, + }, + } + err := tc.run(helper, t.Context(), fixtureContainerName) + assertDockerReturnedDiagnostic(t, err) + if tc.command == fixtureInfoCommand { + err = helper.CheckRuntimeHealth(t.Context()) + require.ErrorIs(t, err, ErrRuntimeUnavailable) + require.NotContains(t, err.Error(), startFoldedFixture) + } + }) + } +} + +func TestDockerCapturedFailureMasksBeforeTrimming(t *testing.T) { + const fixture = " synthetic-trim-canary " + executable, err := os.Executable() + require.NoError(t, err) + for _, tc := range []struct { + command string + run func(*DockerHelper, context.Context) error + }{ + {fixtureInfoCommand, (*DockerHelper).Ping}, + {fixtureStopCommand, func(helper *DockerHelper, ctx context.Context) error { + return helper.Stop(ctx, fixtureContainerName) + }}, + } { + t.Run(tc.command, func(t *testing.T) { + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=" + helperStartFailureMode, + helperFailureCommandEnv + "=" + tc.command, + helperExactFixtureEnv + "=" + fixture, + }, + } + err := tc.run(helper, t.Context()) + require.Error(t, err) + require.NotContains(t, err.Error(), strings.TrimSpace(fixture)) + require.Contains(t, err.Error(), "***") + }) + } +} + +func TestDockerCapturedFailurePreservesDaemonClassification(t *testing.T) { + const fixture = "cannot connect to the docker daemon" + executable, err := os.Executable() + require.NoError(t, err) + for _, tc := range []struct { + command string + run func(*DockerHelper) error + }{ + {fixtureInfoCommand, func(helper *DockerHelper) error { return helper.Ping(t.Context()) }}, + {fixtureStopCommand, func(helper *DockerHelper) error { return helper.Stop(t.Context(), fixtureContainerName) }}, + } { + t.Run(tc.command, func(t *testing.T) { + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=" + helperStartFailureMode, + helperFailureCommandEnv + "=" + tc.command, + helperDaemonFixtureEnv + "=" + fixture, + }, + } + err := tc.run(helper) + require.Error(t, err) + require.NotContains(t, err.Error(), fixture) + require.Contains(t, err.Error(), startPublicDetail) + require.Equal( + t, + clierr.CodeDockerDaemonUnreachable, + clierr.Classify(fmt.Errorf("preflight: %w", err)).Code, + ) + var exit *exec.ExitError + require.ErrorAs(t, err, &exit) + require.Equal(t, 23, exit.ExitCode()) + }) + } +} + +func TestDockerStartMasksSupplementalDiagnostics(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + observed := log.InitTestObserved(t, zapcore.DebugLevel) + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=" + helperStartFailureMode, + helperFailureCommandEnv + "=start", + "DEVSY_DOCKER_TEST_TOKEN=" + startRegisteredFixture, + }, + } + require.Error(t, helper.StartContainer(t.Context(), fixtureContainerName)) + require.Positive(t, observed.FilterMessageSnippet(startInspectionDetail).Len()) + require.Zero(t, observed.FilterMessageSnippet(startRegisteredFixture).Len()) +} + +func assertDockerReturnedDiagnostic(t *testing.T, err error) { + t.Helper() + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 23, exitErr.ExitCode()) + require.Contains(t, err.Error(), startPublicDetail) + require.NotContains(t, err.Error(), startFoldedFixture) + require.NotContains(t, err.Error(), startRegisteredFixture) +} + // spawnChild starts a long-lived descendant immediately and reports its pid, // standing in for docker starting SSH before it exits or waits. func spawnChild() { diff --git a/pkg/driver/microsandbox/cliclient.go b/pkg/driver/microsandbox/cliclient.go index 2f32c041ff..262b63d9b5 100644 --- a/pkg/driver/microsandbox/cliclient.go +++ b/pkg/driver/microsandbox/cliclient.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/flags/names" @@ -45,7 +46,7 @@ func (cliClient) Version(ctx context.Context) (string, error) { if err != nil { return "", fmt.Errorf( "get microsandbox version: %s: %w", - strings.TrimSpace(string(out)), err, + strings.TrimSpace(diagnosticredact.EnvironmentRecord(string(out))), err, ) } return strings.TrimSpace(string(out)), nil @@ -207,7 +208,8 @@ func (cliClient) ensureVolumes(ctx context.Context, mounts []volumeMount) error // #nosec G204 -- args are a resolved binary path and a named-volume name out, err := exec.CommandContext(ctx, msbBinary(), "volume", "create", vol).CombinedOutput() if err != nil && !strings.Contains(strings.ToLower(string(out)), "already exists") { - return fmt.Errorf("create microsandbox volume %q: %s: %w", vol, out, err) + return fmt.Errorf("create microsandbox volume %q: %s: %w", + vol, diagnosticredact.EnvironmentRecord(string(out)), err) } } return nil @@ -319,7 +321,7 @@ func msbRun(ctx context.Context, args ...string) error { return fmt.Errorf( "msb %s: %s: %w", redactArgs(args), - strings.TrimSpace(string(out)), + strings.TrimSpace(diagnosticredact.EnvironmentRecord(string(out))), err, ) } @@ -368,12 +370,19 @@ func loadImageSnapshot(ctx context.Context, imageRef string, img v1.Image) error if err := errors.Join(writeErr, closeErr, ctx.Err()); err != nil { _ = load.Process.Kill() _ = load.Wait() - return fmt.Errorf("stream image %q: %s: %w", imageRef, output.String(), err) + return fmt.Errorf( + "stream image %q: %s: %w", + imageRef, + diagnosticredact.EnvironmentRecord(output.String()), + err, + ) } if err := load.Wait(); err != nil { return fmt.Errorf( "msb load %q: %s: %w", - imageRef, output.String(), errors.Join(err, ctx.Err()), + imageRef, + diagnosticredact.EnvironmentRecord(output.String()), + errors.Join(err, ctx.Err()), ) } return nil diff --git a/pkg/driver/microsandbox/cliclient_test.go b/pkg/driver/microsandbox/cliclient_test.go index 534a8c4fb6..3585085f51 100644 --- a/pkg/driver/microsandbox/cliclient_test.go +++ b/pkg/driver/microsandbox/cliclient_test.go @@ -1,7 +1,10 @@ package microsandbox import ( + "bytes" + "context" "fmt" + "os/exec" "slices" "strings" "testing" @@ -10,9 +13,55 @@ import ( "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/stretchr/testify/require" ) +func TestCLIClientMasksFailureOutput(t *testing.T) { + installRecreateCLIExecutable(t) + t.Setenv(recreateCLIFixtureEnv, diagnosticCLIFailure) + client := cliClient{} + for name, run := range map[string]func(context.Context) error{ + "version": func(ctx context.Context) error { + _, err := client.Version(ctx) + return err + }, + "volume": func(ctx context.Context) error { + return client.PrepareVolumes(ctx, []volumeMount{{Volume: "fixture-volume"}}) + }, + "start sandbox": func(ctx context.Context) error { return client.Start(ctx, "fixture-workspace") }, + "load": func(ctx context.Context) error { return loadImageSnapshot(ctx, testImg, empty.Image) }, + } { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + err := run(ctx) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 23, exitErr.ExitCode()) + require.Contains(t, err.Error(), diagnosticCLIDetail) + require.NotContains(t, err.Error(), diagnosticCLIValue) + }) + } +} + +func TestCLIClientPreservesSuccessfulOutput(t *testing.T) { + installRecreateCLIExecutable(t) + t.Setenv(recreateCLIFixtureEnv, diagnosticCLISuccess) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + client := cliClient{} + version, err := client.Version(ctx) + require.NoError(t, err) + require.Equal(t, diagnosticCLIOutput, version) + var out bytes.Buffer + require.NoError(t, client.Exec(ctx, "fixture-workspace", execRequest{Stdout: &out})) + require.Equal(t, diagnosticCLIOutput, out.String()) + out.Reset() + require.NoError(t, client.Logs(ctx, "fixture-workspace", &out)) + require.Equal(t, diagnosticCLIOutput, out.String()) +} + func runPrefix() []string { return []string{msbCmdRun, msbFlagDetach, names.Flag(names.Name), wsName} } diff --git a/pkg/driver/microsandbox/recreate_cli_test.go b/pkg/driver/microsandbox/recreate_cli_test.go index 6e675d3b53..0087c03356 100644 --- a/pkg/driver/microsandbox/recreate_cli_test.go +++ b/pkg/driver/microsandbox/recreate_cli_test.go @@ -21,10 +21,16 @@ const ( recreateCLIFixtureEnv = "DEVSY_TEST_MSB_CREATE_FIXTURE" recreateCLIFixtureMode = "capture-argv" recreateCLIArgumentFile = "create-argv.json" + diagnosticCLIFailure = "diagnostic-failure" + diagnosticCLISuccess = "diagnostic-success" + diagnosticCLIValue = "synthetic-msb-folded-value" + diagnosticCLIDetail = "public microsandbox detail" + diagnosticCLIOutput = "Authorization:\n Bearer " + diagnosticCLIValue + "\n\n" + diagnosticCLIDetail ) func TestMain(m *testing.M) { - if os.Getenv(recreateCLIFixtureEnv) == recreateCLIFixtureMode { + switch os.Getenv(recreateCLIFixtureEnv) { + case recreateCLIFixtureMode: if handled, err := captureRecreateCLIArguments(); handled { if err != nil { _, _ = fmt.Fprintln(os.Stderr, err) @@ -32,10 +38,29 @@ func TestMain(m *testing.M) { } os.Exit(0) } + case diagnosticCLIFailure, diagnosticCLISuccess: + executable, err := os.Executable() + if err == nil && filepath.Base(executable) == recreateCLIExecutableName() { + os.Exit(runDiagnosticCLIFixture()) + } } os.Exit(m.Run()) } +func runDiagnosticCLIFixture() int { + if len(os.Args) > 1 && os.Args[1] == "load" { + if _, err := io.Copy(io.Discard, os.Stdin); err != nil { + return 1 + } + } + if os.Getenv(recreateCLIFixtureEnv) == diagnosticCLISuccess { + _, _ = fmt.Fprint(os.Stdout, diagnosticCLIOutput) + return 0 + } + _, _ = fmt.Fprint(os.Stderr, diagnosticCLIOutput) + return 23 +} + // Only the test executable copied to an owned msb path enters fixture mode. // Its fixed result file is adjacent to that copy; an environment value cannot // select an arbitrary output path or divert the ordinary package test binary. diff --git a/pkg/git/lfs.go b/pkg/git/lfs.go index b97b9aaffe..a710c1572c 100644 --- a/pkg/git/lfs.go +++ b/pkg/git/lfs.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/log" ) @@ -95,7 +96,11 @@ func ensureLFSBinary(ctx context.Context, allowInstall bool) bool { func (r *Repo) lfs(ctx context.Context, args ...string) error { res, err := r.run(ctx, append([]string{"lfs"}, args...)...) if err != nil { - return fmt.Errorf("%w: %s%s", err, res.Stdout, res.Stderr) + return fmt.Errorf( + "%w: %s", + err, + diagnosticredact.EnvironmentRecord(string(res.Stdout)+string(res.Stderr)), + ) } return nil } diff --git a/pkg/git/lfs_test.go b/pkg/git/lfs_test.go index 018713d4e5..b949613646 100644 --- a/pkg/git/lfs_test.go +++ b/pkg/git/lfs_test.go @@ -11,8 +11,33 @@ import ( "github.com/devsy-org/devsy/pkg/command" "gotest.tools/assert" + "gotest.tools/assert/cmp" ) +const ( + lfsFoldedFixtureValue = "synthetic-lfs-folded-value" + lfsDiagnosticDetail = "public Git LFS detail" +) + +type lfsSplitRunner struct{ err error } + +func (r lfsSplitRunner) Run(context.Context, RunOptions) (RunResult, error) { + return RunResult{ + Stdout: []byte("Authorization:\n"), + Stderr: []byte(" Bearer " + lfsFoldedFixtureValue + "\n\n" + lfsDiagnosticDetail), + }, r.err +} + +func TestLFSFailureMasksFoldedOutputAcrossStreams(t *testing.T) { + cause := errors.New("fixture LFS process failure") + repo := At(t.TempDir(), WithRunner(lfsSplitRunner{err: cause})) + err := repo.lfs(t.Context(), "pull") + assert.Assert(t, errors.Is(err, cause)) + assert.Assert(t, cmp.Contains(err.Error(), lfsDiagnosticDetail)) + assert.Assert(t, !strings.Contains(err.Error(), lfsFoldedFixtureValue)) + assert.NilError(t, At(t.TempDir(), WithRunner(lfsSplitRunner{})).lfs(t.Context(), "pull")) +} + func TestLFSModeResolution(t *testing.T) { cases := []struct { name string diff --git a/pkg/gitsshsigning/server.go b/pkg/gitsshsigning/server.go index 02bd6f6571..53882dd1ed 100644 --- a/pkg/gitsshsigning/server.go +++ b/pkg/gitsshsigning/server.go @@ -8,6 +8,8 @@ import ( "os" "os/exec" "strings" + + "github.com/devsy-org/devsy/internal/diagnosticredact" ) const defaultSigningNamespace = "git" @@ -51,15 +53,22 @@ func (req *GitSSHSignatureRequest) SignWithHostKey( "-n", namespace, ) - cmd.Stdin = strings.NewReader(req.Content) + return runSigningCommand(cmd, req.Content) +} +func runSigningCommand(cmd *exec.Cmd, content string) (*GitSSHSignatureResponse, error) { + cmd.Stdin = strings.NewReader(content) var out bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &out cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("failed to sign commit: %w, stderr: %s", err, stderr.String()) + return nil, fmt.Errorf( + "failed to sign commit: %w, stderr: %s", + err, + diagnosticredact.EnvironmentRecord(stderr.String()), + ) } return &GitSSHSignatureResponse{ diff --git a/pkg/gitsshsigning/server_test.go b/pkg/gitsshsigning/server_test.go index 950a47ddec..399148d097 100644 --- a/pkg/gitsshsigning/server_test.go +++ b/pkg/gitsshsigning/server_test.go @@ -2,6 +2,7 @@ package gitsshsigning import ( "context" + "fmt" "io" "os" "os/exec" @@ -22,8 +23,55 @@ const ( testEd25519Prefix = "ssh-ed25519 " testSignerPrincipal = "test@example.com" envSSHAuthSock = "SSH_AUTH_SOCK" + signingFixtureFail = "signing-output-failure" + signingFixturePass = "signing-output-success" + signingFoldedValue = "synthetic-signing-folded-value" + signingPublicDetail = "public signing detail" ) +func TestSigningMasksOnlyFailureOutput(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + for _, mode := range []string{signingFixturePass, signingFixtureFail} { + t.Run(mode, func(t *testing.T) { + cmd := &exec.Cmd{ + Path: executable, + Args: []string{executable, "-test.run=^TestSigningOutputFixture$", "--", mode}, + } + resp, err := runSigningCommand(cmd, signingFixtureOutput()) + if mode == signingFixturePass { + require.NoError(t, err) + require.Equal(t, []byte(signingFixtureOutput()), resp.Signature) + return + } + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 23, exitErr.ExitCode()) + require.Nil(t, resp) + require.Contains(t, err.Error(), signingPublicDetail) + require.NotContains(t, err.Error(), signingFoldedValue) + }) + } +} + +func signingFixtureOutput() string { + return "Authorization:\n Bearer " + signingFoldedValue + "\n\n" + signingPublicDetail +} + +func TestSigningOutputFixture(t *testing.T) { + switch os.Args[len(os.Args)-1] { + case signingFixturePass: + _, err := io.Copy(os.Stdout, os.Stdin) + if err != nil { + os.Exit(1) + } + os.Exit(0) + case signingFixtureFail: + _, _ = fmt.Fprint(os.Stderr, signingFixtureOutput()) + os.Exit(23) + } +} + func TestSign_WithPublicKeyContent_WritesToTempFile(t *testing.T) { req := &GitSSHSignatureRequest{ Content: "tree abc123\nauthor Test \n\ntest commit", diff --git a/pkg/gpg/gpg_forwarding.go b/pkg/gpg/gpg_forwarding.go index e0a73295c5..b50e68131a 100644 --- a/pkg/gpg/gpg_forwarding.go +++ b/pkg/gpg/gpg_forwarding.go @@ -12,6 +12,7 @@ import ( "time" "al.essio.dev/pkg/shellescape" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/log" devssh "github.com/devsy-org/devsy/pkg/ssh" "golang.org/x/crypto/ssh" @@ -98,7 +99,10 @@ func (g *GPGConf) ImportOwnerTrust() error { func runGpgWithStdin(input []byte, args ...string) error { //nolint:gosec // args are internal gpg directive literals cmd := exec.Command("gpg", args...) + return runGpgImport(cmd, input) +} +func runGpgImport(cmd *exec.Cmd, input []byte) error { stdin, err := cmd.StdinPipe() if err != nil { return err @@ -110,7 +114,8 @@ func runGpgWithStdin(input []byte, args ...string) error { out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("gpg %s: %s: %w", strings.Join(args, " "), out, err) + return fmt.Errorf("gpg %s: %s: %w", + strings.Join(cmd.Args[1:], " "), diagnosticredact.EnvironmentRecord(string(out)), err) } return nil } diff --git a/pkg/gpg/gpg_import_test.go b/pkg/gpg/gpg_import_test.go new file mode 100644 index 0000000000..bd42a7f36e --- /dev/null +++ b/pkg/gpg/gpg_import_test.go @@ -0,0 +1,53 @@ +package gpg + +import ( + "fmt" + "io" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + importFixtureMode = "gpg-import-output-fixture" + importFixtureInput = "literal GPG import fixture input" + importFoldedValue = "synthetic-gpg-import-folded-value" + importPublicDetail = "public GPG import detail" +) + +func TestGpgImportMasksReturnedFailureOutput(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{ + Path: executable, + Args: []string{ + executable, + "-test.run=^TestGpgImportOutputFixture$", + "--", + importFixtureMode, + }, + } + err = runGpgImport(cmd, []byte(importFixtureInput)) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 23, exitErr.ExitCode()) + require.Contains(t, err.Error(), importPublicDetail) + require.NotContains(t, err.Error(), importFoldedValue) +} + +func TestGpgImportOutputFixture(t *testing.T) { + if os.Args[len(os.Args)-1] != importFixtureMode { + return + } + input, err := io.ReadAll(os.Stdin) + if err != nil || string(input) != importFixtureInput { + os.Exit(1) + } + _, _ = fmt.Fprint( + os.Stderr, + "Authorization:\n Bearer "+importFoldedValue+"\n\n"+importPublicDetail, + ) + os.Exit(23) +} diff --git a/pkg/ide/jetbrains/generic.go b/pkg/ide/jetbrains/generic.go index b1a4f749cc..278db23927 100644 --- a/pkg/ide/jetbrains/generic.go +++ b/pkg/ide/jetbrains/generic.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/command" config2 "github.com/devsy-org/devsy/pkg/config" copy2 "github.com/devsy-org/devsy/pkg/copy" @@ -217,7 +218,11 @@ func (o *GenericJetBrainsServer) installPlugins( return err } - out, err := installPluginsCommand.CombinedOutput() + return runPluginInstaller(installPluginsCommand, plugins) +} + +func runPluginInstaller(cmd *exec.Cmd, plugins []string) error { + out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf( "failed to install JetBrains plugins %v: %w", @@ -226,7 +231,10 @@ func (o *GenericJetBrainsServer) installPlugins( ) } - log.Debugf("remote-dev-server.sh output: %s", out) + log.Debugf( + "remote-dev-server.sh output: %s", + diagnosticredact.RecordWithEnvironment(cmd.Environ(), string(out)), + ) log.Infof("installed JetBrains plugins: plugins=%v", plugins) return nil diff --git a/pkg/ide/jetbrains/generic_test.go b/pkg/ide/jetbrains/generic_test.go index cb6bc48c86..3754e44753 100644 --- a/pkg/ide/jetbrains/generic_test.go +++ b/pkg/ide/jetbrains/generic_test.go @@ -8,14 +8,17 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "runtime" "testing" "github.com/devsy-org/devsy/pkg/config" devconfig "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" ) const ( @@ -277,3 +280,26 @@ func TestInstallPluginsSkipsWhenNoneConfigured(t *testing.T) { assert.NoError(t, err) } + +const pluginInstallerFixture = "public plugin diagnostic\nAuthorization:\n Bearer synthetic-plugin-value" + +func TestPluginInstallerMasksSuccessfulDiagnosticOutput(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{ + Path: executable, + Args: []string{executable, "-test.run=^TestPluginInstallerOutputFixture$"}, + Env: append(os.Environ(), "DEVSY_JETBRAINS_DIAGNOSTIC_FIXTURE=1"), + } + observed := log.InitTestObserved(t, zapcore.DebugLevel) + require.NoError(t, runPluginInstaller(cmd, []string{"example.plugin"})) + require.Positive(t, observed.FilterMessageSnippet("public plugin diagnostic").Len()) + require.Zero(t, observed.FilterMessageSnippet("synthetic-plugin-value").Len()) +} + +func TestPluginInstallerOutputFixture(t *testing.T) { + if os.Getenv("DEVSY_JETBRAINS_DIAGNOSTIC_FIXTURE") == "1" { + _, _ = os.Stdout.WriteString(pluginInstallerFixture) + os.Exit(0) + } +} diff --git a/pkg/ide/rstudio/rstudio.go b/pkg/ide/rstudio/rstudio.go index fb7f13aef1..8df9cc6260 100644 --- a/pkg/ide/rstudio/rstudio.go +++ b/pkg/ide/rstudio/rstudio.go @@ -13,6 +13,7 @@ import ( "runtime" "strings" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/config" copypkg "github.com/devsy-org/devsy/pkg/copy" @@ -142,16 +143,11 @@ func (o *RStudioServer) installSteps(debPath string) error { func ensureGdebi() error { if !command.Exists("gdebi") { log.Info("Installing dependency gdebi-core") - out, err := exec.Command("apt", "update").CombinedOutput() - if err != nil { - return fmt.Errorf("apt update: %w: %s", err, string(out)) - } - - out, err = exec.Command("apt", "-y", "install", "--no-install-recommends", "gdebi-core"). - CombinedOutput() - if err != nil { - return fmt.Errorf("install gdebi core: %w: %s", err, string(out)) + if err := runInstallerCommand("apt update", exec.Command("apt", "update")); err != nil { + return err } + return runInstallerCommand("install gdebi core", + exec.Command("apt", "-y", "install", "--no-install-recommends", "gdebi-core")) } return nil @@ -244,22 +240,24 @@ func installDeb(debPath string) error { log.Info("Installing deb") //nolint:gosec // debPath is the file this process just downloaded into its own install folder - out, err := exec.Command("gdebi", "--non-interactive", debPath).CombinedOutput() - if err != nil { - return fmt.Errorf("install deb: %w: %s", err, string(out)) + install := exec.Command("gdebi", "--non-interactive", debPath) + if err := runInstallerCommand("install deb", install); err != nil { + return err } // The installer unfortunately automatically starts a new rstudio process without reading the configuration. // We're stopping that so that our process can take over later on - out, err = exec.Command(defaultBinaryPathRStudioServer, "stop").CombinedOutput() - if err != nil { - return fmt.Errorf("stop initial RStudio process: %w: %s", err, string(out)) + if err := runInstallerCommand( + "stop initial RStudio process", + exec.Command(defaultBinaryPathRStudioServer, "stop"), + ); err != nil { + return err } // The RStudio deb installs into /usr/lib/rstudio-server/bin/rstudio-server by default // and symlinks the binaries to /usr/sbin. // We need to symlink to /usr/local/bin to ensure remoteUser has access later on - err = os.Symlink(defaultBinaryPathRStudioServer, "/usr/local/bin/rstudio-server") + err := os.Symlink(defaultBinaryPathRStudioServer, "/usr/local/bin/rstudio-server") if err != nil { return fmt.Errorf("symlink rstudio-server: %w", err) } @@ -271,6 +269,19 @@ func installDeb(debPath string) error { return nil } +func runInstallerCommand(operation string, cmd *exec.Cmd) error { + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf( + "%s: %w: %s", + operation, + err, + diagnosticredact.EnvironmentRecord(string(out)), + ) + } + return nil +} + func ensureConfigFolder(userName string) error { err := os.MkdirAll(dataFolder, 0o750) if err != nil { diff --git a/pkg/ide/rstudio/rstudio_test.go b/pkg/ide/rstudio/rstudio_test.go new file mode 100644 index 0000000000..95193afca2 --- /dev/null +++ b/pkg/ide/rstudio/rstudio_test.go @@ -0,0 +1,48 @@ +package rstudio + +import ( + "fmt" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + installerFixtureMode = "rstudio-installer-output-fixture" + installerFoldedValue = "synthetic-rstudio-folded-value" + installerDetail = "public RStudio installer detail" +) + +func TestInstallerErrorMasksFoldedAuthorization(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{ + Path: executable, + Args: []string{ + executable, + "-test.run=^TestInstallerOutputFixture$", + "--", + installerFixtureMode, + }, + } + err = runInstallerCommand("fixture installer", cmd) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 23, exitErr.ExitCode()) + require.Contains(t, err.Error(), "fixture installer") + require.Contains(t, err.Error(), installerDetail) + require.NotContains(t, err.Error(), installerFoldedValue) +} + +func TestInstallerOutputFixture(t *testing.T) { + if os.Args[len(os.Args)-1] != installerFixtureMode { + return + } + _, _ = fmt.Fprint( + os.Stderr, + "Authorization:\n Bearer "+installerFoldedValue+"\n\n"+installerDetail, + ) + os.Exit(23) +} diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index f315f872ce..b15adb1866 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/devsy-org/devsy/internal/diagnosticredact" "mvdan.cc/sh/v3/expand" "mvdan.cc/sh/v3/interp" "mvdan.cc/sh/v3/syntax" @@ -177,9 +178,16 @@ func getUserShell(userName string) (string, error) { return "", err } - shell := strings.Split(string(output), ":") + return parseUserShell(string(output)) +} + +func parseUserShell(output string) (string, error) { + shell := strings.Split(output, ":") if len(shell) != 7 { - return "", fmt.Errorf("unexpected getent format: %s", string(output)) + return "", fmt.Errorf( + "unexpected getent format: %s", + diagnosticredact.EnvironmentRecord(output), + ) } loginShell := strings.TrimSpace(filepath.Base(shell[6])) diff --git a/pkg/shell/shell_diagnostic_test.go b/pkg/shell/shell_diagnostic_test.go new file mode 100644 index 0000000000..6fbf8e7470 --- /dev/null +++ b/pkg/shell/shell_diagnostic_test.go @@ -0,0 +1,19 @@ +package shell + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUserShellMalformedOutputMasksFoldedDiagnostic(t *testing.T) { + const fixture = "synthetic-getent-continuation" + shell, err := parseUserShell("user:x:10:10:User:/home/user:/bin/bash") + require.NoError(t, err) + assert.Equal(t, "bash", shell) + _, err = parseUserShell("public account detail\nAuthorization: Bearer\n " + fixture) + require.Error(t, err) + assert.Contains(t, err.Error(), "public account detail") + assert.NotContains(t, err.Error(), fixture) +} From b43fa897f4cc59555a995fe30f12ee2221cef386 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 07:09:44 -0600 Subject: [PATCH 03/11] fix(command): mask effective subprocess environment values --- pkg/apple/helper.go | 33 +++++++++++++++++------ pkg/command/command.go | 18 +++++++++++-- pkg/docker/helper.go | 22 ++++++++++----- pkg/docker/image_platform.go | 5 ++-- pkg/docker/stdio_transport_test.go | 43 ++++++++++++++++++++++++++---- pkg/ide/jetbrains/generic.go | 2 +- 6 files changed, 99 insertions(+), 24 deletions(-) diff --git a/pkg/apple/helper.go b/pkg/apple/helper.go index 01e434a4ee..6a4229ad35 100644 --- a/pkg/apple/helper.go +++ b/pkg/apple/helper.go @@ -202,9 +202,13 @@ func (h *AppleHelper) InspectContainers( return nil, nil } args := append([]string{"inspect"}, ids...) - out, err := h.buildCmd(ctx, args...).Output() + cmd := h.buildCmd(ctx, args...) + out, err := cmd.Output() if err != nil { - return nil, fmt.Errorf("inspect container: %w", command.WrapCommandError(out, err)) + return nil, fmt.Errorf( + "inspect container: %w", + command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + ) } var raw []containerInspect @@ -273,9 +277,14 @@ func (h *AppleHelper) InspectImage( imageName string, tryRemote bool, ) (*config.ImageDetails, error) { - out, err := h.buildCmd(ctx, "image", "inspect", imageName).Output() + cmd := h.buildCmd(ctx, "image", "inspect", imageName) + out, err := cmd.Output() if err != nil { - err = fmt.Errorf("inspect image %s: %w", imageName, command.WrapCommandError(out, err)) + err = fmt.Errorf( + "inspect image %s: %w", + imageName, + command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + ) } else { var raw []imageInspect switch { @@ -309,9 +318,13 @@ func (h *AppleHelper) InspectImage( } func (h *AppleHelper) GetImageTag(ctx context.Context, imageID string) (string, error) { - out, err := h.buildCmd(ctx, "image", "inspect", imageID).Output() + cmd := h.buildCmd(ctx, "image", "inspect", imageID) + out, err := cmd.Output() if err != nil { - return "", fmt.Errorf("inspect image: %w", command.WrapCommandError(out, err)) + return "", fmt.Errorf( + "inspect image: %w", + command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + ) } var raw []imageInspect if err := json.Unmarshal(out, &raw); err != nil { @@ -370,9 +383,13 @@ func (h *AppleHelper) buildCmd(ctx context.Context, args ...string) *exec.Cmd { } func (h *AppleHelper) listContainers(ctx context.Context) ([]containerInspect, error) { - out, err := h.buildCmd(ctx, "list", "--all", "--format", "json").Output() + cmd := h.buildCmd(ctx, "list", "--all", "--format", "json") + out, err := cmd.Output() if err != nil { - return nil, fmt.Errorf("list containers: %w", command.WrapCommandError(out, err)) + return nil, fmt.Errorf( + "list containers: %w", + command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + ) } var raw []containerInspect if err := json.Unmarshal(out, &raw); err != nil { diff --git a/pkg/command/command.go b/pkg/command/command.go index f2daf2ea6a..2f89a0c262 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -20,12 +20,26 @@ func WrapCommandError(stdout []byte, err error) error { } } +func WrapCommandErrorWithEnvironment(stdout []byte, err error, environment []string) error { + if err == nil { + return nil + } + wrapped := &Error{stdout: stdout, err: err} + masked := diagnosticredact.RecordWithEnvironment(environment, wrapped.output()) + wrapped.maskedOutput = &masked + return wrapped +} + type Error struct { - stdout []byte - err error + stdout []byte + err error + maskedOutput *string } func (e *Error) Error() string { + if e.maskedOutput != nil { + return *e.maskedOutput + } return diagnosticredact.EnvironmentRecord(e.output()) } diff --git a/pkg/docker/helper.go b/pkg/docker/helper.go index 9150e3e3cc..e6a29f1199 100644 --- a/pkg/docker/helper.go +++ b/pkg/docker/helper.go @@ -720,9 +720,13 @@ func (r *DockerHelper) GetImageTag(ctx context.Context, imageID string) (string, "{{if .RepoTags}}{{index .RepoTags 0}}{{end}}", } args = append(args, imageID) - out, err := r.buildCmd(ctx, args...).Output() + cmd := r.buildCmd(ctx, args...) + out, err := cmd.Output() if err != nil { - return "", fmt.Errorf("inspect image: %w", command.WrapCommandError(out, err)) + return "", fmt.Errorf( + "inspect image: %w", + command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + ) } repoTag := strings.TrimSpace(string(out)) @@ -812,9 +816,14 @@ func (r *DockerHelper) Inspect( ) error { args := []string{"inspect", "--type", inspectType} args = append(args, ids...) - out, err := r.buildCmd(ctx, args...).Output() + cmd := r.buildCmd(ctx, args...) + out, err := cmd.Output() if err != nil { - return fmt.Errorf("inspect %s: %w", inspectType, command.WrapCommandError(out, err)) + return fmt.Errorf( + "inspect %s: %w", + inspectType, + command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + ) } err = json.Unmarshal(out, obj) @@ -853,9 +862,10 @@ func (r *DockerHelper) FindContainer(ctx context.Context, labels []string) ([]st // This is useful in case the `--filter` doesn't work. func (r *DockerHelper) FindContainerJSON(ctx context.Context, labels []string) ([]string, error) { args := []string{"ps", "-q", "-a"} - out, err := r.buildCmd(ctx, args...).Output() + cmd := r.buildCmd(ctx, args...) + out, err := cmd.Output() if err != nil { - return nil, command.WrapCommandError(out, err) + return nil, command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()) } result := []string{} diff --git a/pkg/docker/image_platform.go b/pkg/docker/image_platform.go index 19b1bbb6b7..2859959026 100644 --- a/pkg/docker/image_platform.go +++ b/pkg/docker/image_platform.go @@ -64,12 +64,13 @@ func (r *DockerHelper) inspectPlatformOrFallback( image, platform string, base *config.ImageDetails, ) (*config.ImageDetails, error) { - out, err := r.buildCmd(ctx, "image", "inspect", "--platform", platform, image).Output() + cmd := r.buildCmd(ctx, "image", "inspect", "--platform", platform, image) + out, err := cmd.Output() if err != nil { if ctx.Err() != nil { return nil, fmt.Errorf("inspect image platform: %w", ctx.Err()) } - wrapped := command.WrapCommandError(nil, err) + wrapped := command.WrapCommandErrorWithEnvironment(nil, err, cmd.Environ()) if command.MatchErrorOutput(wrapped, platformInspectionUnsupported) { // Older engines and compatible runtimes have a single-platform local // image store. The caller still verifies that cached configuration. diff --git a/pkg/docker/stdio_transport_test.go b/pkg/docker/stdio_transport_test.go index cb9b69ce75..fee92e2532 100644 --- a/pkg/docker/stdio_transport_test.go +++ b/pkg/docker/stdio_transport_test.go @@ -23,11 +23,13 @@ import ( ) const ( - helperModeEnv = "DEVSY_TEST_DIAL_STDIO_MODE" - helperEchoMode = "echo" - helperHangMode = "hang" - shortShutdown = 150 * time.Millisecond - boundedInterval = 5 * time.Second + helperModeEnv = "DEVSY_TEST_DIAL_STDIO_MODE" + helperEchoMode = "echo" + helperHangMode = "hang" + shortShutdown = 150 * time.Millisecond + boundedInterval = 5 * time.Second + outputFixtureEnv = "DEVSY_DOCKER_EFFECTIVE_OUTPUT_TOKEN" + stderrFixtureEnv = "DEVSY_DOCKER_EFFECTIVE_STDERR_TOKEN" ) var helperModes = map[string]func(){ @@ -40,6 +42,11 @@ var helperModes = map[string]func(){ _, _ = fmt.Fprintln(os.Stderr, "auth failed for hunter2-token") os.Exit(4) }, + "fail-effective-environment": func() { + _, _ = fmt.Fprintln(os.Stdout, "public output", os.Getenv(outputFixtureEnv)) + _, _ = fmt.Fprintln(os.Stderr, "public diagnostic", os.Getenv(stderrFixtureEnv)) + os.Exit(23) + }, helperHangMode: func() { time.Sleep(time.Minute) }, "lingering": func() { _ = os.Stdout.Close() @@ -64,6 +71,32 @@ func TestMain(m *testing.M) { os.Exit(0) } +func TestDockerInspectMasksEffectiveEnvironment(t *testing.T) { + const privateOutput = "synthetic-docker-output-value" + const privateDiagnostic = "synthetic-docker-diagnostic-value" + executable, err := os.Executable() + require.NoError(t, err) + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=fail-effective-environment", + outputFixtureEnv + "=" + privateOutput, + stderrFixtureEnv + "=" + privateDiagnostic, + }, + } + err = helper.Inspect(t.Context(), []string{"fixture"}, "image", &[]map[string]any{}) + require.Error(t, err) + var exit *exec.ExitError + require.ErrorAs(t, err, &exit) + require.Equal(t, 23, exit.ExitCode()) + for _, text := range []string{"public output", "public diagnostic"} { + require.ErrorContains(t, err, text) + } + for _, text := range []string{privateOutput, privateDiagnostic} { + require.NotContains(t, err.Error(), text) + } +} + // spawnChild starts a long-lived descendant immediately and reports its pid, // standing in for docker starting SSH before it exits or waits. func spawnChild() { diff --git a/pkg/ide/jetbrains/generic.go b/pkg/ide/jetbrains/generic.go index b1a4f749cc..ff86c47cfd 100644 --- a/pkg/ide/jetbrains/generic.go +++ b/pkg/ide/jetbrains/generic.go @@ -222,7 +222,7 @@ func (o *GenericJetBrainsServer) installPlugins( return fmt.Errorf( "failed to install JetBrains plugins %v: %w", plugins, - command.WrapCommandError(out, err), + command.WrapCommandErrorWithEnvironment(out, err, installPluginsCommand.Environ()), ) } From 3cc34264cee7b5bd0a7202ca37a68532e0724a4a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 07:35:41 -0600 Subject: [PATCH 04/11] fix(command): mask diagnostic records before bounding output --- pkg/apple/helper.go | 16 ++-- pkg/command/command.go | 118 +++++++++++++++++++++-- pkg/command/diagnostic_test.go | 17 ++++ pkg/command/output.go | 146 +++++++++++++++++++++++++++++ pkg/docker/helper.go | 12 +-- pkg/docker/image_platform.go | 9 +- pkg/docker/stdio_transport_test.go | 137 +++++++++++++++++++++++++-- 7 files changed, 419 insertions(+), 36 deletions(-) create mode 100644 pkg/command/output.go diff --git a/pkg/apple/helper.go b/pkg/apple/helper.go index 6a4229ad35..3c398f0562 100644 --- a/pkg/apple/helper.go +++ b/pkg/apple/helper.go @@ -203,11 +203,11 @@ func (h *AppleHelper) InspectContainers( } args := append([]string{"inspect"}, ids...) cmd := h.buildCmd(ctx, args...) - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { return nil, fmt.Errorf( "inspect container: %w", - command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + err, ) } @@ -278,12 +278,12 @@ func (h *AppleHelper) InspectImage( tryRemote bool, ) (*config.ImageDetails, error) { cmd := h.buildCmd(ctx, "image", "inspect", imageName) - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { err = fmt.Errorf( "inspect image %s: %w", imageName, - command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + err, ) } else { var raw []imageInspect @@ -319,11 +319,11 @@ func (h *AppleHelper) InspectImage( func (h *AppleHelper) GetImageTag(ctx context.Context, imageID string) (string, error) { cmd := h.buildCmd(ctx, "image", "inspect", imageID) - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { return "", fmt.Errorf( "inspect image: %w", - command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + err, ) } var raw []imageInspect @@ -384,11 +384,11 @@ func (h *AppleHelper) buildCmd(ctx context.Context, args ...string) *exec.Cmd { func (h *AppleHelper) listContainers(ctx context.Context) ([]containerInspect, error) { cmd := h.buildCmd(ctx, "list", "--all", "--format", "json") - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { return nil, fmt.Errorf( "list containers: %w", - command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + err, ) } var raw []containerInspect diff --git a/pkg/command/command.go b/pkg/command/command.go index 2f89a0c262..28bc9035fa 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -1,12 +1,17 @@ package command import ( + "encoding/json" "errors" "fmt" + "io" + "os" "os/exec" "slices" + "strings" "github.com/devsy-org/devsy/internal/diagnosticredact" + "github.com/devsy-org/devsy/logging/redact" ) func WrapCommandError(stdout []byte, err error) error { @@ -25,8 +30,8 @@ func WrapCommandErrorWithEnvironment(stdout []byte, err error, environment []str return nil } wrapped := &Error{stdout: stdout, err: err} - masked := diagnosticredact.RecordWithEnvironment(environment, wrapped.output()) - wrapped.maskedOutput = &masked + redactor := redact.New(redact.Options{Environment: environment}) + wrapped.setDisplay(redactor, maskJSONRecords(redactor, wrapped.stderr())) return wrapped } @@ -40,7 +45,8 @@ func (e *Error) Error() string { if e.maskedOutput != nil { return *e.maskedOutput } - return diagnosticredact.EnvironmentRecord(e.output()) + redactor := redact.New(redact.Options{Environment: os.Environ()}) + return e.display(redactor, maskJSONRecords(redactor, e.stderr())) } func (e *Error) Unwrap() error { @@ -67,17 +73,111 @@ func matchOriginalCommandOutput(err error, match func(string) bool) bool { } func (e *Error) output() string { + return commandErrorOutput(string(e.stdout), e.stderr(), e.err.Error()) +} + +func (e *Error) stderr() string { + if exit, ok := errors.AsType[*exec.ExitError](e.err); ok { + return string(exit.Stderr) + } + return "" +} + +func (e *Error) setDisplay(redactor *redact.Redactor, stderr string) { + masked := e.display(redactor, stderr) + e.maskedOutput = &masked +} + +func (e *Error) display(redactor *redact.Redactor, stderr string) string { + stdout := maskJSONRecords(redactor, string(e.stdout)) + cause := maskJSONRecords(redactor, e.err.Error()) + return diagnosticredact.Record(redactor, commandErrorOutput(stdout, stderr, cause)) +} + +func commandErrorOutput(stdout, stderr, cause string) string { message := "" - if len(e.stdout) > 0 { - message += string(e.stdout) + "\n" + if stdout != "" { + message += stdout + "\n" + } + if stderr != "" { + message += stderr + "\n" + } + return message + cause +} + +func maskJSONRecords(redactor *redact.Redactor, value string) string { + if json.Valid([]byte(value)) { + return diagnosticredact.Record(redactor, value) + } + var output strings.Builder + for value != "" { + if looksLikeStructuredJSON(value) { + value = value[maskStructuredRecord(redactor, value, &output):] + continue + } + line, rest, newline := strings.Cut(value, "\n") + output.WriteString(maskJSONLine(redactor, line)) + if newline { + output.WriteByte('\n') + } + value = rest + } + return output.String() +} + +func looksLikeStructuredJSON(value string) bool { + value = strings.TrimLeft(value, " \t\r") + if value == "" { + return false + } + rest := strings.TrimLeft(value[1:], " \t\r\n") + switch value[0] { + case '{': + return rest == "" || rest[0] == '"' || rest[0] == '}' + case '[': + return looksLikeJSONList(rest) + default: + return false } +} - var exitError *exec.ExitError - if errors.As(e.err, &exitError) && len(exitError.Stderr) > 0 { - message += string(exitError.Stderr) + "\n" +func looksLikeJSONList(rest string) bool { + if rest == "" || strings.ContainsRune(`{[]"`, rune(rest[0])) { + return true } + if !strings.ContainsRune(`-0123456789tfn`, rune(rest[0])) { + return false + } + decoder := json.NewDecoder(strings.NewReader(rest)) + var first json.RawMessage + if err := decoder.Decode(&first); err != nil { + return errors.Is(err, io.ErrUnexpectedEOF) + } + tail := strings.TrimLeft(rest[int(decoder.InputOffset()):], " \t\r\n") + return tail == "" || tail[0] == ',' || tail[0] == ']' +} - return message + e.err.Error() +func maskStructuredRecord(redactor *redact.Redactor, value string, output *strings.Builder) int { + decoder := json.NewDecoder(strings.NewReader(value)) + var record json.RawMessage + if err := decoder.Decode(&record); err != nil { + output.WriteString(redactor.Mask()) + return len(value) + } + output.WriteString(diagnosticredact.Record(redactor, string(record))) + return int(decoder.InputOffset()) +} + +func maskJSONLine(redactor *redact.Redactor, line string) string { + if json.Valid([]byte(line)) { + body, cr := strings.CutSuffix(line, "\r") + masked := diagnosticredact.Record(redactor, body) + if cr { + return masked + "\r" + } + return masked + } + return line } func formatTaskkillDiagnostic(pid int, err error, output []byte) error { diff --git a/pkg/command/diagnostic_test.go b/pkg/command/diagnostic_test.go index c564b295aa..5122fe972e 100644 --- a/pkg/command/diagnostic_test.go +++ b/pkg/command/diagnostic_test.go @@ -1,6 +1,7 @@ package command import ( + "encoding/json" "errors" "go/ast" "go/parser" @@ -51,6 +52,22 @@ func TestCommandErrorMasksFoldedOutput(t *testing.T) { assert.Contains(t, err.Error(), "public stderr") } +func TestCommandErrorMasksEscapedJSONValues(t *testing.T) { + const fixture = `synthetic"json-value` + data, err := json.Marshal(map[string]string{"message": fixture, "detail": "public JSON detail"}) + require.NoError(t, err) + cause := &exec.ExitError{Stderr: data} + wrapped := WrapCommandErrorWithEnvironment( + data, + cause, + []string{"DEVSY_FIXTURE_TOKEN=" + fixture}, + ) + require.ErrorIs(t, wrapped, cause) + assert.NotContains(t, wrapped.Error(), "json-value") + assert.Equal(t, 2, strings.Count(wrapped.Error(), `"message":"***"`)) + assert.Equal(t, 2, strings.Count(wrapped.Error(), "public JSON detail")) +} + func TestWindowsCapturedOutputUsesDiagnosticFormatter(t *testing.T) { file, err := parser.ParseFile( token.NewFileSet(), diff --git a/pkg/command/output.go b/pkg/command/output.go new file mode 100644 index 0000000000..6e741841c3 --- /dev/null +++ b/pkg/command/output.go @@ -0,0 +1,146 @@ +package command + +import ( + "errors" + "fmt" + "os/exec" + + "github.com/devsy-org/devsy/logging/redact" +) + +const stderrCaptureSize = 32 << 10 + +const ( + IncludeFailureStdout = true + ExcludeFailureStdout = false +) + +func Output(cmd *exec.Cmd, includeStdout bool) ([]byte, error) { + redactor := redact.New(redact.Options{Environment: cmd.Environ()}) + var capture *stderrCapture + if cmd.Stderr == nil { + capture = &stderrCapture{stream: redact.NewDiagnosticStream(redactor)} + cmd.Stderr = capture + } + stdout, err := cmd.Output() + if err == nil { + return stdout, nil + } + wrapped := &Error{err: err} + if includeStdout { + wrapped.stdout = stdout + } + stderr := maskJSONRecords(redactor, wrapped.stderr()) + if capture != nil { + stderr = capture.finish(err, redactor) + } + wrapped.setDisplay(redactor, stderr) + return stdout, wrapped +} + +type stderrCapture struct { + stream *redact.Stream + raw boundedCapture + masked boundedCapture + structured structuredLineStart +} + +func (c *stderrCapture) Write(p []byte) (int, error) { + text := string(p) + c.raw.append(text) + c.masked.append(c.stream.RedactChunk(text)) + c.structured.scan(text) + return len(p), nil +} + +func (c *stderrCapture) finish(cause error, redactor *redact.Redactor) string { + raw := c.raw.text() + if exit, ok := errors.AsType[*exec.ExitError](cause); ok { + exit.Stderr = []byte(raw) + } + if c.raw.omitted == 0 { + return maskJSONRecords(redactor, raw) + } + if c.structured.found || c.structured.isCandidate() { + return redactor.Mask() + } + c.masked.append(c.stream.Flush()) + return c.masked.text() +} + +type structuredLineStart struct { + prefix [128]byte + length int + found bool + skip bool +} + +func (s *structuredLineStart) scan(text string) { + for index := range len(text) { + if s.found { + return + } + value := text[index] + if value == '\r' || value == '\n' { + s.found = s.isCandidate() + s.length = 0 + s.skip = false + continue + } + if !s.skip { + s.take(value) + } + } +} + +func (s *structuredLineStart) take(value byte) { + if s.length == 0 { + switch value { + case ' ', '\t': + return + case '[', '{': + default: + s.skip = true + return + } + } + s.prefix[s.length] = value + s.length++ + if s.length == len(s.prefix) { + s.found = s.isCandidate() + s.skip = true + } +} + +func (s *structuredLineStart) isCandidate() bool { + return s.length > 0 && !s.skip && looksLikeStructuredJSON(string(s.prefix[:s.length])) +} + +type boundedCapture struct { + prefix []byte + tail []byte + omitted int64 +} + +func (c *boundedCapture) append(text string) { + first := min(stderrCaptureSize-len(c.prefix), len(text)) + c.prefix = append(c.prefix, text[:first]...) + text = text[first:] + if len(text) >= stderrCaptureSize { + c.omitted += int64(len(c.tail) + len(text) - stderrCaptureSize) + c.tail = append(c.tail[:0], text[len(text)-stderrCaptureSize:]...) + return + } + if excess := len(c.tail) + len(text) - stderrCaptureSize; excess > 0 { + c.omitted += int64(excess) + c.tail = c.tail[:copy(c.tail, c.tail[excess:])] + } + c.tail = append(c.tail, text...) +} + +func (c *boundedCapture) text() string { + if c.omitted == 0 { + return string(c.prefix) + string(c.tail) + } + return fmt.Sprintf("%s\n... omitting %d bytes ...\n%s", c.prefix, c.omitted, c.tail) +} diff --git a/pkg/docker/helper.go b/pkg/docker/helper.go index e6a29f1199..d4b204d2fd 100644 --- a/pkg/docker/helper.go +++ b/pkg/docker/helper.go @@ -721,11 +721,11 @@ func (r *DockerHelper) GetImageTag(ctx context.Context, imageID string) (string, } args = append(args, imageID) cmd := r.buildCmd(ctx, args...) - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { return "", fmt.Errorf( "inspect image: %w", - command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + err, ) } @@ -817,12 +817,12 @@ func (r *DockerHelper) Inspect( args := []string{"inspect", "--type", inspectType} args = append(args, ids...) cmd := r.buildCmd(ctx, args...) - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { return fmt.Errorf( "inspect %s: %w", inspectType, - command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()), + err, ) } @@ -863,9 +863,9 @@ func (r *DockerHelper) FindContainer(ctx context.Context, labels []string) ([]st func (r *DockerHelper) FindContainerJSON(ctx context.Context, labels []string) ([]string, error) { args := []string{"ps", "-q", "-a"} cmd := r.buildCmd(ctx, args...) - out, err := cmd.Output() + out, err := command.Output(cmd, command.IncludeFailureStdout) if err != nil { - return nil, command.WrapCommandErrorWithEnvironment(out, err, cmd.Environ()) + return nil, err } result := []string{} diff --git a/pkg/docker/image_platform.go b/pkg/docker/image_platform.go index 2859959026..bd4d0326c4 100644 --- a/pkg/docker/image_platform.go +++ b/pkg/docker/image_platform.go @@ -65,21 +65,20 @@ func (r *DockerHelper) inspectPlatformOrFallback( base *config.ImageDetails, ) (*config.ImageDetails, error) { cmd := r.buildCmd(ctx, "image", "inspect", "--platform", platform, image) - out, err := cmd.Output() + out, err := command.Output(cmd, command.ExcludeFailureStdout) if err != nil { if ctx.Err() != nil { return nil, fmt.Errorf("inspect image platform: %w", ctx.Err()) } - wrapped := command.WrapCommandErrorWithEnvironment(nil, err, cmd.Environ()) - if command.MatchErrorOutput(wrapped, platformInspectionUnsupported) { + if command.MatchErrorOutput(err, platformInspectionUnsupported) { // Older engines and compatible runtimes have a single-platform local // image store. The caller still verifies that cached configuration. return base, nil } - if command.MatchErrorOutput(wrapped, platformInspectionMiss) { + if command.MatchErrorOutput(err, platformInspectionMiss) { return nil, fmt.Errorf("%s: %w", image, ErrImageNotFound) } - return nil, classifyImageError(image, fmt.Errorf("inspect image platform: %w", wrapped)) + return nil, classifyImageError(image, fmt.Errorf("inspect image platform: %w", err)) } var details []*config.ImageDetails if err := json.Unmarshal(out, &details); err != nil { diff --git a/pkg/docker/stdio_transport_test.go b/pkg/docker/stdio_transport_test.go index fee92e2532..e686a93c6d 100644 --- a/pkg/docker/stdio_transport_test.go +++ b/pkg/docker/stdio_transport_test.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "runtime" + "strconv" "strings" "sync" "testing" @@ -23,13 +24,17 @@ import ( ) const ( - helperModeEnv = "DEVSY_TEST_DIAL_STDIO_MODE" - helperEchoMode = "echo" - helperHangMode = "hang" - shortShutdown = 150 * time.Millisecond - boundedInterval = 5 * time.Second - outputFixtureEnv = "DEVSY_DOCKER_EFFECTIVE_OUTPUT_TOKEN" - stderrFixtureEnv = "DEVSY_DOCKER_EFFECTIVE_STDERR_TOKEN" + helperModeEnv = "DEVSY_TEST_DIAL_STDIO_MODE" + helperEchoMode = "echo" + helperHangMode = "hang" + shortShutdown = 150 * time.Millisecond + boundedInterval = 5 * time.Second + outputFixtureEnv = "DEVSY_DOCKER_EFFECTIVE_OUTPUT_TOKEN" + stderrFixtureEnv = "DEVSY_DOCKER_EFFECTIVE_STDERR_TOKEN" + jsonFixtureMode = "fail-json-stderr" + jsonFixtureSizeEnv = "DEVSY_DOCKER_JSON_FIXTURE_SIZE" + bracketFixtureMode = "fail-bracketed-stderr" + inspectFixture = "fixture" ) var helperModes = map[string]func(){ @@ -47,6 +52,29 @@ var helperModes = map[string]func(){ _, _ = fmt.Fprintln(os.Stderr, "public diagnostic", os.Getenv(stderrFixtureEnv)) os.Exit(23) }, + "fail-long-stderr": func() { + value := os.Getenv(stderrFixtureEnv) + _, _ = fmt.Fprint(os.Stderr, strings.Repeat("a", (32<<10)-len(value)+1), value) + _, _ = fmt.Fprint(os.Stderr, strings.Repeat("b", 64<<10), "\npublic diagnostic tail\n") + os.Exit(23) + }, + jsonFixtureMode: func() { + size, _ := strconv.Atoi(os.Getenv(jsonFixtureSizeEnv)) + encoder := json.NewEncoder(os.Stderr) + encoder.SetIndent("", " ") + _ = encoder.Encode(map[string]string{ + "detail": "public JSON diagnostic", + "filler": strings.Repeat("x", size), + "message": os.Getenv(stderrFixtureEnv), + }) + os.Exit(23) + }, + bracketFixtureMode: func() { + size, _ := strconv.Atoi(os.Getenv(jsonFixtureSizeEnv)) + _, _ = fmt.Fprintln(os.Stderr, "[task] operation blocked\n[1/3] checking runtime") + _, _ = fmt.Fprintln(os.Stderr, strings.Repeat("a", size), "public diagnostic tail") + os.Exit(23) + }, helperHangMode: func() { time.Sleep(time.Minute) }, "lingering": func() { _ = os.Stdout.Close() @@ -84,7 +112,7 @@ func TestDockerInspectMasksEffectiveEnvironment(t *testing.T) { stderrFixtureEnv + "=" + privateDiagnostic, }, } - err = helper.Inspect(t.Context(), []string{"fixture"}, "image", &[]map[string]any{}) + err = helper.Inspect(t.Context(), []string{inspectFixture}, "image", &[]map[string]any{}) require.Error(t, err) var exit *exec.ExitError require.ErrorAs(t, err, &exit) @@ -97,6 +125,99 @@ func TestDockerInspectMasksEffectiveEnvironment(t *testing.T) { } } +func TestDockerInspectMasksBeforeBoundingStderr(t *testing.T) { + const value = "synthetic-stderr-boundary-value" + executable, err := os.Executable() + require.NoError(t, err) + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=fail-long-stderr", + stderrFixtureEnv + "=" + value, + }, + } + err = helper.Inspect(t.Context(), []string{inspectFixture}, "image", &[]map[string]any{}) + require.Error(t, err) + var exit *exec.ExitError + require.ErrorAs(t, err, &exit) + require.Equal(t, 23, exit.ExitCode()) + message := err.Error() + require.False( + t, + strings.Contains(message, "synthetic-stderr-boundary"), + "partial value reached the display", + ) + require.True(t, strings.Contains(message, "public diagnostic tail"), "public tail missing") + require.Less(t, len(message), 66<<10) +} + +func TestDockerInspectMasksPrettyJSONStderr(t *testing.T) { + const value = `synthetic"pretty-json-value` + executable, err := os.Executable() + require.NoError(t, err) + for _, tc := range []struct { + name string + size int + }{ + {"complete", 0}, + {"oversized", 96 << 10}, + } { + t.Run(tc.name, func(t *testing.T) { + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=" + jsonFixtureMode, + jsonFixtureSizeEnv + "=" + strconv.Itoa(tc.size), + stderrFixtureEnv + "=" + value, + }, + } + err := helper.Inspect( + t.Context(), + []string{inspectFixture}, + "image", + &[]map[string]any{}, + ) + require.Error(t, err) + message := err.Error() + require.False( + t, + strings.Contains(message, "pretty-json-value"), + "escaped value reached the display", + ) + require.Contains(t, message, "***") + if tc.size == 0 { + require.Contains(t, message, "public JSON diagnostic") + } + }) + } +} + +func TestDockerInspectKeepsBracketedDiagnostics(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + for _, size := range []int{0, 96 << 10} { + t.Run(strconv.Itoa(size), func(t *testing.T) { + helper := &DockerHelper{ + DockerCommand: executable, + Environment: []string{ + helperModeEnv + "=" + bracketFixtureMode, + jsonFixtureSizeEnv + "=" + strconv.Itoa(size), + }, + } + err := helper.Inspect( + t.Context(), + []string{inspectFixture}, + "image", + &[]map[string]any{}, + ) + require.Error(t, err) + for _, detail := range []string{"[task] operation blocked", "[1/3] checking runtime", "public diagnostic tail"} { + require.True(t, strings.Contains(err.Error(), detail), "missing detail %q", detail) + } + }) + } +} + // spawnChild starts a long-lived descendant immediately and reports its pid, // standing in for docker starting SSH before it exits or waits. func spawnChild() { From d42372fe397362620ad9c6de12c38df914205081 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 13:42:37 -0600 Subject: [PATCH 05/11] fix(logging): mask records and carried stderr in tool diagnostics --- internal/diagnosticredact/records.go | 95 +++++++++++++++++++++++ internal/diagnosticredact/redact.go | 3 +- internal/diagnosticredact/redact_test.go | 21 +++++ pkg/command/command.go | 97 ++---------------------- pkg/command/output.go | 8 +- pkg/git/lfs.go | 14 ++-- pkg/git/lfs_test.go | 25 ++++++ 7 files changed, 162 insertions(+), 101 deletions(-) create mode 100644 internal/diagnosticredact/records.go diff --git a/internal/diagnosticredact/records.go b/internal/diagnosticredact/records.go new file mode 100644 index 0000000000..e4edc4b9a4 --- /dev/null +++ b/internal/diagnosticredact/records.go @@ -0,0 +1,95 @@ +package diagnosticredact + +import ( + "encoding/json" + "errors" + "io" + "strings" + + "github.com/devsy-org/devsy/logging/redact" +) + +func Records(redactor *redact.Redactor, value string) string { + if json.Valid([]byte(value)) { + return Record(redactor, value) + } + var output strings.Builder + for value != "" { + if LooksLikeStructuredJSON(value) { + if consumed := maskStructuredRecord(redactor, value, &output); consumed > 0 { + value = value[consumed:] + continue + } + } + record, separator, rest := splitDiagnosticRecord(value) + if json.Valid([]byte(record)) { + record = Record(redactor, record) + } + output.WriteString(record) + output.WriteString(separator) + value = rest + } + return output.String() +} + +func splitDiagnosticRecord(value string) (record, separator, rest string) { + end := strings.IndexAny(value, "\r\n") + if end < 0 { + return value, "", "" + } + next := end + 1 + if strings.HasPrefix(value[end:], "\r\n") { + next++ + } + return value[:end], value[end:next], value[next:] +} + +func LooksLikeStructuredJSON(value string) bool { + value = strings.TrimLeft(value, " \t") + if value == "" { + return false + } + rest := strings.TrimLeft(value[1:], " \t\r\n") + switch value[0] { + case '{': + return rest == "" || rest[0] == '"' || rest[0] == '}' + case '[': + return looksLikeJSONList(rest) + case '"': + return true + default: + return false + } +} + +func looksLikeJSONList(rest string) bool { + if rest == "" || strings.ContainsRune(`{[]"`, rune(rest[0])) { + return true + } + if !strings.ContainsRune(`-0123456789tfn`, rune(rest[0])) { + return false + } + decoder := json.NewDecoder(strings.NewReader(rest)) + var first json.RawMessage + if err := decoder.Decode(&first); err != nil { + return errors.Is(err, io.ErrUnexpectedEOF) + } + tail := strings.TrimLeft(rest[int(decoder.InputOffset()):], " \t\r\n") + return tail == "" || tail[0] == ',' || tail[0] == ']' +} + +func maskStructuredRecord(redactor *redact.Redactor, value string, output *strings.Builder) int { + decoder := json.NewDecoder(strings.NewReader(value)) + var record json.RawMessage + if err := decoder.Decode(&record); err != nil { + if strings.HasPrefix(strings.TrimLeft(value, " \t"), `"`) { + return 0 + } + output.WriteString(redactor.Mask()) + return len(value) + } + end := int(decoder.InputOffset()) + output.WriteString(value[:end-len(record)]) + output.WriteString(Record(redactor, string(record))) + return end +} diff --git a/internal/diagnosticredact/redact.go b/internal/diagnosticredact/redact.go index cc303d443f..c67268102a 100644 --- a/internal/diagnosticredact/redact.go +++ b/internal/diagnosticredact/redact.go @@ -14,7 +14,8 @@ func EnvironmentRecord(value string) string { } func RecordWithEnvironment(environment []string, value string) string { - return Record(redact.New(redact.Options{Environment: environment}), value) + redactor := redact.New(redact.Options{Environment: environment}) + return Record(redactor, Records(redactor, value)) } func Record(redactor *redact.Redactor, value string) string { diff --git a/internal/diagnosticredact/redact_test.go b/internal/diagnosticredact/redact_test.go index 4854ae051a..721ef4ca07 100644 --- a/internal/diagnosticredact/redact_test.go +++ b/internal/diagnosticredact/redact_test.go @@ -46,3 +46,24 @@ func TestRecordKeepsOrdinaryPrefixAtTerminalFold(t *testing.T) { assert.NotContains(t, output, canary) assert.Contains(t, output, detail) } + +func TestEnvironmentRecordsMaskJSONRecordsAfterPlainLines(t *testing.T) { + const canary = "synthetic-record-bearer" + for name, separator := range map[string]string{"LF": "\n", "CR": "\r", "CRLF": "\r\n"} { + t.Run(name, func(t *testing.T) { + input := "public diagnostic" + separator + + `{"Authorization":"Bearer ` + canary + `"}` + separator + "public tail" + for label, output := range map[string]string{ + "EnvironmentRecord": EnvironmentRecord(input), + "RecordWithEnvironment": RecordWithEnvironment(nil, input), + "RecordsThenRecord": Record(redact.FromEntries(nil), Records(redact.FromEntries(nil), input)), + "RecordsWithOnlyPlainEOL": Records(redact.FromEntries(nil), "plain"+separator+"text"), + } { + assert.NotContains(t, output, canary, label) + } + output := RecordWithEnvironment(nil, input) + assert.Contains(t, output, "public diagnostic"+separator) + assert.Contains(t, output, separator+"public tail") + }) + } +} diff --git a/pkg/command/command.go b/pkg/command/command.go index bb3ad15e2b..8e7a7f5bd9 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -1,10 +1,8 @@ package command import ( - "encoding/json" "errors" "fmt" - "io" "os" "os/exec" "slices" @@ -31,7 +29,7 @@ func WrapCommandErrorWithEnvironment(stdout []byte, err error, environment []str } wrapped := &Error{stdout: stdout, err: err} redactor := redact.New(redact.Options{Environment: environment}) - wrapped.setDisplay(redactor, maskJSONRecords(redactor, wrapped.stderr())) + wrapped.setDisplay(redactor, diagnosticredact.Records(redactor, wrapped.stderr())) return wrapped } @@ -41,7 +39,7 @@ func WrapCombinedOutputError(output string, err error, environment []string) err } redactor := redact.New(redact.Options{Environment: environment}) masked := strings.TrimSpace( - diagnosticredact.Record(redactor, maskJSONRecords(redactor, output)), + diagnosticredact.Record(redactor, diagnosticredact.Records(redactor, output)), ) if masked == "" { return err @@ -69,7 +67,7 @@ func (e *Error) Error() string { return e.maskedCapture + ": " + e.err.Error() } redactor := redact.New(redact.Options{Environment: os.Environ()}) - return e.display(redactor, maskJSONRecords(redactor, e.stderr())) + return e.display(redactor, diagnosticredact.Records(redactor, e.stderr())) } func (e *Error) Unwrap() error { @@ -115,8 +113,8 @@ func (e *Error) setDisplay(redactor *redact.Redactor, stderr string) { } func (e *Error) display(redactor *redact.Redactor, stderr string) string { - stdout := maskJSONRecords(redactor, string(e.stdout)) - cause := maskJSONRecords(redactor, e.err.Error()) + stdout := diagnosticredact.Records(redactor, string(e.stdout)) + cause := diagnosticredact.Records(redactor, e.err.Error()) return diagnosticredact.Record(redactor, commandErrorOutput(stdout, stderr, cause)) } @@ -131,91 +129,6 @@ func commandErrorOutput(stdout, stderr, cause string) string { return message + cause } -func maskJSONRecords(redactor *redact.Redactor, value string) string { - if json.Valid([]byte(value)) { - return diagnosticredact.Record(redactor, value) - } - var output strings.Builder - for value != "" { - if looksLikeStructuredJSON(value) { - if consumed := maskStructuredRecord(redactor, value, &output); consumed > 0 { - value = value[consumed:] - continue - } - } - record, separator, rest := splitDiagnosticRecord(value) - if json.Valid([]byte(record)) { - record = diagnosticredact.Record(redactor, record) - } - output.WriteString(record) - output.WriteString(separator) - value = rest - } - return output.String() -} - -func splitDiagnosticRecord(value string) (record, separator, rest string) { - end := strings.IndexAny(value, "\r\n") - if end < 0 { - return value, "", "" - } - next := end + 1 - if strings.HasPrefix(value[end:], "\r\n") { - next++ - } - return value[:end], value[end:next], value[next:] -} - -func looksLikeStructuredJSON(value string) bool { - value = strings.TrimLeft(value, " \t") - if value == "" { - return false - } - rest := strings.TrimLeft(value[1:], " \t\r\n") - switch value[0] { - case '{': - return rest == "" || rest[0] == '"' || rest[0] == '}' - case '[': - return looksLikeJSONList(rest) - case '"': - return true - default: - return false - } -} - -func looksLikeJSONList(rest string) bool { - if rest == "" || strings.ContainsRune(`{[]"`, rune(rest[0])) { - return true - } - if !strings.ContainsRune(`-0123456789tfn`, rune(rest[0])) { - return false - } - decoder := json.NewDecoder(strings.NewReader(rest)) - var first json.RawMessage - if err := decoder.Decode(&first); err != nil { - return errors.Is(err, io.ErrUnexpectedEOF) - } - tail := strings.TrimLeft(rest[int(decoder.InputOffset()):], " \t\r\n") - return tail == "" || tail[0] == ',' || tail[0] == ']' -} - -func maskStructuredRecord(redactor *redact.Redactor, value string, output *strings.Builder) int { - decoder := json.NewDecoder(strings.NewReader(value)) - var record json.RawMessage - if err := decoder.Decode(&record); err != nil { - if strings.HasPrefix(strings.TrimLeft(value, " \t"), `"`) { - return 0 - } - output.WriteString(redactor.Mask()) - return len(value) - } - end := int(decoder.InputOffset()) - output.WriteString(value[:end-len(record)]) - output.WriteString(diagnosticredact.Record(redactor, string(record))) - return end -} - func formatTaskkillDiagnostic(pid int, err error, output []byte) error { detail := truncateOutput(diagnosticredact.EnvironmentRecord(string(output))) return fmt.Errorf("taskkill /PID %d /T /F: %w: %s", pid, err, detail) diff --git a/pkg/command/output.go b/pkg/command/output.go index 0e373e247a..ca1dfc4824 100644 --- a/pkg/command/output.go +++ b/pkg/command/output.go @@ -6,6 +6,7 @@ import ( "os/exec" "unicode/utf8" + "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/logging/redact" ) @@ -31,7 +32,7 @@ func Output(cmd *exec.Cmd, includeStdout bool) ([]byte, error) { if includeStdout { wrapped.stdout = stdout } - stderr := maskJSONRecords(redactor, wrapped.stderr()) + stderr := diagnosticredact.Records(redactor, wrapped.stderr()) if capture != nil { stderr = capture.finish(err, redactor) } @@ -60,7 +61,7 @@ func (c *stderrCapture) finish(cause error, redactor *redact.Redactor) string { exit.Stderr = []byte(raw) } if c.raw.omitted == 0 { - return maskJSONRecords(redactor, raw) + return diagnosticredact.Records(redactor, raw) } if c.structured.found || c.structured.isCandidate() { return redactor.Mask() @@ -114,7 +115,8 @@ func (s *structuredLineStart) take(value byte) { } func (s *structuredLineStart) isCandidate() bool { - return s.length > 0 && !s.skip && looksLikeStructuredJSON(string(s.prefix[:s.length])) + return s.length > 0 && !s.skip && + diagnosticredact.LooksLikeStructuredJSON(string(s.prefix[:s.length])) } type boundedCapture struct { diff --git a/pkg/git/lfs.go b/pkg/git/lfs.go index a710c1572c..c2351c8807 100644 --- a/pkg/git/lfs.go +++ b/pkg/git/lfs.go @@ -3,13 +3,13 @@ package git import ( "bytes" "context" - "fmt" + "errors" "io/fs" "os" "path/filepath" + "slices" "strings" - "github.com/devsy-org/devsy/internal/diagnosticredact" "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/log" ) @@ -96,10 +96,14 @@ func ensureLFSBinary(ctx context.Context, allowInstall bool) bool { func (r *Repo) lfs(ctx context.Context, args ...string) error { res, err := r.run(ctx, append([]string{"lfs"}, args...)...) if err != nil { - return fmt.Errorf( - "%w: %s", + output := slices.Clone(res.Stdout) + if _, carriesStderr := errors.AsType[*CommandError](err); !carriesStderr { + output = append(output, res.Stderr...) + } + return command.WrapCommandErrorWithEnvironment( + output, err, - diagnosticredact.EnvironmentRecord(string(res.Stdout)+string(res.Stderr)), + append(os.Environ(), r.env...), ) } return nil diff --git a/pkg/git/lfs_test.go b/pkg/git/lfs_test.go index b949613646..1dfaa33c15 100644 --- a/pkg/git/lfs_test.go +++ b/pkg/git/lfs_test.go @@ -38,6 +38,31 @@ func TestLFSFailureMasksFoldedOutputAcrossStreams(t *testing.T) { assert.NilError(t, At(t.TempDir(), WithRunner(lfsSplitRunner{})).lfs(t.Context(), "pull")) } +type lfsCommandErrorRunner struct{ cause error } + +func (r lfsCommandErrorRunner) Run(context.Context, RunOptions) (RunResult, error) { + stderr := "Authorization:\n Bearer " + lfsFoldedFixtureValue + "\n\n" + lfsDiagnosticDetail + return RunResult{Stderr: []byte(stderr)}, &CommandError{ + Args: []string{"lfs", "pull"}, + ExitCode: 2, + Stderr: stderr, + Err: r.cause, + } +} + +func TestLFSFailureMasksStderrCarriedByTheCommandError(t *testing.T) { + cause := errors.New("fixture LFS process failure") + repo := At(t.TempDir(), WithRunner(lfsCommandErrorRunner{cause: cause})) + err := repo.lfs(t.Context(), "pull") + assert.Assert(t, errors.Is(err, cause)) + var commandErr *CommandError + assert.Assert(t, errors.As(err, &commandErr)) + assert.Equal(t, 2, commandErr.ExitCode) + assert.Equal(t, 1, strings.Count(err.Error(), lfsDiagnosticDetail)) + assert.Assert(t, cmp.Contains(err.Error(), lfsDiagnosticDetail)) + assert.Assert(t, !strings.Contains(err.Error(), lfsFoldedFixtureValue)) +} + func TestLFSModeResolution(t *testing.T) { cases := []struct { name string From dff010ec09a5abd8d934bd95657f671936abdf7f Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 15:43:02 -0600 Subject: [PATCH 06/11] fix(logging): make whole-buffer redaction record-aware --- internal/diagnosticredact/records.go | 9 ++++++--- internal/diagnosticredact/redact.go | 13 +++++++++++-- internal/diagnosticredact/redact_test.go | 12 ++++++++++++ pkg/apple/helper_test.go | 11 +++++++++++ pkg/command/command.go | 2 +- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/internal/diagnosticredact/records.go b/internal/diagnosticredact/records.go index e4edc4b9a4..b045f9af30 100644 --- a/internal/diagnosticredact/records.go +++ b/internal/diagnosticredact/records.go @@ -10,8 +10,11 @@ import ( ) func Records(redactor *redact.Redactor, value string) string { + if redactor == nil { + return value + } if json.Valid([]byte(value)) { - return Record(redactor, value) + return wholeRecord(redactor, value) } var output strings.Builder for value != "" { @@ -23,7 +26,7 @@ func Records(redactor *redact.Redactor, value string) string { } record, separator, rest := splitDiagnosticRecord(value) if json.Valid([]byte(record)) { - record = Record(redactor, record) + record = wholeRecord(redactor, record) } output.WriteString(record) output.WriteString(separator) @@ -90,6 +93,6 @@ func maskStructuredRecord(redactor *redact.Redactor, value string, output *strin } end := int(decoder.InputOffset()) output.WriteString(value[:end-len(record)]) - output.WriteString(Record(redactor, string(record))) + output.WriteString(wholeRecord(redactor, string(record))) return end } diff --git a/internal/diagnosticredact/redact.go b/internal/diagnosticredact/redact.go index c67268102a..45b48c7cd3 100644 --- a/internal/diagnosticredact/redact.go +++ b/internal/diagnosticredact/redact.go @@ -14,11 +14,20 @@ func EnvironmentRecord(value string) string { } func RecordWithEnvironment(environment []string, value string) string { - redactor := redact.New(redact.Options{Environment: environment}) - return Record(redactor, Records(redactor, value)) + return Record(redact.New(redact.Options{Environment: environment}), value) } func Record(redactor *redact.Redactor, value string) string { + if redactor == nil { + return value + } + if json.Valid([]byte(value)) { + return wholeRecord(redactor, value) + } + return wholeRecord(redactor, Records(redactor, value)) +} + +func wholeRecord(redactor *redact.Redactor, value string) string { if redactor == nil { return value } diff --git a/internal/diagnosticredact/redact_test.go b/internal/diagnosticredact/redact_test.go index 721ef4ca07..f41ff1709a 100644 --- a/internal/diagnosticredact/redact_test.go +++ b/internal/diagnosticredact/redact_test.go @@ -67,3 +67,15 @@ func TestEnvironmentRecordsMaskJSONRecordsAfterPlainLines(t *testing.T) { }) } } + +func TestRecordMasksJSONCredentialRecordsBeforePublicText(t *testing.T) { + const canary = "synthetic-cookie-record" + for name, separator := range map[string]string{"LF": "\n", "CR": "\r", "CRLF": "\r\n"} { + t.Run(name, func(t *testing.T) { + input := `{"Cookie":"` + canary + `"}` + separator + "public text" + output := Record(redact.FromEntries(nil), input) + assert.NotContains(t, output, canary) + assert.Contains(t, output, separator+"public text") + }) + } +} diff --git a/pkg/apple/helper_test.go b/pkg/apple/helper_test.go index 08f2914f95..7077555e14 100644 --- a/pkg/apple/helper_test.go +++ b/pkg/apple/helper_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "testing" "time" @@ -196,3 +197,13 @@ func TestEnsureBuilderRunning(t *testing.T) { t.Errorf("already-running must be tolerated, got %v", err) } } + +func TestAppleCommandDiagnosticMasksJSONCredentialRecordBeforePublicText(t *testing.T) { + const canary = "synthetic-apple-cookie" + cmd := exec.Command("container", "list") + out := []byte(`{"Cookie":"` + canary + `"}` + "\n" + "public apple detail") + got := appleCommandDiagnostic(cmd, out) + if strings.Contains(got, canary) || !strings.Contains(got, "public apple detail") { + t.Fatalf("unexpected diagnostic %q", got) + } +} diff --git a/pkg/command/command.go b/pkg/command/command.go index 8e7a7f5bd9..c9feee6183 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -39,7 +39,7 @@ func WrapCombinedOutputError(output string, err error, environment []string) err } redactor := redact.New(redact.Options{Environment: environment}) masked := strings.TrimSpace( - diagnosticredact.Record(redactor, diagnosticredact.Records(redactor, output)), + diagnosticredact.Record(redactor, output), ) if masked == "" { return err From 4c78ab268cde00f39ffa72ab3daad5318f4f7b8e Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 16:07:41 -0600 Subject: [PATCH 07/11] fix(logging): mask registered values before splitting records --- internal/diagnosticredact/records.go | 1 + internal/diagnosticredact/redact_test.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/internal/diagnosticredact/records.go b/internal/diagnosticredact/records.go index b045f9af30..7ac03e29ac 100644 --- a/internal/diagnosticredact/records.go +++ b/internal/diagnosticredact/records.go @@ -16,6 +16,7 @@ func Records(redactor *redact.Redactor, value string) string { if json.Valid([]byte(value)) { return wholeRecord(redactor, value) } + value = wholeRecord(redactor, value) var output strings.Builder for value != "" { if LooksLikeStructuredJSON(value) { diff --git a/internal/diagnosticredact/redact_test.go b/internal/diagnosticredact/redact_test.go index f41ff1709a..1961088f6e 100644 --- a/internal/diagnosticredact/redact_test.go +++ b/internal/diagnosticredact/redact_test.go @@ -79,3 +79,25 @@ func TestRecordMasksJSONCredentialRecordsBeforePublicText(t *testing.T) { }) } } + +func TestRecordsMaskRegisteredMultilineValueBeforeSplittingRecords(t *testing.T) { + const cookie = "synthetic-registered-cookie" + const tail = "synthetic registered plain tail" + registered := `{"Cookie":"` + cookie + `"}` + "\n" + tail + redactor := redact.New(redact.Options{ + Environment: []string{"DEVSY_FIXTURE_TOKEN=" + registered}, + }) + input := "public before\n" + registered + "\npublic after" + for name, mask := range map[string]func(*redact.Redactor, string) string{ + "Record": Record, + "Records": Records, + } { + t.Run(name, func(t *testing.T) { + output := mask(redactor, input) + assert.NotContains(t, output, cookie) + assert.NotContains(t, output, tail) + assert.Contains(t, output, "public before") + assert.Contains(t, output, "public after") + }) + } +} From b35f320e6ef8dd99a0ab23fae5e2fadfc7e5dd3b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 17:19:54 -0600 Subject: [PATCH 08/11] fix(logging): run field masking before text patterns unless a value spans records --- internal/diagnosticredact/records.go | 18 ++++++++++++- internal/diagnosticredact/redact_test.go | 34 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/internal/diagnosticredact/records.go b/internal/diagnosticredact/records.go index 7ac03e29ac..4b0c43b61f 100644 --- a/internal/diagnosticredact/records.go +++ b/internal/diagnosticredact/records.go @@ -16,7 +16,9 @@ func Records(redactor *redact.Redactor, value string) string { if json.Valid([]byte(value)) { return wholeRecord(redactor, value) } - value = wholeRecord(redactor, value) + if registeredValueSpansRecords(redactor, value) { + value = wholeRecord(redactor, value) + } var output strings.Builder for value != "" { if LooksLikeStructuredJSON(value) { @@ -36,6 +38,20 @@ func Records(redactor *redact.Redactor, value string) string { return output.String() } +func registeredValueSpansRecords(redactor *redact.Redactor, value string) bool { + if !redactor.ContainsValue(value) { + return false + } + for rest := value; rest != ""; { + record, _, next := splitDiagnosticRecord(rest) + if redactor.ContainsValue(record) { + return false + } + rest = next + } + return true +} + func splitDiagnosticRecord(value string) (record, separator, rest string) { end := strings.IndexAny(value, "\r\n") if end < 0 { diff --git a/internal/diagnosticredact/redact_test.go b/internal/diagnosticredact/redact_test.go index 1961088f6e..0994cf6635 100644 --- a/internal/diagnosticredact/redact_test.go +++ b/internal/diagnosticredact/redact_test.go @@ -101,3 +101,37 @@ func TestRecordsMaskRegisteredMultilineValueBeforeSplittingRecords(t *testing.T) }) } } + +func TestRecordsMaskJSONFieldsBeforeTextPatternsCrossFieldBoundaries(t *testing.T) { + cases := map[string]struct{ input, canary string }{ + "cookie after a URL field": { + input: `pull failed` + "\n" + + `{"url":"https://api.example.test/v1","Cookie":"email=bob@example.test; session=synthetic-cookie-tail"}` + "\n", + canary: "synthetic-cookie-tail", + }, + "authorization after a git URL field": { + input: "x\n" + `{"u":"git://host.test/r","Authorization":"Bearer abc@synthetic-auth-tail"}`, + canary: "synthetic-auth-tail", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + redactor := redact.FromEntries(nil) + assert.NotContains(t, Record(redactor, tc.input), tc.canary) + assert.NotContains(t, Records(redactor, tc.input), tc.canary) + assert.NotContains(t, RecordWithEnvironment(nil, tc.input), tc.canary) + }) + } +} + +func TestRecordsKeepFieldMaskingWhenARegisteredValueIsPresent(t *testing.T) { + const canary = "synthetic-registered-present-tail" + redactor := redact.New(redact.Options{ + Environment: []string{"DEVSY_FIXTURE_TOKEN=synthetic-registered-present"}, + }) + input := "synthetic-registered-present\n" + + `{"url":"https://api.example.test/v1","Cookie":"email=bob@example.test; session=` + canary + `"}` + "\n" + output := Record(redactor, input) + assert.NotContains(t, output, "synthetic-registered-present\n") + assert.NotContains(t, output, canary) +} From 1a2cc0920db74f7aced5d127dbeba13356f0ccc9 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 17:35:50 -0600 Subject: [PATCH 09/11] fix(logging): mask JSON stderr in LFS errors and bound blank record scans --- internal/diagnosticredact/records.go | 4 +-- internal/diagnosticredact/records_test.go | 18 ++++++++++ pkg/git/lfs.go | 34 +++++++++++++------ pkg/git/lfs_test.go | 40 +++++++++++++++++++++++ 4 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 internal/diagnosticredact/records_test.go diff --git a/internal/diagnosticredact/records.go b/internal/diagnosticredact/records.go index 4b0c43b61f..2f28a67ecb 100644 --- a/internal/diagnosticredact/records.go +++ b/internal/diagnosticredact/records.go @@ -69,12 +69,12 @@ func LooksLikeStructuredJSON(value string) bool { if value == "" { return false } - rest := strings.TrimLeft(value[1:], " \t\r\n") switch value[0] { case '{': + rest := strings.TrimLeft(value[1:], " \t\r\n") return rest == "" || rest[0] == '"' || rest[0] == '}' case '[': - return looksLikeJSONList(rest) + return looksLikeJSONList(strings.TrimLeft(value[1:], " \t\r\n")) case '"': return true default: diff --git a/internal/diagnosticredact/records_test.go b/internal/diagnosticredact/records_test.go new file mode 100644 index 0000000000..4ed13f01b5 --- /dev/null +++ b/internal/diagnosticredact/records_test.go @@ -0,0 +1,18 @@ +package diagnosticredact + +import ( + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/logging/redact" + "github.com/stretchr/testify/assert" +) + +func TestRecordsHandleManyBlankRecordsInLinearTime(t *testing.T) { + input := strings.Repeat("\n", 100_000) + "tail" + start := time.Now() + output := Records(redact.FromEntries(nil), input) + assert.Equal(t, input, output) + assert.Less(t, time.Since(start), time.Second) +} diff --git a/pkg/git/lfs.go b/pkg/git/lfs.go index c2351c8807..37f1988b9a 100644 --- a/pkg/git/lfs.go +++ b/pkg/git/lfs.go @@ -3,7 +3,6 @@ package git import ( "bytes" "context" - "errors" "io/fs" "os" "path/filepath" @@ -96,19 +95,34 @@ func ensureLFSBinary(ctx context.Context, allowInstall bool) bool { func (r *Repo) lfs(ctx context.Context, args ...string) error { res, err := r.run(ctx, append([]string{"lfs"}, args...)...) if err != nil { - output := slices.Clone(res.Stdout) - if _, carriesStderr := errors.AsType[*CommandError](err); !carriesStderr { - output = append(output, res.Stderr...) - } - return command.WrapCommandErrorWithEnvironment( - output, - err, - append(os.Environ(), r.env...), - ) + return lfsFailure(res, err, append(os.Environ(), r.env...)) } return nil } +type lfsError struct { + message string + err error +} + +func (e *lfsError) Error() string { return e.message } + +func (e *lfsError) Unwrap() error { return e.err } + +func lfsFailure(res RunResult, err error, environment []string) error { + cause := err + if commandErr, ok := err.(*CommandError); ok { + cause = &CommandError{ + Args: commandErr.Args, + ExitCode: commandErr.ExitCode, + Err: commandErr.Err, + } + } + output := append(slices.Clone(res.Stdout), res.Stderr...) + masked := command.WrapCommandErrorWithEnvironment(output, cause, environment) + return &lfsError{message: masked.Error(), err: err} +} + // lfsDetectMaxDepth bounds how deep repoUsesLFS descends looking for .gitattributes. const lfsDetectMaxDepth = 6 diff --git a/pkg/git/lfs_test.go b/pkg/git/lfs_test.go index 1dfaa33c15..75be83ef7c 100644 --- a/pkg/git/lfs_test.go +++ b/pkg/git/lfs_test.go @@ -63,6 +63,46 @@ func TestLFSFailureMasksStderrCarriedByTheCommandError(t *testing.T) { assert.Assert(t, !strings.Contains(err.Error(), lfsFoldedFixtureValue)) } +type lfsJSONStderrRunner struct { + result RunResult + err error +} + +func (r lfsJSONStderrRunner) Run(context.Context, RunOptions) (RunResult, error) { + return r.result, r.err +} + +func TestLFSFailureMasksJSONStderrRecordCarriedByTheCommandError(t *testing.T) { + const stderr = "{\"Cookie\":\"synthetic-lfs-cookie\"}\npublic lfs tail\n" + original := &CommandError{ + Args: []string{"lfs", "pull"}, + ExitCode: 23, + Stderr: stderr, + Err: errors.New("exit status 23"), + } + repo := At(t.TempDir(), WithRunner(lfsJSONStderrRunner{ + result: RunResult{Stderr: []byte(stderr)}, + err: original, + })) + err := repo.lfs(t.Context(), "pull") + assert.Assert(t, !strings.Contains(err.Error(), "synthetic-lfs-cookie")) + assert.Assert(t, cmp.Contains(err.Error(), "public lfs tail")) + assert.Assert(t, cmp.Contains(err.Error(), "exit status 23")) + var commandErr *CommandError + assert.Assert(t, errors.As(err, &commandErr)) + assert.Equal(t, stderr, commandErr.Stderr) +} + +func TestLFSFailureKeepsRunnerStderrThatTheErrorDoesNotCarry(t *testing.T) { + repo := At(t.TempDir(), WithRunner(lfsJSONStderrRunner{ + result: RunResult{Stderr: []byte("public runner detail")}, + err: &CommandError{Args: []string{"lfs", "pull"}, Err: errors.New("fixture")}, + })) + err := repo.lfs(t.Context(), "pull") + assert.Assert(t, cmp.Contains(err.Error(), "public runner detail")) + assert.Assert(t, cmp.Contains(err.Error(), "fixture")) +} + func TestLFSModeResolution(t *testing.T) { cases := []struct { name string From 88d7df4a3180d0274953142029df551daded10e4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 17:39:39 -0600 Subject: [PATCH 10/11] test(git): share the LFS pull arguments --- pkg/git/lfs_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/git/lfs_test.go b/pkg/git/lfs_test.go index 75be83ef7c..45763ba78c 100644 --- a/pkg/git/lfs_test.go +++ b/pkg/git/lfs_test.go @@ -14,6 +14,8 @@ import ( "gotest.tools/assert/cmp" ) +var lfsPullArgs = []string{"lfs", "pull"} + const ( lfsFoldedFixtureValue = "synthetic-lfs-folded-value" lfsDiagnosticDetail = "public Git LFS detail" @@ -43,7 +45,7 @@ type lfsCommandErrorRunner struct{ cause error } func (r lfsCommandErrorRunner) Run(context.Context, RunOptions) (RunResult, error) { stderr := "Authorization:\n Bearer " + lfsFoldedFixtureValue + "\n\n" + lfsDiagnosticDetail return RunResult{Stderr: []byte(stderr)}, &CommandError{ - Args: []string{"lfs", "pull"}, + Args: lfsPullArgs, ExitCode: 2, Stderr: stderr, Err: r.cause, @@ -75,7 +77,7 @@ func (r lfsJSONStderrRunner) Run(context.Context, RunOptions) (RunResult, error) func TestLFSFailureMasksJSONStderrRecordCarriedByTheCommandError(t *testing.T) { const stderr = "{\"Cookie\":\"synthetic-lfs-cookie\"}\npublic lfs tail\n" original := &CommandError{ - Args: []string{"lfs", "pull"}, + Args: lfsPullArgs, ExitCode: 23, Stderr: stderr, Err: errors.New("exit status 23"), @@ -96,7 +98,7 @@ func TestLFSFailureMasksJSONStderrRecordCarriedByTheCommandError(t *testing.T) { func TestLFSFailureKeepsRunnerStderrThatTheErrorDoesNotCarry(t *testing.T) { repo := At(t.TempDir(), WithRunner(lfsJSONStderrRunner{ result: RunResult{Stderr: []byte("public runner detail")}, - err: &CommandError{Args: []string{"lfs", "pull"}, Err: errors.New("fixture")}, + err: &CommandError{Args: lfsPullArgs, Err: errors.New("fixture")}, })) err := repo.lfs(t.Context(), "pull") assert.Assert(t, cmp.Contains(err.Error(), "public runner detail")) From a581a3330517d38c07416a1de92c488be61eea66 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 21 Nov 2026 23:38:46 -0600 Subject: [PATCH 11/11] fix(logging): mask registered values before framing diagnostics --- go.mod | 2 +- go.sum | 4 +- internal/diagnosticredact/records.go | 130 +++++++++++++++++----- internal/diagnosticredact/records_test.go | 49 ++++++++ internal/diagnosticredact/redact.go | 3 +- internal/diagnosticredact/redact_test.go | 28 ++++- 6 files changed, 183 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index bea4e6ac84..06b348ae03 100644 --- a/go.mod +++ b/go.mod @@ -229,7 +229,7 @@ require ( github.com/davidmz/go-pageant v1.0.2 // indirect github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect github.com/devsy-org/admin-apis v1.2.0 // indirect - github.com/devsy-org/devsy/logging v0.0.0-20261121080943-b1a01fb92a98 + github.com/devsy-org/devsy/logging v0.0.0-20261122052847-badb828ea923 github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb // indirect github.com/dghubble/oauth1 v0.7.3 // indirect github.com/dghubble/sling v1.4.0 // indirect diff --git a/go.sum b/go.sum index 6baf6e1af8..d29f9f1d4a 100644 --- a/go.sum +++ b/go.sum @@ -418,8 +418,8 @@ github.com/devsy-org/apiserver v1.5.4 h1:/bEPrSRSlfii2QHxc9qAiXlQawNJfIaPLraR3bY github.com/devsy-org/apiserver v1.5.4/go.mod h1:sDFCTjCN13wAHhno4KX0Z756gCI8ttW/c7NwTzo16K0= github.com/devsy-org/devsy-runtime-sdk v1.6.0 h1:iEV4Y0Sc3LJNkTLadJSKcWcAf/oCXvC/AkHilrg/KyM= github.com/devsy-org/devsy-runtime-sdk v1.6.0/go.mod h1:MiBP/fiY83DAS0TueEiJZSHYZ1vq85UR4KJTMfHIM+E= -github.com/devsy-org/devsy/logging v0.0.0-20261121080943-b1a01fb92a98 h1:Z+uG77rbf+Msu7LSoWnAe4MylASfsr8gin37+DA4ikk= -github.com/devsy-org/devsy/logging v0.0.0-20261121080943-b1a01fb92a98/go.mod h1:kpwoMhSdrhnJfTJ014ht04BLw0DS7FyF33RqIIe8CKs= +github.com/devsy-org/devsy/logging v0.0.0-20261122052847-badb828ea923 h1:7H4skJl0aR623Sm4jtTai/TjQLnKplMkCkMsxIiz1e0= +github.com/devsy-org/devsy/logging v0.0.0-20261122052847-badb828ea923/go.mod h1:kpwoMhSdrhnJfTJ014ht04BLw0DS7FyF33RqIIe8CKs= github.com/devsy-org/ssh v1.2.10 h1:fCxyqx8ti6qzKhVts55q0zDpx72q1YHNM7VK6TGkAps= github.com/devsy-org/ssh v1.2.10/go.mod h1:IiIPdjt/ilk08Z+I6ofwUaH52I/1PmVnp+0pOVQXlwI= github.com/devsy-org/tailscale v1.102.2 h1:9SB6htvO+HmG8alal8WGCshHapfHR7dUFRSMYiFIzIM= diff --git a/internal/diagnosticredact/records.go b/internal/diagnosticredact/records.go index 2f28a67ecb..4c04df5a24 100644 --- a/internal/diagnosticredact/records.go +++ b/internal/diagnosticredact/records.go @@ -13,17 +13,22 @@ func Records(redactor *redact.Redactor, value string) string { if redactor == nil { return value } + value = redactor.RedactRegistered(value) if json.Valid([]byte(value)) { return wholeRecord(redactor, value) } - if registeredValueSpansRecords(redactor, value) { - value = wholeRecord(redactor, value) - } + return maskRecords(redactor, value) +} + +func maskRecords(redactor *redact.Redactor, value string) string { var output strings.Builder for value != "" { - if LooksLikeStructuredJSON(value) { - if consumed := maskStructuredRecord(redactor, value, &output); consumed > 0 { - value = value[consumed:] + candidate := strings.TrimLeft(value, " \t") + if LooksLikeStructuredJSON(candidate[:min(len(candidate), maxStructuredRecordBytes)]) { + indent := value[:len(value)-len(candidate)] + consumed := maskStructuredRecord(redactor, indent, candidate, &output) + if consumed > 0 { + value = candidate[consumed:] continue } } @@ -38,20 +43,6 @@ func Records(redactor *redact.Redactor, value string) string { return output.String() } -func registeredValueSpansRecords(redactor *redact.Redactor, value string) bool { - if !redactor.ContainsValue(value) { - return false - } - for rest := value; rest != ""; { - record, _, next := splitDiagnosticRecord(rest) - if redactor.ContainsValue(record) { - return false - } - rest = next - } - return true -} - func splitDiagnosticRecord(value string) (record, separator, rest string) { end := strings.IndexAny(value, "\r\n") if end < 0 { @@ -98,18 +89,103 @@ func looksLikeJSONList(rest string) bool { return tail == "" || tail[0] == ',' || tail[0] == ']' } -func maskStructuredRecord(redactor *redact.Redactor, value string, output *strings.Builder) int { - decoder := json.NewDecoder(strings.NewReader(value)) +const ( + maxStructuredRecordBytes = 64 << 10 + maxStructuredRecordDepth = 10_000 +) + +func bracketedProse(line string) bool { + if !strings.HasPrefix(line, "[") { + return false + } + end := strings.IndexByte(line, ']') + return end > 0 && !strings.ContainsAny(line[1:end], `"[{`) +} + +func structuredRecordEnd(value string) int { + if value[0] == '"' { + return 1 + bytesThroughClosingQuote(value[1:]) + } + return bytesThroughClosingBracket(value) +} + +func bytesThroughClosingQuote(value string) int { + for i := 0; i < len(value); i++ { + switch value[i] { + case '\\': + i++ + case '"': + return i + 1 + } + } + return len(value) +} + +func bytesThroughClosingBracket(value string) int { + var opening [maxStructuredRecordDepth]byte + opening[0] = value[0] + depth := 1 + for i := 1; i < len(value); i++ { + switch c := value[i]; c { + case '"': + i += bytesThroughClosingQuote(value[i+1:]) + case '[', '{': + if depth == len(opening) { + return len(value) + } + opening[depth] = c + depth++ + case ']', '}': + depth-- + if !matchingBrackets(opening[depth], c) { + return len(value) + } + if depth == 0 { + return i + 1 + } + } + } + return len(value) +} + +func matchingBrackets(opening, closing byte) bool { + return opening == '[' && closing == ']' || opening == '{' && closing == '}' +} + +func malformedRecordLength(value, prefix string, err error) int { + if errors.Is(err, io.ErrUnexpectedEOF) { + return structuredRecordEnd(value) + } + line, _, _ := splitDiagnosticRecord(prefix) + if strings.HasPrefix(line, `"`) || bracketedProse(line) { + return 0 + } + var syntax *json.SyntaxError + if len(line) == len(prefix) && len(prefix) < len(value) || + (errors.As(err, &syntax) && syntax.Offset > int64(len(line))) { + return structuredRecordEnd(value) + } + return len(line) +} + +func maskStructuredRecord( + redactor *redact.Redactor, + indent, value string, + output *strings.Builder, +) int { + limit := min(len(value), maxStructuredRecordBytes) + prefix := value[:limit] + decoder := json.NewDecoder(strings.NewReader(prefix)) var record json.RawMessage if err := decoder.Decode(&record); err != nil { - if strings.HasPrefix(strings.TrimLeft(value, " \t"), `"`) { - return 0 + consumed := malformedRecordLength(value, prefix, err) + if consumed > 0 { + output.WriteString(redactor.Mask()) } - output.WriteString(redactor.Mask()) - return len(value) + return consumed } end := int(decoder.InputOffset()) - output.WriteString(value[:end-len(record)]) + output.WriteString(indent) output.WriteString(wholeRecord(redactor, string(record))) return end } diff --git a/internal/diagnosticredact/records_test.go b/internal/diagnosticredact/records_test.go index 4ed13f01b5..00d114ccd8 100644 --- a/internal/diagnosticredact/records_test.go +++ b/internal/diagnosticredact/records_test.go @@ -16,3 +16,52 @@ func TestRecordsHandleManyBlankRecordsInLinearTime(t *testing.T) { assert.Equal(t, input, output) assert.Less(t, time.Since(start), time.Second) } + +func TestRecordsKeepPlainBracketedTextUnmasked(t *testing.T) { + redactor := redact.FromEntries(nil) + for _, input := range []string{ + "[1, 2 workers] ready\npublic tail", + "[INFO] ready\n[1, 2 workers] ready\npublic tail", + "{not json} ready\npublic tail", + } { + assert.Equal(t, input, Records(redactor, input), input) + } +} + +func TestRecordsMaskUnterminatedJSONRecordThroughItsEnd(t *testing.T) { + const canary = "synthetic-unterminated-canary" + redactor := redact.FromEntries(nil) + input := "public head\n{\n \"Cookie\": \"" + canary + "\"\n" + output := Records(redactor, input) + assert.NotContains(t, output, canary) + assert.Contains(t, output, "public head\n") +} + +func TestRecordsMaskOversizedStructuredRecordsWithoutExposingTheirTail(t *testing.T) { + const canary = "synthetic-oversized-cookie" + redactor := redact.FromEntries(nil) + head := "{\"details\":[\"" + strings.Repeat("x", maxStructuredRecordBytes) + `\"}]","next"],` + input := "public head\n" + head + "\n" + `"Cookie":"` + canary + `"` + assert.Equal(t, "public head\n"+redactor.Mask(), Records(redactor, input)) + assert.Equal(t, "public head\n"+redactor.Mask()+"\r\npublic tail", + Records(redactor, input+"}\r\npublic tail")) +} + +func TestRecordsMaskStructuredArraysThatStartWithScalars(t *testing.T) { + redactor := redact.FromEntries(nil) + const canary = "synthetic-array-cookie" + input := "public head\n[1,\n{\"Cookie\":\"" + canary + "\"}]\npublic tail" + output := Records(redactor, input) + assert.NotContains(t, output, canary) + assert.Contains(t, output, "public head\n[1,") + assert.Contains(t, output, "\"Cookie\":\"***\"") + assert.Contains(t, output, "]\npublic tail") +} + +func TestRecordsMaskMalformedJSONOnlyThroughTheLine(t *testing.T) { + const canary = "synthetic-malformed-canary" + redactor := redact.FromEntries(nil) + output := Records(redactor, "{\"a\": "+canary+"\npublic tail") + assert.NotContains(t, output, canary) + assert.Contains(t, output, "\npublic tail") +} diff --git a/internal/diagnosticredact/redact.go b/internal/diagnosticredact/redact.go index 45b48c7cd3..f93ee67afb 100644 --- a/internal/diagnosticredact/redact.go +++ b/internal/diagnosticredact/redact.go @@ -21,10 +21,11 @@ func Record(redactor *redact.Redactor, value string) string { if redactor == nil { return value } + value = redactor.RedactRegistered(value) if json.Valid([]byte(value)) { return wholeRecord(redactor, value) } - return wholeRecord(redactor, Records(redactor, value)) + return wholeRecord(redactor, maskRecords(redactor, value)) } func wholeRecord(redactor *redact.Redactor, value string) string { diff --git a/internal/diagnosticredact/redact_test.go b/internal/diagnosticredact/redact_test.go index 0994cf6635..7baef834a0 100644 --- a/internal/diagnosticredact/redact_test.go +++ b/internal/diagnosticredact/redact_test.go @@ -81,19 +81,21 @@ func TestRecordMasksJSONCredentialRecordsBeforePublicText(t *testing.T) { } func TestRecordsMaskRegisteredMultilineValueBeforeSplittingRecords(t *testing.T) { + const single = "synthetic-single-record-value" const cookie = "synthetic-registered-cookie" const tail = "synthetic registered plain tail" registered := `{"Cookie":"` + cookie + `"}` + "\n" + tail redactor := redact.New(redact.Options{ - Environment: []string{"DEVSY_FIXTURE_TOKEN=" + registered}, + Values: []string{single, registered}, }) - input := "public before\n" + registered + "\npublic after" + input := "public before\n" + single + "\n" + registered + "\npublic after" for name, mask := range map[string]func(*redact.Redactor, string) string{ "Record": Record, "Records": Records, } { t.Run(name, func(t *testing.T) { output := mask(redactor, input) + assert.NotContains(t, output, single) assert.NotContains(t, output, cookie) assert.NotContains(t, output, tail) assert.Contains(t, output, "public before") @@ -102,6 +104,28 @@ func TestRecordsMaskRegisteredMultilineValueBeforeSplittingRecords(t *testing.T) } } +func TestRecordsMaskEscapedRegisteredValueBeforeProcessingJSON(t *testing.T) { + const registered = "\"Authorization: Bearer synthetic-initial\"\n synthetic-continuation" + redactor := redact.New(redact.Options{Values: []string{registered}}) + encoded, err := json.Marshal(registered) + require.NoError(t, err) + record := `{"detail":` + string(encoded) + `,"Cookie":"synthetic-cookie","public":"keep"}` + for name, mask := range map[string]func(*redact.Redactor, string) string{ + "Record": Record, + "Records": Records, + } { + t.Run(name, func(t *testing.T) { + for _, input := range []string{record, "public head\n" + record + "\npublic tail"} { + output := mask(redactor, input) + assert.NotContains(t, output, "synthetic-initial") + assert.NotContains(t, output, "synthetic-continuation") + assert.NotContains(t, output, "synthetic-cookie") + assert.Contains(t, output, `"public":"keep"`) + } + }) + } +} + func TestRecordsMaskJSONFieldsBeforeTextPatternsCrossFieldBoundaries(t *testing.T) { cases := map[string]struct{ input, canary string }{ "cookie after a URL field": {