From 99cadc3e8230770d207ca29859cfae7fe4a3f65e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 23 Jun 2026 07:38:41 +0000 Subject: [PATCH 01/53] bundle/fuzz: add create-payload parity fuzz test for terraform vs direct Implements the first technique from DECO-25361: generate random job configs and check for differences in the create payload between the terraform and direct deploy engines. Both engines run the same `bundle deploy` pipeline in-process (via testcli) against a testserver, differing only in DATABRICKS_BUNDLE_ENGINE, and the POST /api/2.2/jobs/create body each sends is captured and diffed. Because only the engine differs, shared mutators cancel out and any remaining diff is a genuine engine divergence. The fuzzer already surfaced two real (benign) divergences, documented in DefaultIgnorePaths: - num_workers: 0 is sent explicitly by terraform but dropped by direct (omitempty). - the terraform provider strips the deprecated spark conf "spark.databricks.delta.preview.enabled"; direct forwards it. Run with: go test ./bundle/fuzz -run TestJobCreateParity (FUZZ_SEEDS overrides the seed count; auto-skips when terraform is not provisioned via acceptance/install_terraform.py). --- bundle/fuzz/capture.go | 59 +++++ bundle/fuzz/capture_deploy.go | 145 ++++++++++++ bundle/fuzz/capture_deploy_test.go | 35 +++ bundle/fuzz/compare.go | 204 +++++++++++++++++ bundle/fuzz/compare_test.go | 95 ++++++++ bundle/fuzz/fuzz_test.go | 68 ++++++ bundle/fuzz/generate.go | 349 +++++++++++++++++++++++++++++ bundle/fuzz/generate_test.go | 47 ++++ bundle/fuzz/rand.go | 47 ++++ 9 files changed, 1049 insertions(+) create mode 100644 bundle/fuzz/capture.go create mode 100644 bundle/fuzz/capture_deploy.go create mode 100644 bundle/fuzz/capture_deploy_test.go create mode 100644 bundle/fuzz/compare.go create mode 100644 bundle/fuzz/compare_test.go create mode 100644 bundle/fuzz/fuzz_test.go create mode 100644 bundle/fuzz/generate.go create mode 100644 bundle/fuzz/generate_test.go create mode 100644 bundle/fuzz/rand.go diff --git a/bundle/fuzz/capture.go b/bundle/fuzz/capture.go new file mode 100644 index 00000000000..330f485f824 --- /dev/null +++ b/bundle/fuzz/capture.go @@ -0,0 +1,59 @@ +package fuzz + +import ( + "encoding/json" + "sync" + + "github.com/databricks/cli/libs/testserver" +) + +// jobsCreatePath is the Jobs API route both engines must hit on create. The +// direct engine posts here via the SDK; the terraform provider is expected to +// post here too, and a mismatch (e.g. a different API version) is itself a +// divergence worth surfacing. +const jobsCreatePath = "/api/2.2/jobs/create" + +// CapturedRequest is a single mutating API request observed by the testserver. +type CapturedRequest struct { + Method string + Path string + Body json.RawMessage +} + +// recorder collects request bodies sent to a testserver. It is safe for +// concurrent use because the SDK and terraform may issue requests from multiple +// goroutines. +type recorder struct { + mu sync.Mutex + requests []CapturedRequest +} + +func (r *recorder) callback(req *testserver.Request) { + r.mu.Lock() + defer r.mu.Unlock() + + var body json.RawMessage + if json.Valid(req.Body) { + // Copy: testserver reuses the underlying buffer across requests. + body = append(json.RawMessage(nil), req.Body...) + } + + r.requests = append(r.requests, CapturedRequest{ + Method: req.Method, + Path: req.URL.Path, + Body: body, + }) +} + +// find returns the body of the first recorded request matching method and path. +func (r *recorder) find(method, path string) (json.RawMessage, bool) { + r.mu.Lock() + defer r.mu.Unlock() + + for _, req := range r.requests { + if req.Method == method && req.Path == path { + return req.Body, true + } + } + return nil, false +} diff --git a/bundle/fuzz/capture_deploy.go b/bundle/fuzz/capture_deploy.go new file mode 100644 index 00000000000..6f06487bf3b --- /dev/null +++ b/bundle/fuzz/capture_deploy.go @@ -0,0 +1,145 @@ +package fuzz + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/internal/testcli" + "github.com/databricks/cli/libs/testserver" +) + +const ( + // bundleResourceKey is the map key the generated job is registered under. + bundleResourceKey = "fuzz_job" + fakeToken = "testtoken" +) + +// CaptureJobCreate deploys a bundle containing job through the given engine +// ("direct" or "terraform") and returns the create request body sent to the +// Jobs API. +// +// Both engines run the full `bundle deploy` pipeline against an in-process +// testserver, so the only difference between two captures with different engines +// is the engine itself. That is what makes the resulting payloads directly +// comparable: shared mutators (deployment metadata, presets, ...) are applied +// identically on both sides and cancel out in the diff. +// +// The terraform engine additionally requires DATABRICKS_TF_EXEC_PATH and +// DATABRICKS_TF_CLI_CONFIG_FILE to point at a provisioned terraform binary and +// provider mirror; see RequireTerraform. +func CaptureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, engine string) (json.RawMessage, error) { + rec := &recorder{} + server := testserver.New(t) + server.RequestCallback = rec.callback + testserver.AddDefaultHandlers(server) + + dir := t.TempDir() + if err := writeJobBundle(dir, server.URL, job); err != nil { + return nil, err + } + + t.Setenv("DATABRICKS_HOST", server.URL) + t.Setenv("DATABRICKS_TOKEN", fakeToken) + t.Setenv("DATABRICKS_BUNDLE_ENGINE", engine) + t.Chdir(dir) + + stdout, stderr, err := testcli.NewRunner(t, ctx, "bundle", "deploy").Run() + if err != nil { + return nil, fmt.Errorf("bundle deploy (engine=%s) failed: %w\nstdout:\n%s\nstderr:\n%s", + engine, err, stdout.String(), stderr.String()) + } + + body, ok := rec.find("POST", jobsCreatePath) + if !ok { + return nil, fmt.Errorf("engine=%s did not POST %s during deploy", engine, jobsCreatePath) + } + return body, nil +} + +// CompareJobEngines deploys job under both engines and returns the create-payload +// differences that are not covered by DefaultIgnorePaths. An empty result means +// the engines produced equivalent create payloads. +func CompareJobEngines(ctx context.Context, t *testing.T, job *resources.Job) ([]Difference, error) { + direct, err := CaptureJobCreate(ctx, t, job, "direct") + if err != nil { + return nil, fmt.Errorf("capturing direct payload: %w", err) + } + terraform, err := CaptureJobCreate(ctx, t, job, "terraform") + if err != nil { + return nil, fmt.Errorf("capturing terraform payload: %w", err) + } + return DiffPayloads(direct, terraform, DefaultIgnorePaths) +} + +// writeJobBundle writes a minimal databricks.yml describing a single job. The +// document is emitted as JSON, which is valid YAML, so we can reuse the job's +// own JSON marshaling (which honors ForceSendFields) without a YAML dependency. +func writeJobBundle(dir, host string, job *resources.Job) error { + jobJSON, err := json.Marshal(job) + if err != nil { + return fmt.Errorf("marshaling job: %w", err) + } + + var jobMap map[string]any + if err := json.Unmarshal(jobJSON, &jobMap); err != nil { + return fmt.Errorf("unmarshaling job: %w", err) + } + + doc := map[string]any{ + "bundle": map[string]any{"name": "fuzz"}, + "workspace": map[string]any{"host": host}, + "resources": map[string]any{ + "jobs": map[string]any{bundleResourceKey: jobMap}, + }, + } + + data, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return fmt.Errorf("marshaling bundle: %w", err) + } + + return os.WriteFile(filepath.Join(dir, "databricks.yml"), data, 0o600) +} + +// RequireTerraform points the terraform engine at the binary and provider mirror +// provisioned by acceptance/install_terraform.py into /build, and skips the +// test when they are absent so the suite still runs where terraform is not set up. +func RequireTerraform(t testing.TB) { + buildDir := filepath.Join(repoRoot(t), "build") + execPath := filepath.Join(buildDir, "terraform") + cfgFile := filepath.Join(buildDir, ".terraformrc") + + if _, err := os.Stat(execPath); err != nil { + t.Skipf("terraform not provisioned (%s); run: python3 acceptance/install_terraform.py --targetdir build", execPath) + } + + t.Setenv("DATABRICKS_TF_EXEC_PATH", execPath) + t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", cfgFile) + t.Setenv("TF_CLI_CONFIG_FILE", cfgFile) + // Terraform phones home to checkpoint-api.hashicorp.com otherwise; disable it + // so the testserver/network isn't hit. See acceptance_test.go. + t.Setenv("CHECKPOINT_DISABLE", "1") +} + +// repoRoot returns the repository root by walking up from the current directory. +func repoRoot(t testing.TB) string { + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %s", err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not locate repo root (go.mod not found)") + } + dir = parent + } +} diff --git a/bundle/fuzz/capture_deploy_test.go b/bundle/fuzz/capture_deploy_test.go new file mode 100644 index 00000000000..2518265d756 --- /dev/null +++ b/bundle/fuzz/capture_deploy_test.go @@ -0,0 +1,35 @@ +package fuzz + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCaptureJobCreateDirect(t *testing.T) { + job := GenerateJob(newRNG(1)) + + body, err := CaptureJobCreate(t.Context(), t, job, "direct") + require.NoError(t, err) + require.NotEmpty(t, body) + + var payload map[string]any + require.NoError(t, json.Unmarshal(body, &payload)) + assert.Equal(t, job.Name, payload["name"]) + assert.Contains(t, payload, "tasks") +} + +func TestCaptureJobCreateTerraform(t *testing.T) { + RequireTerraform(t) + job := GenerateJob(newRNG(1)) + + body, err := CaptureJobCreate(t.Context(), t, job, "terraform") + require.NoError(t, err) + require.NotEmpty(t, body) + + var payload map[string]any + require.NoError(t, json.Unmarshal(body, &payload)) + assert.Equal(t, job.Name, payload["name"]) +} diff --git a/bundle/fuzz/compare.go b/bundle/fuzz/compare.go new file mode 100644 index 00000000000..48b7d3e6481 --- /dev/null +++ b/bundle/fuzz/compare.go @@ -0,0 +1,204 @@ +package fuzz + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "slices" + "strconv" + "strings" +) + +// Difference is a single mismatch between the two engines' create payloads, +// located by a JSON-ish path (e.g. "tasks[0].new_cluster.num_workers"). +type Difference struct { + Path string + Direct any + Terraform any +} + +func (d Difference) String() string { + return fmt.Sprintf("%s: direct=%s terraform=%s", d.Path, render(d.Direct), render(d.Terraform)) +} + +// missing marks a value that is absent on one side. +type missing struct{} + +func render(v any) string { + if _, ok := v.(missing); ok { + return "" + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Sprintf("%v", v) + } + return string(b) +} + +// DiffPayloads decodes both create payloads and returns every difference whose +// path is not explicitly ignored. ignorePaths are matched exactly against the +// rendered path, with "[*]" standing in for any slice index. +func DiffPayloads(direct, terraform json.RawMessage, ignorePaths []string) ([]Difference, error) { + d, err := decode(direct) + if err != nil { + return nil, fmt.Errorf("decoding direct payload: %w", err) + } + tf, err := decode(terraform) + if err != nil { + return nil, fmt.Errorf("decoding terraform payload: %w", err) + } + + var diffs []Difference + diffValue("", d, tf, &diffs) + + ignore := make(map[string]bool, len(ignorePaths)) + for _, p := range ignorePaths { + ignore[p] = true + } + + filtered := diffs[:0] + for _, diff := range diffs { + if !ignore[normalizePath(diff.Path)] { + filtered = append(filtered, diff) + } + } + return filtered, nil +} + +// decode unmarshals JSON using UseNumber so large int64 values (e.g. job ids, +// spark_context_id) are not corrupted by float64 rounding. See the encoding rule +// in the repo style guide. +func decode(raw json.RawMessage) (any, error) { + if len(raw) == 0 { + return nil, nil + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + return v, nil +} + +func diffValue(path string, a, b any, diffs *[]Difference) { + switch av := a.(type) { + case map[string]any: + bv, ok := b.(map[string]any) + if !ok { + *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + return + } + keys := unionKeys(av, bv) + for _, k := range keys { + achild, aok := av[k] + bchild, bok := bv[k] + child := joinKey(path, k) + switch { + case aok && bok: + diffValue(child, achild, bchild, diffs) + case aok: + *diffs = append(*diffs, Difference{Path: child, Direct: achild, Terraform: missing{}}) + default: + *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bchild}) + } + } + case []any: + bv, ok := b.([]any) + if !ok { + *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + return + } + n := max(len(av), len(bv)) + for i := range n { + child := fmt.Sprintf("%s[%d]", path, i) + switch { + case i < len(av) && i < len(bv): + diffValue(child, av[i], bv[i], diffs) + case i < len(av): + *diffs = append(*diffs, Difference{Path: child, Direct: av[i], Terraform: missing{}}) + default: + *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bv[i]}) + } + } + default: + if !scalarEqual(a, b) { + *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + } + } +} + +// scalarEqual compares two JSON scalars. json.Number is compared by its string +// form so 1 and 1.0 don't masquerade as equal across engines. +func scalarEqual(a, b any) bool { + an, aok := a.(json.Number) + bn, bok := b.(json.Number) + if aok && bok { + return an.String() == bn.String() + } + return a == b +} + +func unionKeys(a, b map[string]any) []string { + seen := map[string]bool{} + var keys []string + for k := range a { + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + } + for k := range b { + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + } + slices.Sort(keys) + return keys +} + +func joinKey(path, key string) string { + // Map keys can themselves contain dots or brackets (e.g. spark_conf entries + // like "spark.databricks.delta.preview.enabled"). Render those as bracketed, + // quoted segments so the path stays unambiguous and ignore entries can target + // a single key. + if key == "" || strings.ContainsAny(key, `.[]"`) { + return path + "[" + strconv.Quote(key) + "]" + } + if path == "" { + return key + } + return path + "." + key +} + +// indexRe matches numeric slice indices like "[12]" but not quoted string keys +// like ["spark.x"]. +var indexRe = regexp.MustCompile(`\[\d+\]`) + +// normalizePath replaces concrete slice indices with [*] so a single ignore +// entry can cover every element of a slice. +func normalizePath(path string) string { + return indexRe.ReplaceAllString(path, "[*]") +} + +// DefaultIgnorePaths lists create-payload paths that legitimately differ between +// the engines and are not parity bugs. Keep this list small and well-justified; +// every entry is a known, intentional divergence. +var DefaultIgnorePaths = []string{ + // num_workers is a zero-able int: when a cluster has num_workers: 0 the + // terraform provider serializes it explicitly while the direct engine drops + // it via omitempty. The backend treats absent and 0 identically, so this is a + // benign serialization difference. See the update_single_node acceptance test + // ("issues with zero conversion"). + "tasks[*].new_cluster.num_workers", + "job_clusters[*].new_cluster.num_workers", + + // The terraform provider strips the deprecated/ignored spark conf + // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while + // the direct engine forwards it verbatim. The backend ignores the key either + // way, so this is a benign provider-side filter rather than a parity bug. + `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, + `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, +} diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go new file mode 100644 index 00000000000..ec5818468b8 --- /dev/null +++ b/bundle/fuzz/compare_test.go @@ -0,0 +1,95 @@ +package fuzz + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiffPayloads(t *testing.T) { + tests := []struct { + name string + direct string + terraform string + ignore []string + want []string + }{ + { + name: "identical", + direct: `{"name":"a","tasks":[{"task_key":"t"}]}`, + terraform: `{"name":"a","tasks":[{"task_key":"t"}]}`, + want: nil, + }, + { + name: "scalar mismatch", + direct: `{"name":"a"}`, + terraform: `{"name":"b"}`, + want: []string{"name"}, + }, + { + name: "missing on terraform", + direct: `{"name":"a","queue":{"enabled":true}}`, + terraform: `{"name":"a"}`, + want: []string{"queue"}, + }, + { + name: "missing on direct", + direct: `{"name":"a"}`, + terraform: `{"name":"a","max_concurrent_runs":1}`, + want: []string{"max_concurrent_runs"}, + }, + { + name: "nested slice element mismatch", + direct: `{"tasks":[{"task_key":"t","timeout_seconds":1}]}`, + terraform: `{"tasks":[{"task_key":"t","timeout_seconds":2}]}`, + want: []string{"tasks[0].timeout_seconds"}, + }, + { + name: "slice length mismatch", + direct: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + terraform: `{"tasks":[{"task_key":"a"}]}`, + want: []string{"tasks[1]"}, + }, + { + name: "number 1 vs 1.0 differ", + direct: `{"n":1}`, + terraform: `{"n":1.0}`, + want: []string{"n"}, + }, + { + name: "ignored path", + direct: `{"tasks":[{"timeout_seconds":1}]}`, + terraform: `{"tasks":[{"timeout_seconds":2}]}`, + ignore: []string{"tasks[*].timeout_seconds"}, + want: nil, + }, + { + name: "dotted map key is bracket-quoted", + direct: `{"spark_conf":{"spark.x.y":"1"}}`, + terraform: `{"spark_conf":{}}`, + want: []string{`spark_conf["spark.x.y"]`}, + }, + { + name: "dotted map key can be ignored", + direct: `{"c":{"spark_conf":{"spark.x.y":"1"}}}`, + terraform: `{"c":{"spark_conf":{}}}`, + ignore: []string{`c.spark_conf["spark.x.y"]`}, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + diffs, err := DiffPayloads(json.RawMessage(tt.direct), json.RawMessage(tt.terraform), tt.ignore) + require.NoError(t, err) + + var paths []string + for _, d := range diffs { + paths = append(paths, d.Path) + } + assert.ElementsMatch(t, tt.want, paths) + }) + } +} diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go new file mode 100644 index 00000000000..55e52eb0bb7 --- /dev/null +++ b/bundle/fuzz/fuzz_test.go @@ -0,0 +1,68 @@ +package fuzz + +import ( + "encoding/json" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +// defaultParitySeeds is the number of random jobs TestJobCreateParity checks by +// default. Each seed runs two real deploys (direct + terraform), so the count is +// kept modest; override with FUZZ_SEEDS for a deeper local run. +const defaultParitySeeds = 20 + +// TestJobCreateParity is the first DECO-25361 technique: for many random job +// configs, assert the terraform and direct engines produce equivalent create +// payloads. On divergence it prints the seed and the generated job so the failure +// can be reproduced and inspected. +func TestJobCreateParity(t *testing.T) { + RequireTerraform(t) + + seeds := defaultParitySeeds + if v := os.Getenv("FUZZ_SEEDS"); v != "" { + n, err := strconv.Atoi(v) + require.NoErrorf(t, err, "invalid FUZZ_SEEDS=%q", v) + seeds = n + } + + for seed := int64(0); seed < int64(seeds); seed++ { + t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { + checkJobParity(t, seed) + }) + } +} + +// FuzzJobCreateParity exposes the same parity check to Go's native fuzzer +// (`go test -fuzz=FuzzJobCreateParity`). Note each input runs two real deploys, +// so this is intended for ad-hoc deep runs, not the default `go test` path. +func FuzzJobCreateParity(f *testing.F) { + RequireTerraform(f) + for seed := int64(0); seed < 5; seed++ { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, seed int64) { + checkJobParity(t, seed) + }) +} + +// checkJobParity generates the job for seed, deploys it under both engines, and +// fails the test with reproduction details if the create payloads diverge. +func checkJobParity(t *testing.T, seed int64) { + t.Helper() + job := GenerateJob(newRNG(seed)) + + diffs, err := CompareJobEngines(t.Context(), t, job) + require.NoErrorf(t, err, "seed %d", seed) + + if len(diffs) > 0 { + jobJSON, _ := json.MarshalIndent(job, "", " ") + t.Errorf("seed %d: terraform/direct create payloads diverge (%d differences):", seed, len(diffs)) + for _, d := range diffs { + t.Errorf(" %s", d) + } + t.Logf("reproduce with GenerateJob(newRNG(%d)):\n%s", seed, jobJSON) + } +} diff --git a/bundle/fuzz/generate.go b/bundle/fuzz/generate.go new file mode 100644 index 00000000000..a7c5e6056f9 --- /dev/null +++ b/bundle/fuzz/generate.go @@ -0,0 +1,349 @@ +// Package fuzz provides randomized generators and harnesses that compare how the +// terraform and direct deploy engines translate the same bundle resource into an +// API create payload. See DECO-25361. +// +// The first technique implemented here generates a random resource config and +// checks for differences in the create payload between the terraform and direct +// engines. Generators are seeded so that any divergence found by the fuzz driver +// can be reproduced from the printed seed. +package fuzz + +import ( + "fmt" + "math/rand/v2" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// Value pools are intentionally small and valid-looking: the goal is to exercise +// the engines' config->payload translation across many field combinations, not to +// stress the API with invalid values (which the testserver would reject before we +// can compare payloads). +var ( + sparkVersions = []string{"13.3.x-scala2.12", "14.3.x-scala2.12", "15.4.x-scala2.12", "16.4.x-scala2.12"} + nodeTypeIDs = []string{"i3.xlarge", "m5.large", "r5.xlarge", "Standard_DS3_v2"} + timezones = []string{"UTC", "America/Los_Angeles", "Europe/Amsterdam"} + cronExprs = []string{"0 0 12 * * ?", "0 15 10 ? * MON-FRI", "0 0/30 * * * ?"} + pauseStatuses = []jobs.PauseStatus{jobs.PauseStatusPaused, jobs.PauseStatusUnpaused} + performance = []jobs.PerformanceTarget{jobs.PerformanceTargetPerformanceOptimized, jobs.PerformanceTargetStandard} + timeUnits = []string{"HOURS", "DAYS", "WEEKS"} + healthMetrics = []string{"RUN_DURATION_SECONDS", "STREAMING_BACKLOG_BYTES", "STREAMING_BACKLOG_RECORDS"} + conditionOps = []string{"EQUAL_TO", "NOT_EQUAL", "GREATER_THAN", "LESS_THAN_OR_EQUAL"} + runIfs = []string{"ALL_SUCCESS", "AT_LEAST_ONE_SUCCESS", "NONE_FAILED", "ALL_DONE"} + gitProviders = []jobs.GitProvider{jobs.GitProviderGitHub, jobs.GitProviderGitLab, jobs.GitProviderAzureDevOpsServices} +) + +// GenerateJob builds a random, well-formed job config driven entirely by rng, so +// the same seed always produces the same job. It deliberately favors fields whose +// translation tends to differ between engines (tasks, clusters, schedules, +// notifications, tags, zero-able scalars). +func GenerateJob(rng *rand.Rand) *resources.Job { + job := &resources.Job{} + job.Name = randName(rng, "job") + + if chance(rng, 0.5) { + job.Description = randSentence(rng) + } + if chance(rng, 0.4) { + job.MaxConcurrentRuns = rng.IntN(10) + 1 + } + if chance(rng, 0.4) { + job.TimeoutSeconds = rng.IntN(7200) + } + if chance(rng, 0.3) { + job.PerformanceTarget = oneOf(rng, performance) + } + if chance(rng, 0.5) { + job.Tags = randTags(rng) + } + if chance(rng, 0.3) { + job.GitSource = randGitSource(rng) + } + + randScheduling(rng, job) + + if chance(rng, 0.3) { + job.EmailNotifications = randEmailNotifications(rng) + } + if chance(rng, 0.2) { + job.WebhookNotifications = randWebhookNotifications(rng) + } + if chance(rng, 0.3) { + job.NotificationSettings = &jobs.JobNotificationSettings{ + NoAlertForCanceledRuns: chance(rng, 0.5), + NoAlertForSkippedRuns: chance(rng, 0.5), + } + } + if chance(rng, 0.3) { + job.Health = randHealth(rng) + } + if chance(rng, 0.3) { + job.Parameters = randParameters(rng) + } + if chance(rng, 0.3) { + job.Queue = &jobs.QueueSettings{Enabled: chance(rng, 0.5)} + } + + // Generate shared job clusters first so tasks can reference them by key. + var jobClusterKeys []string + if chance(rng, 0.5) { + n := rng.IntN(2) + 1 + for i := range n { + key := fmt.Sprintf("cluster_%d", i) + jobClusterKeys = append(jobClusterKeys, key) + job.JobClusters = append(job.JobClusters, jobs.JobCluster{ + JobClusterKey: key, + NewCluster: randClusterSpec(rng), + }) + } + } + + nTasks := rng.IntN(3) + 1 + var taskKeys []string + for i := range nTasks { + task := randTask(rng, i, jobClusterKeys) + // Randomly chain dependencies onto previously generated tasks. + if len(taskKeys) > 0 && chance(rng, 0.4) { + dep := taskKeys[rng.IntN(len(taskKeys))] + task.DependsOn = []jobs.TaskDependency{{TaskKey: dep}} + if chance(rng, 0.5) { + task.RunIf = jobs.RunIf(oneOf(rng, runIfs)) + } + } + taskKeys = append(taskKeys, task.TaskKey) + job.Tasks = append(job.Tasks, task) + } + + return job +} + +// randScheduling sets at most one of schedule/trigger/continuous, which are +// mutually exclusive ways to launch a job. +func randScheduling(rng *rand.Rand, job *resources.Job) { + switch rng.IntN(5) { + case 0: + job.Schedule = &jobs.CronSchedule{ + QuartzCronExpression: oneOf(rng, cronExprs), + TimezoneId: oneOf(rng, timezones), + PauseStatus: oneOf(rng, pauseStatuses), + } + case 1: + job.Trigger = &jobs.TriggerSettings{ + PauseStatus: oneOf(rng, pauseStatuses), + Periodic: &jobs.PeriodicTriggerConfiguration{ + Interval: rng.IntN(12) + 1, + Unit: jobs.PeriodicTriggerConfigurationTimeUnit(oneOf(rng, timeUnits)), + }, + } + case 2: + job.Trigger = &jobs.TriggerSettings{ + PauseStatus: oneOf(rng, pauseStatuses), + FileArrival: &jobs.FileArrivalTriggerConfiguration{ + Url: "s3://" + randWord(rng) + "/" + randWord(rng), + }, + } + case 3: + job.Continuous = &jobs.Continuous{PauseStatus: oneOf(rng, pauseStatuses)} + default: + // no scheduling + } +} + +func randTask(rng *rand.Rand, idx int, jobClusterKeys []string) jobs.Task { + task := jobs.Task{TaskKey: fmt.Sprintf("task_%d", idx)} + + // Use absolute workspace paths with source=WORKSPACE so the generated bundle + // never depends on local files existing on disk (which deploy would reject). + // condition_task needs no compute, so it is handled separately below. + needsCompute := true + switch rng.IntN(4) { + case 0: + task.NotebookTask = &jobs.NotebookTask{ + NotebookPath: "/Workspace/Users/test/" + randName(rng, "nb"), + Source: jobs.SourceWorkspace, + } + case 1: + task.SparkPythonTask = &jobs.SparkPythonTask{ + PythonFile: "/Workspace/Users/test/" + randName(rng, "main") + ".py", + Source: jobs.SourceWorkspace, + } + case 2: + task.PythonWheelTask = &jobs.PythonWheelTask{ + PackageName: randName(rng, "pkg"), + EntryPoint: "main", + } + case 3: + task.ConditionTask = &jobs.ConditionTask{ + Left: randWord(rng), + Op: jobs.ConditionTaskOp(oneOf(rng, conditionOps)), + Right: randWord(rng), + } + needsCompute = false + } + + if needsCompute { + assignCompute(rng, &task, jobClusterKeys) + if chance(rng, 0.4) { + task.Libraries = randLibraries(rng) + } + } + + if chance(rng, 0.3) { + task.TimeoutSeconds = rng.IntN(3600) + } + if chance(rng, 0.3) { + task.MaxRetries = rng.IntN(5) + task.MinRetryIntervalMillis = rng.IntN(60000) + task.RetryOnTimeout = chance(rng, 0.5) + } + return task +} + +// assignCompute attaches exactly one compute source, which notebook/python/wheel +// tasks require: a shared job cluster (when available), a brand-new cluster, or an +// existing cluster id. +func assignCompute(rng *rand.Rand, task *jobs.Task, jobClusterKeys []string) { + const ( + computeNew = iota + computeExisting + computeShared + ) + options := []int{computeNew, computeExisting} + if len(jobClusterKeys) > 0 { + options = append(options, computeShared) + } + switch oneOf(rng, options) { + case computeNew: + spec := randClusterSpec(rng) + task.NewCluster = &spec + case computeExisting: + task.ExistingClusterId = randName(rng, "cluster") + case computeShared: + task.JobClusterKey = oneOf(rng, jobClusterKeys) + } +} + +func randClusterSpec(rng *rand.Rand) compute.ClusterSpec { + spec := compute.ClusterSpec{ + SparkVersion: oneOf(rng, sparkVersions), + NodeTypeId: oneOf(rng, nodeTypeIDs), + } + if chance(rng, 0.5) { + spec.NumWorkers = rng.IntN(8) + } else { + spec.Autoscale = &compute.AutoScale{ + MinWorkers: 1, + MaxWorkers: rng.IntN(8) + 2, + } + } + if chance(rng, 0.4) { + spec.SparkConf = map[string]string{ + "spark.databricks.delta.preview.enabled": "true", + "spark.speculation": fmt.Sprintf("%t", chance(rng, 0.5)), + } + } + if chance(rng, 0.3) { + spec.CustomTags = randTags(rng) + } + if chance(rng, 0.3) { + spec.SparkEnvVars = map[string]string{"PYSPARK_PYTHON": "/databricks/python3/bin/python3"} + } + if chance(rng, 0.3) { + spec.DriverNodeTypeId = oneOf(rng, nodeTypeIDs) + } + return spec +} + +func randGitSource(rng *rand.Rand) *jobs.GitSource { + src := &jobs.GitSource{ + GitProvider: oneOf(rng, gitProviders), + GitUrl: "https://example.com/" + randWord(rng) + "/" + randWord(rng) + ".git", + } + switch rng.IntN(3) { + case 0: + src.GitBranch = oneOf(rng, []string{"main", "develop", "release"}) + case 1: + src.GitTag = "v" + fmt.Sprintf("%d.%d.0", rng.IntN(5), rng.IntN(10)) + case 2: + src.GitCommit = fmt.Sprintf("%040x", rng.Int64()) + } + return src +} + +func randEmailNotifications(rng *rand.Rand) *jobs.JobEmailNotifications { + email := randWord(rng) + "@example.com" + n := &jobs.JobEmailNotifications{NoAlertForSkippedRuns: chance(rng, 0.5)} + if chance(rng, 0.6) { + n.OnFailure = []string{email} + } + if chance(rng, 0.4) { + n.OnSuccess = []string{email} + } + if chance(rng, 0.3) { + n.OnStart = []string{email} + } + return n +} + +func randWebhookNotifications(rng *rand.Rand) *jobs.WebhookNotifications { + hook := []jobs.Webhook{{Id: randName(rng, "hook")}} + n := &jobs.WebhookNotifications{} + if chance(rng, 0.6) { + n.OnFailure = hook + } + if chance(rng, 0.4) { + n.OnSuccess = hook + } + return n +} + +func randHealth(rng *rand.Rand) *jobs.JobsHealthRules { + return &jobs.JobsHealthRules{ + Rules: []jobs.JobsHealthRule{ + { + Metric: jobs.JobsHealthMetric(oneOf(rng, healthMetrics)), + Op: jobs.JobsHealthOperatorGreaterThan, + Value: int64(rng.IntN(3600) + 1), + }, + }, + } +} + +func randLibraries(rng *rand.Rand) []compute.Library { + n := rng.IntN(2) + 1 + libs := make([]compute.Library, 0, n) + for range n { + switch rng.IntN(3) { + case 0: + libs = append(libs, compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: randWord(rng)}}) + case 1: + libs = append(libs, compute.Library{Maven: &compute.MavenLibrary{Coordinates: "org.example:" + randWord(rng) + ":1.0.0"}}) + case 2: + libs = append(libs, compute.Library{Whl: "/Workspace/Users/test/" + randName(rng, "lib") + ".whl"}) + } + } + return libs +} + +func randParameters(rng *rand.Rand) []jobs.JobParameterDefinition { + n := rng.IntN(3) + 1 + params := make([]jobs.JobParameterDefinition, 0, n) + for i := range n { + params = append(params, jobs.JobParameterDefinition{ + Name: fmt.Sprintf("param_%d", i), + Default: randWord(rng), + }) + } + return params +} + +func randTags(rng *rand.Rand) map[string]string { + n := rng.IntN(3) + 1 + tags := make(map[string]string, n) + for i := range n { + tags[fmt.Sprintf("tag_%d", i)] = randWord(rng) + } + return tags +} diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go new file mode 100644 index 00000000000..524e84864c3 --- /dev/null +++ b/bundle/fuzz/generate_test.go @@ -0,0 +1,47 @@ +package fuzz + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateJobIsDeterministic(t *testing.T) { + a := GenerateJob(newRNG(42)) + b := GenerateJob(newRNG(42)) + assert.Equal(t, a, b, "same seed must produce identical job") +} + +func TestGenerateJobIsWellFormed(t *testing.T) { + for seed := int64(0); seed < 200; seed++ { + job := GenerateJob(newRNG(seed)) + require.NotEmptyf(t, job.Name, "seed %d: job must have a name", seed) + require.NotEmptyf(t, job.Tasks, "seed %d: job must have at least one task", seed) + + clusterKeys := map[string]bool{} + for _, jc := range job.JobClusters { + clusterKeys[jc.JobClusterKey] = true + } + + taskKeys := map[string]bool{} + for _, task := range job.Tasks { + require.NotEmptyf(t, task.TaskKey, "seed %d: task must have a key", seed) + taskKeys[task.TaskKey] = true + + // A task referencing a job cluster must reference one we generated. + if task.JobClusterKey != "" { + assert.Containsf(t, clusterKeys, task.JobClusterKey, + "seed %d: task %q references unknown job cluster %q", seed, task.TaskKey, task.JobClusterKey) + } + } + + // Every dependency must point at a task that exists in this job. + for _, task := range job.Tasks { + for _, dep := range task.DependsOn { + assert.Containsf(t, taskKeys, dep.TaskKey, + "seed %d: task %q depends on unknown task %q", seed, task.TaskKey, dep.TaskKey) + } + } + } +} diff --git a/bundle/fuzz/rand.go b/bundle/fuzz/rand.go new file mode 100644 index 00000000000..529e4da1153 --- /dev/null +++ b/bundle/fuzz/rand.go @@ -0,0 +1,47 @@ +package fuzz + +import ( + "fmt" + "math/rand/v2" + "strings" +) + +var words = []string{ + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", + "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa", +} + +// newRNG returns a deterministic RNG for the given seed, so any job the fuzzer +// flags can be regenerated from the printed seed alone. +func newRNG(seed int64) *rand.Rand { + return rand.New(rand.NewPCG(uint64(seed), 0)) +} + +// chance returns true with probability p (0..1). +func chance(rng *rand.Rand, p float64) bool { + return rng.Float64() < p +} + +// oneOf returns a random element of s. s must be non-empty. +func oneOf[T any](rng *rand.Rand, s []T) T { + return s[rng.IntN(len(s))] +} + +func randWord(rng *rand.Rand) string { + return oneOf(rng, words) +} + +// randName returns a deterministic-but-varied identifier with the given prefix, +// e.g. "job_alpha_4271". +func randName(rng *rand.Rand, prefix string) string { + return fmt.Sprintf("%s_%s_%d", prefix, randWord(rng), rng.IntN(10000)) +} + +func randSentence(rng *rand.Rand) string { + n := rng.IntN(4) + 2 + parts := make([]string, 0, n) + for range n { + parts = append(parts, randWord(rng)) + } + return strings.Join(parts, " ") +} From 495d1523aaa4cbaf7c061023e1e330295916efd4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 23 Jun 2026 08:06:00 +0000 Subject: [PATCH 02/53] bundle/fuzz: fix lint (intrange, perfsprint) and correct num_workers ignore Address golangci-lint failures (intrange loops, strconv.FormatBool over fmt.Sprintf) and tighten the create-payload ignore list: drop the dead job_clusters num_workers entry (those are at parity) and document the task-level num_workers divergence as a real CLI gap to fix separately. --- bundle/fuzz/compare.go | 14 ++++++++------ bundle/fuzz/fuzz_test.go | 4 ++-- bundle/fuzz/generate.go | 3 ++- bundle/fuzz/generate_test.go | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bundle/fuzz/compare.go b/bundle/fuzz/compare.go index 48b7d3e6481..e893ab443d5 100644 --- a/bundle/fuzz/compare.go +++ b/bundle/fuzz/compare.go @@ -187,13 +187,15 @@ func normalizePath(path string) string { // the engines and are not parity bugs. Keep this list small and well-justified; // every entry is a known, intentional divergence. var DefaultIgnorePaths = []string{ - // num_workers is a zero-able int: when a cluster has num_workers: 0 the - // terraform provider serializes it explicitly while the direct engine drops - // it via omitempty. The backend treats absent and 0 identically, so this is a - // benign serialization difference. See the update_single_node acceptance test - // ("issues with zero conversion"). + // A single-node task cluster (num_workers: 0, no autoscale) diverges: the + // terraform provider sends num_workers: 0 while the direct engine omits it. + // JobClustersFixups.initializeNumWorkers force-sends num_workers for + // job_clusters but is NOT applied to task-level new_cluster, so the fix-up + // only covers job_clusters (those are at parity and need no ignore here). + // This is a real CLI gap surfaced by the fuzzer, tracked separately; ignore + // it here so the fuzz suite stays green until the fix-up is extended to task + // clusters. "tasks[*].new_cluster.num_workers", - "job_clusters[*].new_cluster.num_workers", // The terraform provider strips the deprecated/ignored spark conf // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 55e52eb0bb7..ace7a5efd30 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -28,7 +28,7 @@ func TestJobCreateParity(t *testing.T) { seeds = n } - for seed := int64(0); seed < int64(seeds); seed++ { + for seed := range int64(seeds) { t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { checkJobParity(t, seed) }) @@ -40,7 +40,7 @@ func TestJobCreateParity(t *testing.T) { // so this is intended for ad-hoc deep runs, not the default `go test` path. func FuzzJobCreateParity(f *testing.F) { RequireTerraform(f) - for seed := int64(0); seed < 5; seed++ { + for seed := range int64(5) { f.Add(seed) } f.Fuzz(func(t *testing.T, seed int64) { diff --git a/bundle/fuzz/generate.go b/bundle/fuzz/generate.go index a7c5e6056f9..98db7a70f5e 100644 --- a/bundle/fuzz/generate.go +++ b/bundle/fuzz/generate.go @@ -11,6 +11,7 @@ package fuzz import ( "fmt" "math/rand/v2" + "strconv" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/compute" @@ -241,7 +242,7 @@ func randClusterSpec(rng *rand.Rand) compute.ClusterSpec { if chance(rng, 0.4) { spec.SparkConf = map[string]string{ "spark.databricks.delta.preview.enabled": "true", - "spark.speculation": fmt.Sprintf("%t", chance(rng, 0.5)), + "spark.speculation": strconv.FormatBool(chance(rng, 0.5)), } } if chance(rng, 0.3) { diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go index 524e84864c3..f7a797e8f59 100644 --- a/bundle/fuzz/generate_test.go +++ b/bundle/fuzz/generate_test.go @@ -14,7 +14,7 @@ func TestGenerateJobIsDeterministic(t *testing.T) { } func TestGenerateJobIsWellFormed(t *testing.T) { - for seed := int64(0); seed < 200; seed++ { + for seed := range int64(200) { job := GenerateJob(newRNG(seed)) require.NotEmptyf(t, job.Name, "seed %d: job must have a name", seed) require.NotEmptyf(t, job.Tasks, "seed %d: job must have at least one task", seed) From df8bc36d375d716783203562b29d87d10bcbc71a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 23 Jun 2026 09:46:58 +0000 Subject: [PATCH 03/53] bundle/fuzz: wire parity tests into CI and harden harness - Add a `test-fuzz` task and a nightly CI job that provisions terraform and runs the create-payload parity tests. They previously always skipped because terraform was never provisioned in the test path. - Ignore repo-root build/ so the provisioned terraform binary and provider mirror are not accidentally committed. - Skip cleanly when build/ is only partially provisioned (missing provider mirror or .terraformrc) instead of failing mid-deploy. - Document that the harness covers jobs only for now (DECO-25361). --- .github/workflows/push.yml | 35 +++++++++++++++++++++++++++++++++++ .gitignore | 4 ++++ Taskfile.yml | 15 +++++++++++++++ bundle/fuzz/capture_deploy.go | 10 ++++++++-- bundle/fuzz/generate.go | 6 ++++++ 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 593b0ad8229..8de730fbb04 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -393,6 +393,41 @@ jobs: run: | go tool -modfile=tools/task/go.mod task test-sandbox + test-fuzz: + needs: + - cleanups + + # The terraform/direct create-payload parity tests run two real `bundle deploy` + # invocations per seed, so they are too slow for every PR and too noisy to gate + # the merge queue. Run them on the nightly schedule to catch engine drift; not + # part of test-result for that reason. + if: ${{ github.event_name == 'schedule' }} + name: "task test-fuzz" + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: test-fuzz + + - name: Run tests + run: | + go tool -modfile=tools/task/go.mod task test-fuzz + # This job groups the result of all the above test jobs. # It is a required check, so it blocks auto-merge and the merge queue. # diff --git a/.gitignore b/.gitignore index 543b638a537..a38c9c22f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,10 @@ tools/testmask/testmask # Release artifacts dist/ +# Terraform binary + provider mirror provisioned by acceptance/install_terraform.py +# for the bundle/fuzz parity tests (see Taskfile `test-fuzz`). +/build/ + # Local development notes, tmp /pr-* /tmp/ diff --git a/Taskfile.yml b/Taskfile.yml index 1eb41d6ac53..22f5cca5031 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -730,6 +730,21 @@ tasks: --packages ./acceptance/... \ -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox" + test-fuzz: + desc: Run terraform/direct create-payload parity fuzz tests (provisions terraform) + sources: + - bundle/fuzz/** + cmds: + # The parity harness expects terraform + the provider mirror at /build; + # RequireTerraform skips when it's absent, so provision it first. + - python3 acceptance/install_terraform.py --targetdir build + - | + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./bundle/fuzz/... \ + -- -timeout=${LOCAL_TIMEOUT:-30m} + # --- Integration tests --- integration: diff --git a/bundle/fuzz/capture_deploy.go b/bundle/fuzz/capture_deploy.go index 6f06487bf3b..0efeaa9ed13 100644 --- a/bundle/fuzz/capture_deploy.go +++ b/bundle/fuzz/capture_deploy.go @@ -114,8 +114,14 @@ func RequireTerraform(t testing.TB) { execPath := filepath.Join(buildDir, "terraform") cfgFile := filepath.Join(buildDir, ".terraformrc") - if _, err := os.Stat(execPath); err != nil { - t.Skipf("terraform not provisioned (%s); run: python3 acceptance/install_terraform.py --targetdir build", execPath) + // install_terraform.py provisions all three together; a partial build/ (e.g. + // the binary without the provider mirror or .terraformrc) would otherwise fail + // mid-deploy with a confusing error instead of skipping cleanly. + tfpluginsDir := filepath.Join(buildDir, "tfplugins") + for _, p := range []string{execPath, cfgFile, tfpluginsDir} { + if _, err := os.Stat(p); err != nil { + t.Skipf("terraform not fully provisioned (%s); run: python3 acceptance/install_terraform.py --targetdir build", p) + } } t.Setenv("DATABRICKS_TF_EXEC_PATH", execPath) diff --git a/bundle/fuzz/generate.go b/bundle/fuzz/generate.go index 98db7a70f5e..697748e03ff 100644 --- a/bundle/fuzz/generate.go +++ b/bundle/fuzz/generate.go @@ -6,6 +6,9 @@ // checks for differences in the create payload between the terraform and direct // engines. Generators are seeded so that any divergence found by the fuzz driver // can be reproduced from the printed seed. +// +// Only jobs are covered for now. Extending the harness to other resource kinds +// (pipelines, apps, ...) is tracked as follow-up work under DECO-25361. package fuzz import ( @@ -40,6 +43,9 @@ var ( // the same seed always produces the same job. It deliberately favors fields whose // translation tends to differ between engines (tasks, clusters, schedules, // notifications, tags, zero-able scalars). +// +// TODO(DECO-25361): generalize the harness across resource kinds so pipelines, +// apps, etc. get the same create-payload parity coverage as jobs. func GenerateJob(rng *rand.Rand) *resources.Job { job := &resources.Job{} job.Name = randName(rng, "job") From a24e58eb64fae6c2439e2bdf577f8ac21acf9f92 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 24 Jun 2026 08:28:25 +0000 Subject: [PATCH 04/53] bundle/fuzz: rotate nightly seeds and add single-seed reproduction Make the create-payload parity fuzz suite explore new configs over time and be reproducible from a reported seed: - FUZZ_SEED (comma-separated) runs exactly those seeds, overriding the range, so a reported divergence reproduces with one command. The failure message now prints this knob. - FUZZ_SEED_OFFSET shifts the deterministic window; push.yml derives it from GITHUB_RUN_NUMBER so each nightly run checks seeds it has never tested before instead of re-checking a fixed set. Windows are non-overlapping because the run number is unique and monotonic. - Guard FUZZ_SEEDS > 0 so a negative value no longer panics make() and zero no longer passes as a no-op. - Drop the test-fuzz Task sources fingerprint: the seeds depend on env vars Task can't see, so skipping on an unchanged checksum would silently no-op a repro run or a shifted window. - Keep the nightly window modest (25); exploration comes from rotation, not size, and it can be raised once nightly timings are known. --- .github/workflows/push.yml | 14 ++++++++++ Taskfile.yml | 6 ++-- bundle/fuzz/fuzz_test.go | 57 +++++++++++++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8de730fbb04..ba852d9c958 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -425,7 +425,21 @@ jobs: cache-key: test-fuzz - name: Run tests + env: + # Shift the seed window by the run number every nightly run so CI + # explores configs it has never tested before instead of re-checking a + # fixed set. The window is kept modest (each seed runs two real deploys) + # since the exploration comes from rotating the window, not its size; + # raise it once nightly timings are known. A divergence prints + # FUZZ_SEED= for one-command reproduction. + # + # offset = GITHUB_RUN_NUMBER * FUZZ_SEEDS. GITHUB_RUN_NUMBER is a + # built-in, monotonically increasing, unique-per-run integer, so as long + # as FUZZ_SEEDS is constant the windows are non-overlapping (gaps from + # non-schedule runs are fine; we only need fresh seeds, not every seed). + FUZZ_SEEDS: "25" run: | + export FUZZ_SEED_OFFSET=$(( GITHUB_RUN_NUMBER * FUZZ_SEEDS )) go tool -modfile=tools/task/go.mod task test-fuzz # This job groups the result of all the above test jobs. diff --git a/Taskfile.yml b/Taskfile.yml index 22f5cca5031..04f77e444fe 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -732,8 +732,10 @@ tasks: test-fuzz: desc: Run terraform/direct create-payload parity fuzz tests (provisions terraform) - sources: - - bundle/fuzz/** + # No `sources:` fingerprint: the seeds checked are a function of the FUZZ_SEED, + # FUZZ_SEEDS, and FUZZ_SEED_OFFSET env vars, which Task can't see. Skipping on + # an unchanged source checksum would silently no-op a FUZZ_SEED= repro run + # or a shifted nightly window, so always run. cmds: # The parity harness expects terraform + the provider mirror at /build; # RequireTerraform skips when it's absent, so provision it first. diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index ace7a5efd30..51471b35333 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "strconv" + "strings" "testing" "github.com/stretchr/testify/require" @@ -21,18 +22,60 @@ const defaultParitySeeds = 20 func TestJobCreateParity(t *testing.T) { RequireTerraform(t) - seeds := defaultParitySeeds + for _, seed := range paritySeeds(t) { + t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { + checkJobParity(t, seed) + }) + } +} + +// paritySeeds returns the seeds TestJobCreateParity should check. +// +// FUZZ_SEED (comma-separated list) runs exactly those seeds and overrides +// everything else. This is the knob the failure message prints so a single +// reported divergence can be reproduced with one command, without re-running +// every seed before it. +// +// Otherwise the test runs FUZZ_SEEDS seeds (default defaultParitySeeds) starting +// at FUZZ_SEED_OFFSET. The offset lets the nightly job shift the window every run +// (push.yml derives it from the run number) so CI explores configs it has never +// tested before instead of re-checking the same fixed set forever. +func paritySeeds(t *testing.T) []int64 { + if v := os.Getenv("FUZZ_SEED"); v != "" { + var seeds []int64 + for _, part := range strings.Split(v, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + n, err := strconv.ParseInt(part, 10, 64) + require.NoErrorf(t, err, "invalid FUZZ_SEED entry %q", part) + seeds = append(seeds, n) + } + require.NotEmptyf(t, seeds, "FUZZ_SEED=%q contained no seeds", v) + return seeds + } + + count := defaultParitySeeds if v := os.Getenv("FUZZ_SEEDS"); v != "" { n, err := strconv.Atoi(v) require.NoErrorf(t, err, "invalid FUZZ_SEEDS=%q", v) - seeds = n + require.Greaterf(t, n, 0, "FUZZ_SEEDS must be positive, got %d", n) + count = n } - for seed := range int64(seeds) { - t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { - checkJobParity(t, seed) - }) + var offset int64 + if v := os.Getenv("FUZZ_SEED_OFFSET"); v != "" { + n, err := strconv.ParseInt(v, 10, 64) + require.NoErrorf(t, err, "invalid FUZZ_SEED_OFFSET=%q", v) + offset = n + } + + seeds := make([]int64, 0, count) + for i := range int64(count) { + seeds = append(seeds, offset+i) } + return seeds } // FuzzJobCreateParity exposes the same parity check to Go's native fuzzer @@ -63,6 +106,6 @@ func checkJobParity(t *testing.T, seed int64) { for _, d := range diffs { t.Errorf(" %s", d) } - t.Logf("reproduce with GenerateJob(newRNG(%d)):\n%s", seed, jobJSON) + t.Logf("reproduce with: FUZZ_SEED=%d go test ./bundle/fuzz -run TestJobCreateParity\n%s", seed, jobJSON) } } From 51921ab42f1a778930199932ad138c904b72a4e6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 24 Jun 2026 12:03:48 +0000 Subject: [PATCH 05/53] bundle: force-send num_workers for single-node task clusters The terraform provider force-sends num_workers: 0 for a single-node new_cluster (no autoscale) on both job_clusters and task-level clusters, but JobClustersFixups only applied initializeNumWorkers to job_clusters. The direct engine therefore omitted num_workers on task clusters, so the two engines produced divergent create payloads. This divergence was surfaced by the bundle/fuzz parity harness. Apply initializeNumWorkers to task new_cluster too so the direct engine matches terraform, and drop the now-obsolete tasks[*].new_cluster.num_workers entry from the fuzz DefaultIgnorePaths. --- acceptance/bundle/deploy/wal/chain-3-jobs/output.txt | 2 ++ .../bundle/deploy/wal/crash-after-create/output.txt | 1 + acceptance/bundle/override/job_tasks/output.txt | 2 ++ .../missing_map_key/out.validate.direct.json | 3 ++- .../missing_map_key/out.validate.terraform.json | 3 ++- .../config/mutator/resourcemutator/cluster_fixups.go | 1 + bundle/fuzz/compare.go | 10 ---------- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index 7cbc9f31a85..5cb87e0c097 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -35,6 +35,7 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { @@ -73,6 +74,7 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index e9e2c19c094..d0a32c78254 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -39,6 +39,7 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt index 2bee9738e33..59b6fc1c397 100644 --- a/acceptance/bundle/override/job_tasks/output.txt +++ b/acceptance/bundle/override/job_tasks/output.txt @@ -18,6 +18,7 @@ }, { "new_cluster": { + "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { @@ -42,6 +43,7 @@ Exit code: 1 "tasks": [ { "new_cluster": { + "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json index cfd1427ce4d..7279aaeba31 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json @@ -30,7 +30,8 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - } + }, + "num_workers": 0 }, "task_key": "test-task" } diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json index 3cdf58f84ea..3bad6f46193 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json @@ -30,7 +30,8 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - } + }, + "num_workers": 0 }, "task_key": "test-task" } diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups.go b/bundle/config/mutator/resourcemutator/cluster_fixups.go index 893cd248aa4..04ddef6cc2f 100644 --- a/bundle/config/mutator/resourcemutator/cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/cluster_fixups.go @@ -94,6 +94,7 @@ func prepareJobSettingsForUpdate(js *jobs.JobSettings) { for _, task := range js.Tasks { if task.NewCluster != nil { ModifyRequestOnInstancePool(task.NewCluster) + initializeNumWorkers(task.NewCluster) } } for ind := range js.JobClusters { diff --git a/bundle/fuzz/compare.go b/bundle/fuzz/compare.go index e893ab443d5..de681719622 100644 --- a/bundle/fuzz/compare.go +++ b/bundle/fuzz/compare.go @@ -187,16 +187,6 @@ func normalizePath(path string) string { // the engines and are not parity bugs. Keep this list small and well-justified; // every entry is a known, intentional divergence. var DefaultIgnorePaths = []string{ - // A single-node task cluster (num_workers: 0, no autoscale) diverges: the - // terraform provider sends num_workers: 0 while the direct engine omits it. - // JobClustersFixups.initializeNumWorkers force-sends num_workers for - // job_clusters but is NOT applied to task-level new_cluster, so the fix-up - // only covers job_clusters (those are at parity and need no ignore here). - // This is a real CLI gap surfaced by the fuzzer, tracked separately; ignore - // it here so the fuzz suite stays green until the fix-up is extended to task - // clusters. - "tasks[*].new_cluster.num_workers", - // The terraform provider strips the deprecated/ignored spark conf // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while // the direct engine forwards it verbatim. The backend ignores the key either From b2c02392af33f6e935a247046664d61785ed955d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 24 Jun 2026 12:04:02 +0000 Subject: [PATCH 06/53] bundle/fuzz: report nightly parity failures and fix create-path comment The nightly test-fuzz job is intentionally excluded from test-result, so a failure was only visible in the Actions tab. Add a failure step that opens (or comments on) a single deduped GitHub issue with a one-command repro. Also correct the jobsCreatePath comment: a different API version shows up as a capture failure (the testserver registers only this route, so a mismatched version 404s and the deploy fails), not as a payload diff. --- .github/workflows/push.yml | 37 +++++++++++++++++++++++++++++++++++++ bundle/fuzz/capture.go | 8 +++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ba852d9c958..61b68388b72 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -414,6 +414,8 @@ jobs: permissions: id-token: write contents: read + # Needed by the failure-reporting step below to open/comment a tracking issue. + issues: write steps: - name: Checkout repository and submodules @@ -442,6 +444,41 @@ jobs: export FUZZ_SEED_OFFSET=$(( GITHUB_RUN_NUMBER * FUZZ_SEEDS )) go tool -modfile=tools/task/go.mod task test-fuzz + # This job is intentionally excluded from test-result, so a failure here is + # invisible unless someone watches the Actions tab. Surface it as a GitHub + # issue instead. Reuse a single open issue (deduped by label) so a recurring + # divergence doesn't open one issue per night. + - name: Report failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + gh label create fuzz-nightly \ + --description "Nightly terraform/direct create-payload parity failures" \ + --color FBCA04 2>/dev/null || true + + body=$(cat <\`. + Reproduce locally with: + + \`\`\` + FUZZ_SEED= go test ./bundle/fuzz -run TestJobCreateParity + \`\`\` + EOF + ) + + existing=$(gh issue list --state open --label fuzz-nightly --json number --jq '.[0].number') + if [ -n "$existing" ]; then + gh issue comment "$existing" --body "$body" + else + gh issue create --title "Nightly fuzz parity failure" --label fuzz-nightly --body "$body" + fi + # This job groups the result of all the above test jobs. # It is a required check, so it blocks auto-merge and the merge queue. # diff --git a/bundle/fuzz/capture.go b/bundle/fuzz/capture.go index 330f485f824..fe10bc10be8 100644 --- a/bundle/fuzz/capture.go +++ b/bundle/fuzz/capture.go @@ -8,9 +8,11 @@ import ( ) // jobsCreatePath is the Jobs API route both engines must hit on create. The -// direct engine posts here via the SDK; the terraform provider is expected to -// post here too, and a mismatch (e.g. a different API version) is itself a -// divergence worth surfacing. +// direct engine posts here via the SDK and the terraform provider is expected to +// as well. The testserver registers only this exact route, so if an engine ever +// posted to a different version the deploy would 404 and CaptureJobCreate would +// fail with "did not POST". A version skew therefore surfaces as a capture +// failure, not as a payload diff. const jobsCreatePath = "/api/2.2/jobs/create" // CapturedRequest is a single mutating API request observed by the testserver. From e70783ec916d0bfb938dcd6c47afded04e052821 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 24 Jun 2026 13:26:12 +0000 Subject: [PATCH 07/53] bundle/fuzz: make harness files test-only and add num_workers regression test Rename the capture/deploy/recorder helpers to *_test.go so the parity harness compiles only under `go test` instead of into the package's regular build, and add a committed regression test (cluster_fixups_test.go) covering the single-node task-cluster num_workers force-send fix so the divergence is guarded at PR time, not just in the nightly suite. --- .github/workflows/push.yml | 5 +- Taskfile.yml | 8 +- .../resourcemutator/cluster_fixups_test.go | 92 +++++++++++++++++++ bundle/fuzz/compare.go | 72 +++++++++++++++ bundle/fuzz/compare_test.go | 24 +++++ ...re_deploy_test.go => deploy_smoke_test.go} | 6 +- .../{capture_deploy.go => deploy_test.go} | 44 ++++++--- bundle/fuzz/fuzz_test.go | 81 ++++++++++++++-- bundle/fuzz/{capture.go => recorder_test.go} | 10 +- 9 files changed, 311 insertions(+), 31 deletions(-) create mode 100644 bundle/config/mutator/resourcemutator/cluster_fixups_test.go rename bundle/fuzz/{capture_deploy_test.go => deploy_smoke_test.go} (82%) rename bundle/fuzz/{capture_deploy.go => deploy_test.go} (73%) rename bundle/fuzz/{capture.go => recorder_test.go} (86%) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 61b68388b72..8029ef6dbb5 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -467,8 +467,11 @@ jobs: Reproduce locally with: \`\`\` - FUZZ_SEED= go test ./bundle/fuzz -run TestJobCreateParity + FUZZ_SEED= task test-fuzz \`\`\` + + Once fixed, add the seed to \`regressionSeeds\` in \`bundle/fuzz/fuzz_test.go\` + in the same PR so the divergence can never silently regress. EOF ) diff --git a/Taskfile.yml b/Taskfile.yml index 04f77e444fe..b8d6a139166 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -736,9 +736,15 @@ tasks: # FUZZ_SEEDS, and FUZZ_SEED_OFFSET env vars, which Task can't see. Skipping on # an unchanged source checksum would silently no-op a FUZZ_SEED= repro run # or a shifted nightly window, so always run. + env: + # The terraform parity tests are opt-in (see requireFuzzOptIn): they skip + # unless a FUZZ_* var is set, so a leftover build/ never makes them run as + # part of a plain `task test`. This constant flag opts this target in + # without overriding the FUZZ_SEED(S)/OFFSET tuning knobs. + FUZZ_PARITY: "1" cmds: # The parity harness expects terraform + the provider mirror at /build; - # RequireTerraform skips when it's absent, so provision it first. + # requireTerraform skips when it's absent, so provision it first. - python3 acceptance/install_terraform.py --targetdir build - | {{.GO_TOOL}} gotestsum \ diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups_test.go b/bundle/config/mutator/resourcemutator/cluster_fixups_test.go new file mode 100644 index 00000000000..5cb2e937494 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/cluster_fixups_test.go @@ -0,0 +1,92 @@ +package resourcemutator + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestInitializeNumWorkers(t *testing.T) { + tests := []struct { + name string + spec compute.ClusterSpec + wantForceSend bool + }{ + { + name: "single-node cluster force-sends num_workers", + spec: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, + wantForceSend: true, + }, + { + name: "autoscale cluster does not force-send", + spec: compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, + wantForceSend: false, + }, + { + name: "multi-node cluster does not force-send", + spec: compute.ClusterSpec{NumWorkers: 3}, + wantForceSend: false, + }, + { + name: "already force-sent stays force-sent without duplicating", + spec: compute.ClusterSpec{ForceSendFields: []string{"NumWorkers"}}, + wantForceSend: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec + initializeNumWorkers(&spec) + + count := 0 + for _, f := range spec.ForceSendFields { + if f == "NumWorkers" { + count++ + } + } + if tt.wantForceSend { + assert.Equal(t, 1, count, "NumWorkers must appear in ForceSendFields exactly once") + } else { + assert.Equal(t, 0, count, "NumWorkers must not be in ForceSendFields") + } + }) + } +} + +// TestPrepareJobSettingsForUpdateForcesNumWorkers locks the DECO-25361 fix: a +// single-node new_cluster must force-send num_workers on task-level clusters too, +// not just shared job_clusters. The terraform provider always sends num_workers:0 +// for such clusters, so missing it on the task side made the direct engine +// produce a divergent create payload. +func TestPrepareJobSettingsForUpdateForcesNumWorkers(t *testing.T) { + js := &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "single_node_task", + NewCluster: &compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, + }, + { + TaskKey: "autoscale_task", + NewCluster: &compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, + }, + }, + JobClusters: []jobs.JobCluster{ + { + JobClusterKey: "single_node_cluster", + NewCluster: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, + }, + }, + } + + prepareJobSettingsForUpdate(js) + + assert.Contains(t, js.Tasks[0].NewCluster.ForceSendFields, "NumWorkers", + "single-node task cluster must force-send num_workers") + assert.NotContains(t, js.Tasks[1].NewCluster.ForceSendFields, "NumWorkers", + "autoscale task cluster must not force-send num_workers") + assert.Contains(t, js.JobClusters[0].NewCluster.ForceSendFields, "NumWorkers", + "single-node job cluster must force-send num_workers") +} diff --git a/bundle/fuzz/compare.go b/bundle/fuzz/compare.go index de681719622..81c1bc7afb4 100644 --- a/bundle/fuzz/compare.go +++ b/bundle/fuzz/compare.go @@ -110,6 +110,14 @@ func diffValue(path string, a, b any, diffs *[]Difference) { *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) return } + // Slices whose elements carry a natural identity key (tasks, job clusters) + // are matched by that key so an engine emitting the same elements in a + // different order is not reported as a difference. Everything else is + // compared positionally. + if key := identityKey(av, bv); key != "" { + diffKeyedSlice(path, key, av, bv, diffs) + return + } n := max(len(av), len(bv)) for i := range n { child := fmt.Sprintf("%s[%d]", path, i) @@ -129,6 +137,70 @@ func diffValue(path string, a, b any, diffs *[]Difference) { } } +// identityFields are the keys, in priority order, that uniquely identify the +// elements of a payload slice. Job tasks and shared job clusters are the slices +// whose order is not significant but which the engines may emit differently. +var identityFields = []string{"task_key", "job_cluster_key"} + +// identityKey returns the field that identifies every element of both slices, or +// "" if the elements are not uniformly keyed objects (in which case the caller +// falls back to positional comparison). +func identityKey(a, b []any) string { + for _, field := range identityFields { + if allHaveKey(a, field) && allHaveKey(b, field) { + return field + } + } + return "" +} + +func allHaveKey(s []any, field string) bool { + if len(s) == 0 { + return false + } + for _, el := range s { + m, ok := el.(map[string]any) + if !ok { + return false + } + if _, ok := m[field].(string); !ok { + return false + } + } + return true +} + +// diffKeyedSlice matches elements of a and b by the value of key (which is unique +// within each slice for tasks/job clusters) and diffs each matched pair, +// reporting unmatched elements as present-on-one-side. Paths keep numeric indices +// so ignore-path [*] normalization still applies. +func diffKeyedSlice(path, key string, a, b []any, diffs *[]Difference) { + bByKey := make(map[string]any, len(b)) + for _, el := range b { + bByKey[el.(map[string]any)[key].(string)] = el + } + + matched := make(map[string]bool, len(a)) + for i, el := range a { + child := fmt.Sprintf("%s[%d]", path, i) + k := el.(map[string]any)[key].(string) + matched[k] = true + if bel, ok := bByKey[k]; ok { + diffValue(child, el, bel, diffs) + } else { + *diffs = append(*diffs, Difference{Path: child, Direct: el, Terraform: missing{}}) + } + } + for j, el := range b { + k := el.(map[string]any)[key].(string) + if matched[k] { + continue + } + child := fmt.Sprintf("%s[%d]", path, j) + *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: el}) + } +} + // scalarEqual compares two JSON scalars. json.Number is compared by its string // form so 1 and 1.0 don't masquerade as equal across engines. func scalarEqual(a, b any) bool { diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go index ec5818468b8..46e506d75c6 100644 --- a/bundle/fuzz/compare_test.go +++ b/bundle/fuzz/compare_test.go @@ -78,6 +78,30 @@ func TestDiffPayloads(t *testing.T) { ignore: []string{`c.spark_conf["spark.x.y"]`}, want: nil, }, + { + name: "tasks matched by key ignore order", + direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, + terraform: `{"tasks":[{"task_key":"b","timeout_seconds":2},{"task_key":"a","timeout_seconds":1}]}`, + want: nil, + }, + { + name: "tasks matched by key surface real diff at direct index", + direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, + terraform: `{"tasks":[{"task_key":"b","timeout_seconds":9},{"task_key":"a","timeout_seconds":1}]}`, + want: []string{"tasks[1].timeout_seconds"}, + }, + { + name: "task only on terraform reported at its index", + direct: `{"tasks":[{"task_key":"a"}]}`, + terraform: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + want: []string{"tasks[1]"}, + }, + { + name: "job_clusters matched by key ignore order", + direct: `{"job_clusters":[{"job_cluster_key":"x","new_cluster":{"num_workers":1}},{"job_cluster_key":"y","new_cluster":{"num_workers":2}}]}`, + terraform: `{"job_clusters":[{"job_cluster_key":"y","new_cluster":{"num_workers":2}},{"job_cluster_key":"x","new_cluster":{"num_workers":1}}]}`, + want: nil, + }, } for _, tt := range tests { diff --git a/bundle/fuzz/capture_deploy_test.go b/bundle/fuzz/deploy_smoke_test.go similarity index 82% rename from bundle/fuzz/capture_deploy_test.go rename to bundle/fuzz/deploy_smoke_test.go index 2518265d756..d501ee78089 100644 --- a/bundle/fuzz/capture_deploy_test.go +++ b/bundle/fuzz/deploy_smoke_test.go @@ -11,7 +11,7 @@ import ( func TestCaptureJobCreateDirect(t *testing.T) { job := GenerateJob(newRNG(1)) - body, err := CaptureJobCreate(t.Context(), t, job, "direct") + body, err := captureJobCreate(t.Context(), t, job, "direct") require.NoError(t, err) require.NotEmpty(t, body) @@ -22,10 +22,10 @@ func TestCaptureJobCreateDirect(t *testing.T) { } func TestCaptureJobCreateTerraform(t *testing.T) { - RequireTerraform(t) + requireTerraform(t) job := GenerateJob(newRNG(1)) - body, err := CaptureJobCreate(t.Context(), t, job, "terraform") + body, err := captureJobCreate(t.Context(), t, job, "terraform") require.NoError(t, err) require.NotEmpty(t, body) diff --git a/bundle/fuzz/capture_deploy.go b/bundle/fuzz/deploy_test.go similarity index 73% rename from bundle/fuzz/capture_deploy.go rename to bundle/fuzz/deploy_test.go index 0efeaa9ed13..e42dbb74346 100644 --- a/bundle/fuzz/capture_deploy.go +++ b/bundle/fuzz/deploy_test.go @@ -19,7 +19,7 @@ const ( fakeToken = "testtoken" ) -// CaptureJobCreate deploys a bundle containing job through the given engine +// captureJobCreate deploys a bundle containing job through the given engine // ("direct" or "terraform") and returns the create request body sent to the // Jobs API. // @@ -31,8 +31,8 @@ const ( // // The terraform engine additionally requires DATABRICKS_TF_EXEC_PATH and // DATABRICKS_TF_CLI_CONFIG_FILE to point at a provisioned terraform binary and -// provider mirror; see RequireTerraform. -func CaptureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, engine string) (json.RawMessage, error) { +// provider mirror; see requireTerraform. +func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, engine string) (json.RawMessage, error) { rec := &recorder{} server := testserver.New(t) server.RequestCallback = rec.callback @@ -61,15 +61,15 @@ func CaptureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, eng return body, nil } -// CompareJobEngines deploys job under both engines and returns the create-payload +// compareJobEngines deploys job under both engines and returns the create-payload // differences that are not covered by DefaultIgnorePaths. An empty result means // the engines produced equivalent create payloads. -func CompareJobEngines(ctx context.Context, t *testing.T, job *resources.Job) ([]Difference, error) { - direct, err := CaptureJobCreate(ctx, t, job, "direct") +func compareJobEngines(ctx context.Context, t *testing.T, job *resources.Job) ([]Difference, error) { + direct, err := captureJobCreate(ctx, t, job, "direct") if err != nil { return nil, fmt.Errorf("capturing direct payload: %w", err) } - terraform, err := CaptureJobCreate(ctx, t, job, "terraform") + terraform, err := captureJobCreate(ctx, t, job, "terraform") if err != nil { return nil, fmt.Errorf("capturing terraform payload: %w", err) } @@ -106,10 +106,32 @@ func writeJobBundle(dir, host string, job *resources.Job) error { return os.WriteFile(filepath.Join(dir, "databricks.yml"), data, 0o600) } -// RequireTerraform points the terraform engine at the binary and provider mirror -// provisioned by acceptance/install_terraform.py into /build, and skips the -// test when they are absent so the suite still runs where terraform is not set up. -func RequireTerraform(t testing.TB) { +// fuzzOptInVars are the environment variables that opt a run into the +// terraform-backed parity suite. FUZZ_SEED / FUZZ_SEEDS / FUZZ_SEED_OFFSET double +// as the tuning knobs (see paritySeeds), so setting any of them implies opt-in; +// FUZZ_PARITY is a no-tuning switch used by `task test-fuzz`. +var fuzzOptInVars = []string{"FUZZ_PARITY", "FUZZ_SEED", "FUZZ_SEEDS", "FUZZ_SEED_OFFSET"} + +// requireFuzzOptIn skips unless the run explicitly opted into the terraform +// parity suite. Gating on an env var rather than on the presence of build/ keeps +// a leftover terraform install (from a prior `task test-fuzz` or acceptance run) +// from silently turning a plain `task test` into dozens of real deploys. +func requireFuzzOptIn(t testing.TB) { + for _, name := range fuzzOptInVars { + if os.Getenv(name) != "" { + return + } + } + t.Skip("terraform parity suite is opt-in; run `task test-fuzz` or set FUZZ_SEED= to reproduce a single seed") +} + +// requireTerraform opts in via requireFuzzOptIn, then points the terraform engine +// at the binary and provider mirror provisioned by acceptance/install_terraform.py +// into /build, skipping when they are absent so the suite still skips +// cleanly where terraform is not set up. +func requireTerraform(t testing.TB) { + requireFuzzOptIn(t) + buildDir := filepath.Join(repoRoot(t), "build") execPath := filepath.Join(buildDir, "terraform") cfgFile := filepath.Join(buildDir, ".terraformrc") diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 51471b35333..7b0d0df8ea3 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,12 +16,27 @@ import ( // kept modest; override with FUZZ_SEEDS for a deeper local run. const defaultParitySeeds = 20 +// regressionSeeds are seeds that previously surfaced a terraform/direct create +// payload divergence. They are always checked (in addition to the rotating +// nightly window) so a fixed divergence can never silently regress, even though +// the nightly window moves on every run and would otherwise never revisit them. +// +// When the nightly job reports a new failing FUZZ_SEED, add it here in the same +// PR that fixes the divergence. +// +// - 29: first seed that generates a single-node task-level new_cluster +// (num_workers 0, no autoscale). The direct engine omitted num_workers on +// task clusters while terraform force-sent num_workers:0, so the create +// payloads diverged. Fixed by applying initializeNumWorkers to task clusters +// in resourcemutator.prepareJobSettingsForUpdate. +var regressionSeeds = []int64{29} + // TestJobCreateParity is the first DECO-25361 technique: for many random job // configs, assert the terraform and direct engines produce equivalent create // payloads. On divergence it prints the seed and the generated job so the failure // can be reproduced and inspected. func TestJobCreateParity(t *testing.T) { - RequireTerraform(t) + requireTerraform(t) for _, seed := range paritySeeds(t) { t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { @@ -36,10 +52,12 @@ func TestJobCreateParity(t *testing.T) { // reported divergence can be reproduced with one command, without re-running // every seed before it. // -// Otherwise the test runs FUZZ_SEEDS seeds (default defaultParitySeeds) starting -// at FUZZ_SEED_OFFSET. The offset lets the nightly job shift the window every run -// (push.yml derives it from the run number) so CI explores configs it has never -// tested before instead of re-checking the same fixed set forever. +// Otherwise the test runs the regressionSeeds plus FUZZ_SEEDS seeds (default +// defaultParitySeeds) starting at FUZZ_SEED_OFFSET. The offset lets the nightly +// job shift the window every run (push.yml derives it from the run number) so CI +// explores configs it has never tested before instead of re-checking the same +// fixed set forever; the regressionSeeds are always included on top so known +// past divergences keep being verified. func paritySeeds(t *testing.T) []int64 { if v := os.Getenv("FUZZ_SEED"); v != "" { var seeds []int64 @@ -71,21 +89,64 @@ func paritySeeds(t *testing.T) []int64 { offset = n } - seeds := make([]int64, 0, count) + seeds := make([]int64, 0, len(regressionSeeds)+count) + seen := make(map[int64]bool, len(regressionSeeds)+count) + for _, s := range regressionSeeds { + if !seen[s] { + seen[s] = true + seeds = append(seeds, s) + } + } for i := range int64(count) { - seeds = append(seeds, offset+i) + s := offset + i + if !seen[s] { + seen[s] = true + seeds = append(seeds, s) + } } return seeds } +func TestParitySeeds(t *testing.T) { + t.Run("default includes regression seeds then window", func(t *testing.T) { + t.Setenv("FUZZ_SEEDS", "3") + t.Setenv("FUZZ_SEED_OFFSET", "100") + want := append(append([]int64{}, regressionSeeds...), 100, 101, 102) + assert.Equal(t, want, paritySeeds(t)) + }) + + t.Run("window overlapping a regression seed is deduplicated", func(t *testing.T) { + t.Setenv("FUZZ_SEEDS", "5") + t.Setenv("FUZZ_SEED_OFFSET", "27") + seeds := paritySeeds(t) + count := 0 + for _, s := range seeds { + if s == 29 { + count++ + } + } + assert.Equal(t, 1, count, "seed 29 must appear once even though it is both a regression seed and inside the window") + }) + + t.Run("FUZZ_SEED override ignores regression seeds", func(t *testing.T) { + t.Setenv("FUZZ_SEED", "7, 8") + assert.Equal(t, []int64{7, 8}, paritySeeds(t)) + }) +} + // FuzzJobCreateParity exposes the same parity check to Go's native fuzzer // (`go test -fuzz=FuzzJobCreateParity`). Note each input runs two real deploys, // so this is intended for ad-hoc deep runs, not the default `go test` path. func FuzzJobCreateParity(f *testing.F) { - RequireTerraform(f) + requireTerraform(f) for seed := range int64(5) { f.Add(seed) } + // Seed the corpus with known past divergences so the fuzzer always starts + // from inputs that previously exposed a bug. + for _, seed := range regressionSeeds { + f.Add(seed) + } f.Fuzz(func(t *testing.T, seed int64) { checkJobParity(t, seed) }) @@ -97,7 +158,7 @@ func checkJobParity(t *testing.T, seed int64) { t.Helper() job := GenerateJob(newRNG(seed)) - diffs, err := CompareJobEngines(t.Context(), t, job) + diffs, err := compareJobEngines(t.Context(), t, job) require.NoErrorf(t, err, "seed %d", seed) if len(diffs) > 0 { @@ -106,6 +167,6 @@ func checkJobParity(t *testing.T, seed int64) { for _, d := range diffs { t.Errorf(" %s", d) } - t.Logf("reproduce with: FUZZ_SEED=%d go test ./bundle/fuzz -run TestJobCreateParity\n%s", seed, jobJSON) + t.Logf("reproduce with: FUZZ_SEED=%d task test-fuzz\nonce fixed, add %d to regressionSeeds in bundle/fuzz/fuzz_test.go\n%s", seed, seed, jobJSON) } } diff --git a/bundle/fuzz/capture.go b/bundle/fuzz/recorder_test.go similarity index 86% rename from bundle/fuzz/capture.go rename to bundle/fuzz/recorder_test.go index fe10bc10be8..244cb81480f 100644 --- a/bundle/fuzz/capture.go +++ b/bundle/fuzz/recorder_test.go @@ -10,13 +10,13 @@ import ( // jobsCreatePath is the Jobs API route both engines must hit on create. The // direct engine posts here via the SDK and the terraform provider is expected to // as well. The testserver registers only this exact route, so if an engine ever -// posted to a different version the deploy would 404 and CaptureJobCreate would +// posted to a different version the deploy would 404 and captureJobCreate would // fail with "did not POST". A version skew therefore surfaces as a capture // failure, not as a payload diff. const jobsCreatePath = "/api/2.2/jobs/create" -// CapturedRequest is a single mutating API request observed by the testserver. -type CapturedRequest struct { +// capturedRequest is a single mutating API request observed by the testserver. +type capturedRequest struct { Method string Path string Body json.RawMessage @@ -27,7 +27,7 @@ type CapturedRequest struct { // goroutines. type recorder struct { mu sync.Mutex - requests []CapturedRequest + requests []capturedRequest } func (r *recorder) callback(req *testserver.Request) { @@ -40,7 +40,7 @@ func (r *recorder) callback(req *testserver.Request) { body = append(json.RawMessage(nil), req.Body...) } - r.requests = append(r.requests, CapturedRequest{ + r.requests = append(r.requests, capturedRequest{ Method: req.Method, Path: req.URL.Path, Body: body, From a69b57439a56cc2978548e735d4db4aaba492127 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 24 Jun 2026 13:34:51 +0000 Subject: [PATCH 08/53] bundle/fuzz: make the whole package test-only and harden parity reporting Move the remaining generator/diff/rand implementation into _test.go files (keeping only a doc.go for the package comment) so nothing in the harness compiles into the regular build, since no product code imports it. Distinguish deploy/capture failures from create-payload divergences in checkJobParity: skip when neither engine deploys the generated config, fail distinctly when exactly one engine accepts it (an acceptance divergence, not a payload diff), and only diff payloads when both deploys succeed. This keeps nightly triage from misdirecting a deploy failure into regressionSeeds. Also document the unique-identity-key assumption in diffKeyedSlice. --- bundle/fuzz/compare.go | 268 ----------------- bundle/fuzz/compare_cases_test.go | 119 ++++++++ bundle/fuzz/compare_test.go | 374 +++++++++++++++++------- bundle/fuzz/deploy_test.go | 15 - bundle/fuzz/doc.go | 17 ++ bundle/fuzz/fuzz_test.go | 27 +- bundle/fuzz/generate.go | 356 ---------------------- bundle/fuzz/generate_invariants_test.go | 47 +++ bundle/fuzz/generate_test.go | 358 +++++++++++++++++++++-- bundle/fuzz/{rand.go => rand_test.go} | 0 10 files changed, 800 insertions(+), 781 deletions(-) delete mode 100644 bundle/fuzz/compare.go create mode 100644 bundle/fuzz/compare_cases_test.go create mode 100644 bundle/fuzz/doc.go delete mode 100644 bundle/fuzz/generate.go create mode 100644 bundle/fuzz/generate_invariants_test.go rename bundle/fuzz/{rand.go => rand_test.go} (100%) diff --git a/bundle/fuzz/compare.go b/bundle/fuzz/compare.go deleted file mode 100644 index 81c1bc7afb4..00000000000 --- a/bundle/fuzz/compare.go +++ /dev/null @@ -1,268 +0,0 @@ -package fuzz - -import ( - "bytes" - "encoding/json" - "fmt" - "regexp" - "slices" - "strconv" - "strings" -) - -// Difference is a single mismatch between the two engines' create payloads, -// located by a JSON-ish path (e.g. "tasks[0].new_cluster.num_workers"). -type Difference struct { - Path string - Direct any - Terraform any -} - -func (d Difference) String() string { - return fmt.Sprintf("%s: direct=%s terraform=%s", d.Path, render(d.Direct), render(d.Terraform)) -} - -// missing marks a value that is absent on one side. -type missing struct{} - -func render(v any) string { - if _, ok := v.(missing); ok { - return "" - } - b, err := json.Marshal(v) - if err != nil { - return fmt.Sprintf("%v", v) - } - return string(b) -} - -// DiffPayloads decodes both create payloads and returns every difference whose -// path is not explicitly ignored. ignorePaths are matched exactly against the -// rendered path, with "[*]" standing in for any slice index. -func DiffPayloads(direct, terraform json.RawMessage, ignorePaths []string) ([]Difference, error) { - d, err := decode(direct) - if err != nil { - return nil, fmt.Errorf("decoding direct payload: %w", err) - } - tf, err := decode(terraform) - if err != nil { - return nil, fmt.Errorf("decoding terraform payload: %w", err) - } - - var diffs []Difference - diffValue("", d, tf, &diffs) - - ignore := make(map[string]bool, len(ignorePaths)) - for _, p := range ignorePaths { - ignore[p] = true - } - - filtered := diffs[:0] - for _, diff := range diffs { - if !ignore[normalizePath(diff.Path)] { - filtered = append(filtered, diff) - } - } - return filtered, nil -} - -// decode unmarshals JSON using UseNumber so large int64 values (e.g. job ids, -// spark_context_id) are not corrupted by float64 rounding. See the encoding rule -// in the repo style guide. -func decode(raw json.RawMessage) (any, error) { - if len(raw) == 0 { - return nil, nil - } - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - var v any - if err := dec.Decode(&v); err != nil { - return nil, err - } - return v, nil -} - -func diffValue(path string, a, b any, diffs *[]Difference) { - switch av := a.(type) { - case map[string]any: - bv, ok := b.(map[string]any) - if !ok { - *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) - return - } - keys := unionKeys(av, bv) - for _, k := range keys { - achild, aok := av[k] - bchild, bok := bv[k] - child := joinKey(path, k) - switch { - case aok && bok: - diffValue(child, achild, bchild, diffs) - case aok: - *diffs = append(*diffs, Difference{Path: child, Direct: achild, Terraform: missing{}}) - default: - *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bchild}) - } - } - case []any: - bv, ok := b.([]any) - if !ok { - *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) - return - } - // Slices whose elements carry a natural identity key (tasks, job clusters) - // are matched by that key so an engine emitting the same elements in a - // different order is not reported as a difference. Everything else is - // compared positionally. - if key := identityKey(av, bv); key != "" { - diffKeyedSlice(path, key, av, bv, diffs) - return - } - n := max(len(av), len(bv)) - for i := range n { - child := fmt.Sprintf("%s[%d]", path, i) - switch { - case i < len(av) && i < len(bv): - diffValue(child, av[i], bv[i], diffs) - case i < len(av): - *diffs = append(*diffs, Difference{Path: child, Direct: av[i], Terraform: missing{}}) - default: - *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bv[i]}) - } - } - default: - if !scalarEqual(a, b) { - *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) - } - } -} - -// identityFields are the keys, in priority order, that uniquely identify the -// elements of a payload slice. Job tasks and shared job clusters are the slices -// whose order is not significant but which the engines may emit differently. -var identityFields = []string{"task_key", "job_cluster_key"} - -// identityKey returns the field that identifies every element of both slices, or -// "" if the elements are not uniformly keyed objects (in which case the caller -// falls back to positional comparison). -func identityKey(a, b []any) string { - for _, field := range identityFields { - if allHaveKey(a, field) && allHaveKey(b, field) { - return field - } - } - return "" -} - -func allHaveKey(s []any, field string) bool { - if len(s) == 0 { - return false - } - for _, el := range s { - m, ok := el.(map[string]any) - if !ok { - return false - } - if _, ok := m[field].(string); !ok { - return false - } - } - return true -} - -// diffKeyedSlice matches elements of a and b by the value of key (which is unique -// within each slice for tasks/job clusters) and diffs each matched pair, -// reporting unmatched elements as present-on-one-side. Paths keep numeric indices -// so ignore-path [*] normalization still applies. -func diffKeyedSlice(path, key string, a, b []any, diffs *[]Difference) { - bByKey := make(map[string]any, len(b)) - for _, el := range b { - bByKey[el.(map[string]any)[key].(string)] = el - } - - matched := make(map[string]bool, len(a)) - for i, el := range a { - child := fmt.Sprintf("%s[%d]", path, i) - k := el.(map[string]any)[key].(string) - matched[k] = true - if bel, ok := bByKey[k]; ok { - diffValue(child, el, bel, diffs) - } else { - *diffs = append(*diffs, Difference{Path: child, Direct: el, Terraform: missing{}}) - } - } - for j, el := range b { - k := el.(map[string]any)[key].(string) - if matched[k] { - continue - } - child := fmt.Sprintf("%s[%d]", path, j) - *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: el}) - } -} - -// scalarEqual compares two JSON scalars. json.Number is compared by its string -// form so 1 and 1.0 don't masquerade as equal across engines. -func scalarEqual(a, b any) bool { - an, aok := a.(json.Number) - bn, bok := b.(json.Number) - if aok && bok { - return an.String() == bn.String() - } - return a == b -} - -func unionKeys(a, b map[string]any) []string { - seen := map[string]bool{} - var keys []string - for k := range a { - if !seen[k] { - seen[k] = true - keys = append(keys, k) - } - } - for k := range b { - if !seen[k] { - seen[k] = true - keys = append(keys, k) - } - } - slices.Sort(keys) - return keys -} - -func joinKey(path, key string) string { - // Map keys can themselves contain dots or brackets (e.g. spark_conf entries - // like "spark.databricks.delta.preview.enabled"). Render those as bracketed, - // quoted segments so the path stays unambiguous and ignore entries can target - // a single key. - if key == "" || strings.ContainsAny(key, `.[]"`) { - return path + "[" + strconv.Quote(key) + "]" - } - if path == "" { - return key - } - return path + "." + key -} - -// indexRe matches numeric slice indices like "[12]" but not quoted string keys -// like ["spark.x"]. -var indexRe = regexp.MustCompile(`\[\d+\]`) - -// normalizePath replaces concrete slice indices with [*] so a single ignore -// entry can cover every element of a slice. -func normalizePath(path string) string { - return indexRe.ReplaceAllString(path, "[*]") -} - -// DefaultIgnorePaths lists create-payload paths that legitimately differ between -// the engines and are not parity bugs. Keep this list small and well-justified; -// every entry is a known, intentional divergence. -var DefaultIgnorePaths = []string{ - // The terraform provider strips the deprecated/ignored spark conf - // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while - // the direct engine forwards it verbatim. The backend ignores the key either - // way, so this is a benign provider-side filter rather than a parity bug. - `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, - `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, -} diff --git a/bundle/fuzz/compare_cases_test.go b/bundle/fuzz/compare_cases_test.go new file mode 100644 index 00000000000..46e506d75c6 --- /dev/null +++ b/bundle/fuzz/compare_cases_test.go @@ -0,0 +1,119 @@ +package fuzz + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiffPayloads(t *testing.T) { + tests := []struct { + name string + direct string + terraform string + ignore []string + want []string + }{ + { + name: "identical", + direct: `{"name":"a","tasks":[{"task_key":"t"}]}`, + terraform: `{"name":"a","tasks":[{"task_key":"t"}]}`, + want: nil, + }, + { + name: "scalar mismatch", + direct: `{"name":"a"}`, + terraform: `{"name":"b"}`, + want: []string{"name"}, + }, + { + name: "missing on terraform", + direct: `{"name":"a","queue":{"enabled":true}}`, + terraform: `{"name":"a"}`, + want: []string{"queue"}, + }, + { + name: "missing on direct", + direct: `{"name":"a"}`, + terraform: `{"name":"a","max_concurrent_runs":1}`, + want: []string{"max_concurrent_runs"}, + }, + { + name: "nested slice element mismatch", + direct: `{"tasks":[{"task_key":"t","timeout_seconds":1}]}`, + terraform: `{"tasks":[{"task_key":"t","timeout_seconds":2}]}`, + want: []string{"tasks[0].timeout_seconds"}, + }, + { + name: "slice length mismatch", + direct: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + terraform: `{"tasks":[{"task_key":"a"}]}`, + want: []string{"tasks[1]"}, + }, + { + name: "number 1 vs 1.0 differ", + direct: `{"n":1}`, + terraform: `{"n":1.0}`, + want: []string{"n"}, + }, + { + name: "ignored path", + direct: `{"tasks":[{"timeout_seconds":1}]}`, + terraform: `{"tasks":[{"timeout_seconds":2}]}`, + ignore: []string{"tasks[*].timeout_seconds"}, + want: nil, + }, + { + name: "dotted map key is bracket-quoted", + direct: `{"spark_conf":{"spark.x.y":"1"}}`, + terraform: `{"spark_conf":{}}`, + want: []string{`spark_conf["spark.x.y"]`}, + }, + { + name: "dotted map key can be ignored", + direct: `{"c":{"spark_conf":{"spark.x.y":"1"}}}`, + terraform: `{"c":{"spark_conf":{}}}`, + ignore: []string{`c.spark_conf["spark.x.y"]`}, + want: nil, + }, + { + name: "tasks matched by key ignore order", + direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, + terraform: `{"tasks":[{"task_key":"b","timeout_seconds":2},{"task_key":"a","timeout_seconds":1}]}`, + want: nil, + }, + { + name: "tasks matched by key surface real diff at direct index", + direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, + terraform: `{"tasks":[{"task_key":"b","timeout_seconds":9},{"task_key":"a","timeout_seconds":1}]}`, + want: []string{"tasks[1].timeout_seconds"}, + }, + { + name: "task only on terraform reported at its index", + direct: `{"tasks":[{"task_key":"a"}]}`, + terraform: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + want: []string{"tasks[1]"}, + }, + { + name: "job_clusters matched by key ignore order", + direct: `{"job_clusters":[{"job_cluster_key":"x","new_cluster":{"num_workers":1}},{"job_cluster_key":"y","new_cluster":{"num_workers":2}}]}`, + terraform: `{"job_clusters":[{"job_cluster_key":"y","new_cluster":{"num_workers":2}},{"job_cluster_key":"x","new_cluster":{"num_workers":1}}]}`, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + diffs, err := DiffPayloads(json.RawMessage(tt.direct), json.RawMessage(tt.terraform), tt.ignore) + require.NoError(t, err) + + var paths []string + for _, d := range diffs { + paths = append(paths, d.Path) + } + assert.ElementsMatch(t, tt.want, paths) + }) + } +} diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go index 46e506d75c6..fd6807b56cc 100644 --- a/bundle/fuzz/compare_test.go +++ b/bundle/fuzz/compare_test.go @@ -1,119 +1,273 @@ package fuzz import ( + "bytes" "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "fmt" + "regexp" + "slices" + "strconv" + "strings" ) -func TestDiffPayloads(t *testing.T) { - tests := []struct { - name string - direct string - terraform string - ignore []string - want []string - }{ - { - name: "identical", - direct: `{"name":"a","tasks":[{"task_key":"t"}]}`, - terraform: `{"name":"a","tasks":[{"task_key":"t"}]}`, - want: nil, - }, - { - name: "scalar mismatch", - direct: `{"name":"a"}`, - terraform: `{"name":"b"}`, - want: []string{"name"}, - }, - { - name: "missing on terraform", - direct: `{"name":"a","queue":{"enabled":true}}`, - terraform: `{"name":"a"}`, - want: []string{"queue"}, - }, - { - name: "missing on direct", - direct: `{"name":"a"}`, - terraform: `{"name":"a","max_concurrent_runs":1}`, - want: []string{"max_concurrent_runs"}, - }, - { - name: "nested slice element mismatch", - direct: `{"tasks":[{"task_key":"t","timeout_seconds":1}]}`, - terraform: `{"tasks":[{"task_key":"t","timeout_seconds":2}]}`, - want: []string{"tasks[0].timeout_seconds"}, - }, - { - name: "slice length mismatch", - direct: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - terraform: `{"tasks":[{"task_key":"a"}]}`, - want: []string{"tasks[1]"}, - }, - { - name: "number 1 vs 1.0 differ", - direct: `{"n":1}`, - terraform: `{"n":1.0}`, - want: []string{"n"}, - }, - { - name: "ignored path", - direct: `{"tasks":[{"timeout_seconds":1}]}`, - terraform: `{"tasks":[{"timeout_seconds":2}]}`, - ignore: []string{"tasks[*].timeout_seconds"}, - want: nil, - }, - { - name: "dotted map key is bracket-quoted", - direct: `{"spark_conf":{"spark.x.y":"1"}}`, - terraform: `{"spark_conf":{}}`, - want: []string{`spark_conf["spark.x.y"]`}, - }, - { - name: "dotted map key can be ignored", - direct: `{"c":{"spark_conf":{"spark.x.y":"1"}}}`, - terraform: `{"c":{"spark_conf":{}}}`, - ignore: []string{`c.spark_conf["spark.x.y"]`}, - want: nil, - }, - { - name: "tasks matched by key ignore order", - direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, - terraform: `{"tasks":[{"task_key":"b","timeout_seconds":2},{"task_key":"a","timeout_seconds":1}]}`, - want: nil, - }, - { - name: "tasks matched by key surface real diff at direct index", - direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, - terraform: `{"tasks":[{"task_key":"b","timeout_seconds":9},{"task_key":"a","timeout_seconds":1}]}`, - want: []string{"tasks[1].timeout_seconds"}, - }, - { - name: "task only on terraform reported at its index", - direct: `{"tasks":[{"task_key":"a"}]}`, - terraform: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - want: []string{"tasks[1]"}, - }, - { - name: "job_clusters matched by key ignore order", - direct: `{"job_clusters":[{"job_cluster_key":"x","new_cluster":{"num_workers":1}},{"job_cluster_key":"y","new_cluster":{"num_workers":2}}]}`, - terraform: `{"job_clusters":[{"job_cluster_key":"y","new_cluster":{"num_workers":2}},{"job_cluster_key":"x","new_cluster":{"num_workers":1}}]}`, - want: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - diffs, err := DiffPayloads(json.RawMessage(tt.direct), json.RawMessage(tt.terraform), tt.ignore) - require.NoError(t, err) - - var paths []string - for _, d := range diffs { - paths = append(paths, d.Path) +// Difference is a single mismatch between the two engines' create payloads, +// located by a JSON-ish path (e.g. "tasks[0].new_cluster.num_workers"). +type Difference struct { + Path string + Direct any + Terraform any +} + +func (d Difference) String() string { + return fmt.Sprintf("%s: direct=%s terraform=%s", d.Path, render(d.Direct), render(d.Terraform)) +} + +// missing marks a value that is absent on one side. +type missing struct{} + +func render(v any) string { + if _, ok := v.(missing); ok { + return "" + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Sprintf("%v", v) + } + return string(b) +} + +// DiffPayloads decodes both create payloads and returns every difference whose +// path is not explicitly ignored. ignorePaths are matched exactly against the +// rendered path, with "[*]" standing in for any slice index. +func DiffPayloads(direct, terraform json.RawMessage, ignorePaths []string) ([]Difference, error) { + d, err := decode(direct) + if err != nil { + return nil, fmt.Errorf("decoding direct payload: %w", err) + } + tf, err := decode(terraform) + if err != nil { + return nil, fmt.Errorf("decoding terraform payload: %w", err) + } + + var diffs []Difference + diffValue("", d, tf, &diffs) + + ignore := make(map[string]bool, len(ignorePaths)) + for _, p := range ignorePaths { + ignore[p] = true + } + + filtered := diffs[:0] + for _, diff := range diffs { + if !ignore[normalizePath(diff.Path)] { + filtered = append(filtered, diff) + } + } + return filtered, nil +} + +// decode unmarshals JSON using UseNumber so large int64 values (e.g. job ids, +// spark_context_id) are not corrupted by float64 rounding. See the encoding rule +// in the repo style guide. +func decode(raw json.RawMessage) (any, error) { + if len(raw) == 0 { + return nil, nil + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + return v, nil +} + +func diffValue(path string, a, b any, diffs *[]Difference) { + switch av := a.(type) { + case map[string]any: + bv, ok := b.(map[string]any) + if !ok { + *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + return + } + keys := unionKeys(av, bv) + for _, k := range keys { + achild, aok := av[k] + bchild, bok := bv[k] + child := joinKey(path, k) + switch { + case aok && bok: + diffValue(child, achild, bchild, diffs) + case aok: + *diffs = append(*diffs, Difference{Path: child, Direct: achild, Terraform: missing{}}) + default: + *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bchild}) + } + } + case []any: + bv, ok := b.([]any) + if !ok { + *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + return + } + // Slices whose elements carry a natural identity key (tasks, job clusters) + // are matched by that key so an engine emitting the same elements in a + // different order is not reported as a difference. Everything else is + // compared positionally. + if key := identityKey(av, bv); key != "" { + diffKeyedSlice(path, key, av, bv, diffs) + return + } + n := max(len(av), len(bv)) + for i := range n { + child := fmt.Sprintf("%s[%d]", path, i) + switch { + case i < len(av) && i < len(bv): + diffValue(child, av[i], bv[i], diffs) + case i < len(av): + *diffs = append(*diffs, Difference{Path: child, Direct: av[i], Terraform: missing{}}) + default: + *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bv[i]}) } - assert.ElementsMatch(t, tt.want, paths) - }) + } + default: + if !scalarEqual(a, b) { + *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + } + } +} + +// identityFields are the keys, in priority order, that uniquely identify the +// elements of a payload slice. Job tasks and shared job clusters are the slices +// whose order is not significant but which the engines may emit differently. +var identityFields = []string{"task_key", "job_cluster_key"} + +// identityKey returns the field that identifies every element of both slices, or +// "" if the elements are not uniformly keyed objects (in which case the caller +// falls back to positional comparison). +func identityKey(a, b []any) string { + for _, field := range identityFields { + if allHaveKey(a, field) && allHaveKey(b, field) { + return field + } + } + return "" +} + +func allHaveKey(s []any, field string) bool { + if len(s) == 0 { + return false + } + for _, el := range s { + m, ok := el.(map[string]any) + if !ok { + return false + } + if _, ok := m[field].(string); !ok { + return false + } + } + return true +} + +// diffKeyedSlice matches elements of a and b by the value of key (which is unique +// within each slice for tasks/job clusters) and diffs each matched pair, +// reporting unmatched elements as present-on-one-side. Paths keep numeric indices +// so ignore-path [*] normalization still applies. +func diffKeyedSlice(path, key string, a, b []any, diffs *[]Difference) { + // identityFields are unique within a slice by API contract (no two job tasks + // share a task_key, no two job_clusters share a job_cluster_key), so keying by + // them is unambiguous. If a payload ever repeated a key, last-one-wins here and + // the duplicate would be mismatched rather than reported precisely; callers + // outside the job-create harness must not rely on this for non-unique keys. + bByKey := make(map[string]any, len(b)) + for _, el := range b { + bByKey[el.(map[string]any)[key].(string)] = el + } + + matched := make(map[string]bool, len(a)) + for i, el := range a { + child := fmt.Sprintf("%s[%d]", path, i) + k := el.(map[string]any)[key].(string) + matched[k] = true + if bel, ok := bByKey[k]; ok { + diffValue(child, el, bel, diffs) + } else { + *diffs = append(*diffs, Difference{Path: child, Direct: el, Terraform: missing{}}) + } + } + for j, el := range b { + k := el.(map[string]any)[key].(string) + if matched[k] { + continue + } + child := fmt.Sprintf("%s[%d]", path, j) + *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: el}) + } +} + +// scalarEqual compares two JSON scalars. json.Number is compared by its string +// form so 1 and 1.0 don't masquerade as equal across engines. +func scalarEqual(a, b any) bool { + an, aok := a.(json.Number) + bn, bok := b.(json.Number) + if aok && bok { + return an.String() == bn.String() } + return a == b +} + +func unionKeys(a, b map[string]any) []string { + seen := map[string]bool{} + var keys []string + for k := range a { + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + } + for k := range b { + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + } + slices.Sort(keys) + return keys +} + +func joinKey(path, key string) string { + // Map keys can themselves contain dots or brackets (e.g. spark_conf entries + // like "spark.databricks.delta.preview.enabled"). Render those as bracketed, + // quoted segments so the path stays unambiguous and ignore entries can target + // a single key. + if key == "" || strings.ContainsAny(key, `.[]"`) { + return path + "[" + strconv.Quote(key) + "]" + } + if path == "" { + return key + } + return path + "." + key +} + +// indexRe matches numeric slice indices like "[12]" but not quoted string keys +// like ["spark.x"]. +var indexRe = regexp.MustCompile(`\[\d+\]`) + +// normalizePath replaces concrete slice indices with [*] so a single ignore +// entry can cover every element of a slice. +func normalizePath(path string) string { + return indexRe.ReplaceAllString(path, "[*]") +} + +// DefaultIgnorePaths lists create-payload paths that legitimately differ between +// the engines and are not parity bugs. Keep this list small and well-justified; +// every entry is a known, intentional divergence. +var DefaultIgnorePaths = []string{ + // The terraform provider strips the deprecated/ignored spark conf + // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while + // the direct engine forwards it verbatim. The backend ignores the key either + // way, so this is a benign provider-side filter rather than a parity bug. + `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, + `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, } diff --git a/bundle/fuzz/deploy_test.go b/bundle/fuzz/deploy_test.go index e42dbb74346..2328e0354e2 100644 --- a/bundle/fuzz/deploy_test.go +++ b/bundle/fuzz/deploy_test.go @@ -61,21 +61,6 @@ func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, eng return body, nil } -// compareJobEngines deploys job under both engines and returns the create-payload -// differences that are not covered by DefaultIgnorePaths. An empty result means -// the engines produced equivalent create payloads. -func compareJobEngines(ctx context.Context, t *testing.T, job *resources.Job) ([]Difference, error) { - direct, err := captureJobCreate(ctx, t, job, "direct") - if err != nil { - return nil, fmt.Errorf("capturing direct payload: %w", err) - } - terraform, err := captureJobCreate(ctx, t, job, "terraform") - if err != nil { - return nil, fmt.Errorf("capturing terraform payload: %w", err) - } - return DiffPayloads(direct, terraform, DefaultIgnorePaths) -} - // writeJobBundle writes a minimal databricks.yml describing a single job. The // document is emitted as JSON, which is valid YAML, so we can reuse the job's // own JSON marshaling (which honors ForceSendFields) without a YAML dependency. diff --git a/bundle/fuzz/doc.go b/bundle/fuzz/doc.go new file mode 100644 index 00000000000..cf898d3ec14 --- /dev/null +++ b/bundle/fuzz/doc.go @@ -0,0 +1,17 @@ +// Package fuzz provides randomized generators and harnesses that compare how the +// terraform and direct deploy engines translate the same bundle resource into an +// API create payload. See DECO-25361. +// +// The first technique implemented here generates a random resource config and +// checks for differences in the create payload between the terraform and direct +// engines. Generators are seeded so that any divergence found by the fuzz driver +// can be reproduced from the printed seed. +// +// Only jobs are covered for now. Extending the harness to other resource kinds +// (pipelines, apps, ...) is tracked as follow-up work under DECO-25361. +// +// Everything else in the package lives in _test.go files: the package is a +// test-only utility and nothing in the product imports it, so keeping the logic +// out of the regular build avoids shipping dead code. This file exists only to +// carry the package documentation in a non-test file. +package fuzz diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 7b0d0df8ea3..88a3c5a3b6d 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -154,12 +154,35 @@ func FuzzJobCreateParity(f *testing.F) { // checkJobParity generates the job for seed, deploys it under both engines, and // fails the test with reproduction details if the create payloads diverge. +// +// A deploy/capture failure is not a create-payload divergence, so the three +// outcomes are handled distinctly to keep nightly triage from misdirecting a +// deploy failure into regressionSeeds (which is only for real payload diffs): +// - neither engine deployed: the generator produced a config nothing accepts, +// so skip (logging both errors) rather than flag a parity bug. +// - exactly one engine deployed: the engines disagree on whether the config is +// even valid. That is a real divergence worth failing on, but an acceptance +// divergence, not a payload diff, so it is reported as such. +// - both deployed: compare the captured create payloads. func checkJobParity(t *testing.T, seed int64) { t.Helper() job := GenerateJob(newRNG(seed)) - diffs, err := compareJobEngines(t.Context(), t, job) - require.NoErrorf(t, err, "seed %d", seed) + ctx := t.Context() + direct, directErr := captureJobCreate(ctx, t, job, "direct") + terraform, tfErr := captureJobCreate(ctx, t, job, "terraform") + + switch { + case directErr != nil && tfErr != nil: + t.Skipf("seed %d: config did not deploy under either engine (not a parity divergence)\ndirect: %v\nterraform: %v", seed, directErr, tfErr) + case directErr != nil: + t.Fatalf("seed %d: direct rejected a config terraform accepted (engine acceptance divergence, not a payload diff): %v", seed, directErr) + case tfErr != nil: + t.Fatalf("seed %d: terraform rejected a config direct accepted (engine acceptance divergence, not a payload diff): %v", seed, tfErr) + } + + diffs, err := DiffPayloads(direct, terraform, DefaultIgnorePaths) + require.NoErrorf(t, err, "seed %d: comparing create payloads", seed) if len(diffs) > 0 { jobJSON, _ := json.MarshalIndent(job, "", " ") diff --git a/bundle/fuzz/generate.go b/bundle/fuzz/generate.go deleted file mode 100644 index 697748e03ff..00000000000 --- a/bundle/fuzz/generate.go +++ /dev/null @@ -1,356 +0,0 @@ -// Package fuzz provides randomized generators and harnesses that compare how the -// terraform and direct deploy engines translate the same bundle resource into an -// API create payload. See DECO-25361. -// -// The first technique implemented here generates a random resource config and -// checks for differences in the create payload between the terraform and direct -// engines. Generators are seeded so that any divergence found by the fuzz driver -// can be reproduced from the printed seed. -// -// Only jobs are covered for now. Extending the harness to other resource kinds -// (pipelines, apps, ...) is tracked as follow-up work under DECO-25361. -package fuzz - -import ( - "fmt" - "math/rand/v2" - "strconv" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" -) - -// Value pools are intentionally small and valid-looking: the goal is to exercise -// the engines' config->payload translation across many field combinations, not to -// stress the API with invalid values (which the testserver would reject before we -// can compare payloads). -var ( - sparkVersions = []string{"13.3.x-scala2.12", "14.3.x-scala2.12", "15.4.x-scala2.12", "16.4.x-scala2.12"} - nodeTypeIDs = []string{"i3.xlarge", "m5.large", "r5.xlarge", "Standard_DS3_v2"} - timezones = []string{"UTC", "America/Los_Angeles", "Europe/Amsterdam"} - cronExprs = []string{"0 0 12 * * ?", "0 15 10 ? * MON-FRI", "0 0/30 * * * ?"} - pauseStatuses = []jobs.PauseStatus{jobs.PauseStatusPaused, jobs.PauseStatusUnpaused} - performance = []jobs.PerformanceTarget{jobs.PerformanceTargetPerformanceOptimized, jobs.PerformanceTargetStandard} - timeUnits = []string{"HOURS", "DAYS", "WEEKS"} - healthMetrics = []string{"RUN_DURATION_SECONDS", "STREAMING_BACKLOG_BYTES", "STREAMING_BACKLOG_RECORDS"} - conditionOps = []string{"EQUAL_TO", "NOT_EQUAL", "GREATER_THAN", "LESS_THAN_OR_EQUAL"} - runIfs = []string{"ALL_SUCCESS", "AT_LEAST_ONE_SUCCESS", "NONE_FAILED", "ALL_DONE"} - gitProviders = []jobs.GitProvider{jobs.GitProviderGitHub, jobs.GitProviderGitLab, jobs.GitProviderAzureDevOpsServices} -) - -// GenerateJob builds a random, well-formed job config driven entirely by rng, so -// the same seed always produces the same job. It deliberately favors fields whose -// translation tends to differ between engines (tasks, clusters, schedules, -// notifications, tags, zero-able scalars). -// -// TODO(DECO-25361): generalize the harness across resource kinds so pipelines, -// apps, etc. get the same create-payload parity coverage as jobs. -func GenerateJob(rng *rand.Rand) *resources.Job { - job := &resources.Job{} - job.Name = randName(rng, "job") - - if chance(rng, 0.5) { - job.Description = randSentence(rng) - } - if chance(rng, 0.4) { - job.MaxConcurrentRuns = rng.IntN(10) + 1 - } - if chance(rng, 0.4) { - job.TimeoutSeconds = rng.IntN(7200) - } - if chance(rng, 0.3) { - job.PerformanceTarget = oneOf(rng, performance) - } - if chance(rng, 0.5) { - job.Tags = randTags(rng) - } - if chance(rng, 0.3) { - job.GitSource = randGitSource(rng) - } - - randScheduling(rng, job) - - if chance(rng, 0.3) { - job.EmailNotifications = randEmailNotifications(rng) - } - if chance(rng, 0.2) { - job.WebhookNotifications = randWebhookNotifications(rng) - } - if chance(rng, 0.3) { - job.NotificationSettings = &jobs.JobNotificationSettings{ - NoAlertForCanceledRuns: chance(rng, 0.5), - NoAlertForSkippedRuns: chance(rng, 0.5), - } - } - if chance(rng, 0.3) { - job.Health = randHealth(rng) - } - if chance(rng, 0.3) { - job.Parameters = randParameters(rng) - } - if chance(rng, 0.3) { - job.Queue = &jobs.QueueSettings{Enabled: chance(rng, 0.5)} - } - - // Generate shared job clusters first so tasks can reference them by key. - var jobClusterKeys []string - if chance(rng, 0.5) { - n := rng.IntN(2) + 1 - for i := range n { - key := fmt.Sprintf("cluster_%d", i) - jobClusterKeys = append(jobClusterKeys, key) - job.JobClusters = append(job.JobClusters, jobs.JobCluster{ - JobClusterKey: key, - NewCluster: randClusterSpec(rng), - }) - } - } - - nTasks := rng.IntN(3) + 1 - var taskKeys []string - for i := range nTasks { - task := randTask(rng, i, jobClusterKeys) - // Randomly chain dependencies onto previously generated tasks. - if len(taskKeys) > 0 && chance(rng, 0.4) { - dep := taskKeys[rng.IntN(len(taskKeys))] - task.DependsOn = []jobs.TaskDependency{{TaskKey: dep}} - if chance(rng, 0.5) { - task.RunIf = jobs.RunIf(oneOf(rng, runIfs)) - } - } - taskKeys = append(taskKeys, task.TaskKey) - job.Tasks = append(job.Tasks, task) - } - - return job -} - -// randScheduling sets at most one of schedule/trigger/continuous, which are -// mutually exclusive ways to launch a job. -func randScheduling(rng *rand.Rand, job *resources.Job) { - switch rng.IntN(5) { - case 0: - job.Schedule = &jobs.CronSchedule{ - QuartzCronExpression: oneOf(rng, cronExprs), - TimezoneId: oneOf(rng, timezones), - PauseStatus: oneOf(rng, pauseStatuses), - } - case 1: - job.Trigger = &jobs.TriggerSettings{ - PauseStatus: oneOf(rng, pauseStatuses), - Periodic: &jobs.PeriodicTriggerConfiguration{ - Interval: rng.IntN(12) + 1, - Unit: jobs.PeriodicTriggerConfigurationTimeUnit(oneOf(rng, timeUnits)), - }, - } - case 2: - job.Trigger = &jobs.TriggerSettings{ - PauseStatus: oneOf(rng, pauseStatuses), - FileArrival: &jobs.FileArrivalTriggerConfiguration{ - Url: "s3://" + randWord(rng) + "/" + randWord(rng), - }, - } - case 3: - job.Continuous = &jobs.Continuous{PauseStatus: oneOf(rng, pauseStatuses)} - default: - // no scheduling - } -} - -func randTask(rng *rand.Rand, idx int, jobClusterKeys []string) jobs.Task { - task := jobs.Task{TaskKey: fmt.Sprintf("task_%d", idx)} - - // Use absolute workspace paths with source=WORKSPACE so the generated bundle - // never depends on local files existing on disk (which deploy would reject). - // condition_task needs no compute, so it is handled separately below. - needsCompute := true - switch rng.IntN(4) { - case 0: - task.NotebookTask = &jobs.NotebookTask{ - NotebookPath: "/Workspace/Users/test/" + randName(rng, "nb"), - Source: jobs.SourceWorkspace, - } - case 1: - task.SparkPythonTask = &jobs.SparkPythonTask{ - PythonFile: "/Workspace/Users/test/" + randName(rng, "main") + ".py", - Source: jobs.SourceWorkspace, - } - case 2: - task.PythonWheelTask = &jobs.PythonWheelTask{ - PackageName: randName(rng, "pkg"), - EntryPoint: "main", - } - case 3: - task.ConditionTask = &jobs.ConditionTask{ - Left: randWord(rng), - Op: jobs.ConditionTaskOp(oneOf(rng, conditionOps)), - Right: randWord(rng), - } - needsCompute = false - } - - if needsCompute { - assignCompute(rng, &task, jobClusterKeys) - if chance(rng, 0.4) { - task.Libraries = randLibraries(rng) - } - } - - if chance(rng, 0.3) { - task.TimeoutSeconds = rng.IntN(3600) - } - if chance(rng, 0.3) { - task.MaxRetries = rng.IntN(5) - task.MinRetryIntervalMillis = rng.IntN(60000) - task.RetryOnTimeout = chance(rng, 0.5) - } - return task -} - -// assignCompute attaches exactly one compute source, which notebook/python/wheel -// tasks require: a shared job cluster (when available), a brand-new cluster, or an -// existing cluster id. -func assignCompute(rng *rand.Rand, task *jobs.Task, jobClusterKeys []string) { - const ( - computeNew = iota - computeExisting - computeShared - ) - options := []int{computeNew, computeExisting} - if len(jobClusterKeys) > 0 { - options = append(options, computeShared) - } - switch oneOf(rng, options) { - case computeNew: - spec := randClusterSpec(rng) - task.NewCluster = &spec - case computeExisting: - task.ExistingClusterId = randName(rng, "cluster") - case computeShared: - task.JobClusterKey = oneOf(rng, jobClusterKeys) - } -} - -func randClusterSpec(rng *rand.Rand) compute.ClusterSpec { - spec := compute.ClusterSpec{ - SparkVersion: oneOf(rng, sparkVersions), - NodeTypeId: oneOf(rng, nodeTypeIDs), - } - if chance(rng, 0.5) { - spec.NumWorkers = rng.IntN(8) - } else { - spec.Autoscale = &compute.AutoScale{ - MinWorkers: 1, - MaxWorkers: rng.IntN(8) + 2, - } - } - if chance(rng, 0.4) { - spec.SparkConf = map[string]string{ - "spark.databricks.delta.preview.enabled": "true", - "spark.speculation": strconv.FormatBool(chance(rng, 0.5)), - } - } - if chance(rng, 0.3) { - spec.CustomTags = randTags(rng) - } - if chance(rng, 0.3) { - spec.SparkEnvVars = map[string]string{"PYSPARK_PYTHON": "/databricks/python3/bin/python3"} - } - if chance(rng, 0.3) { - spec.DriverNodeTypeId = oneOf(rng, nodeTypeIDs) - } - return spec -} - -func randGitSource(rng *rand.Rand) *jobs.GitSource { - src := &jobs.GitSource{ - GitProvider: oneOf(rng, gitProviders), - GitUrl: "https://example.com/" + randWord(rng) + "/" + randWord(rng) + ".git", - } - switch rng.IntN(3) { - case 0: - src.GitBranch = oneOf(rng, []string{"main", "develop", "release"}) - case 1: - src.GitTag = "v" + fmt.Sprintf("%d.%d.0", rng.IntN(5), rng.IntN(10)) - case 2: - src.GitCommit = fmt.Sprintf("%040x", rng.Int64()) - } - return src -} - -func randEmailNotifications(rng *rand.Rand) *jobs.JobEmailNotifications { - email := randWord(rng) + "@example.com" - n := &jobs.JobEmailNotifications{NoAlertForSkippedRuns: chance(rng, 0.5)} - if chance(rng, 0.6) { - n.OnFailure = []string{email} - } - if chance(rng, 0.4) { - n.OnSuccess = []string{email} - } - if chance(rng, 0.3) { - n.OnStart = []string{email} - } - return n -} - -func randWebhookNotifications(rng *rand.Rand) *jobs.WebhookNotifications { - hook := []jobs.Webhook{{Id: randName(rng, "hook")}} - n := &jobs.WebhookNotifications{} - if chance(rng, 0.6) { - n.OnFailure = hook - } - if chance(rng, 0.4) { - n.OnSuccess = hook - } - return n -} - -func randHealth(rng *rand.Rand) *jobs.JobsHealthRules { - return &jobs.JobsHealthRules{ - Rules: []jobs.JobsHealthRule{ - { - Metric: jobs.JobsHealthMetric(oneOf(rng, healthMetrics)), - Op: jobs.JobsHealthOperatorGreaterThan, - Value: int64(rng.IntN(3600) + 1), - }, - }, - } -} - -func randLibraries(rng *rand.Rand) []compute.Library { - n := rng.IntN(2) + 1 - libs := make([]compute.Library, 0, n) - for range n { - switch rng.IntN(3) { - case 0: - libs = append(libs, compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: randWord(rng)}}) - case 1: - libs = append(libs, compute.Library{Maven: &compute.MavenLibrary{Coordinates: "org.example:" + randWord(rng) + ":1.0.0"}}) - case 2: - libs = append(libs, compute.Library{Whl: "/Workspace/Users/test/" + randName(rng, "lib") + ".whl"}) - } - } - return libs -} - -func randParameters(rng *rand.Rand) []jobs.JobParameterDefinition { - n := rng.IntN(3) + 1 - params := make([]jobs.JobParameterDefinition, 0, n) - for i := range n { - params = append(params, jobs.JobParameterDefinition{ - Name: fmt.Sprintf("param_%d", i), - Default: randWord(rng), - }) - } - return params -} - -func randTags(rng *rand.Rand) map[string]string { - n := rng.IntN(3) + 1 - tags := make(map[string]string, n) - for i := range n { - tags[fmt.Sprintf("tag_%d", i)] = randWord(rng) - } - return tags -} diff --git a/bundle/fuzz/generate_invariants_test.go b/bundle/fuzz/generate_invariants_test.go new file mode 100644 index 00000000000..f7a797e8f59 --- /dev/null +++ b/bundle/fuzz/generate_invariants_test.go @@ -0,0 +1,47 @@ +package fuzz + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateJobIsDeterministic(t *testing.T) { + a := GenerateJob(newRNG(42)) + b := GenerateJob(newRNG(42)) + assert.Equal(t, a, b, "same seed must produce identical job") +} + +func TestGenerateJobIsWellFormed(t *testing.T) { + for seed := range int64(200) { + job := GenerateJob(newRNG(seed)) + require.NotEmptyf(t, job.Name, "seed %d: job must have a name", seed) + require.NotEmptyf(t, job.Tasks, "seed %d: job must have at least one task", seed) + + clusterKeys := map[string]bool{} + for _, jc := range job.JobClusters { + clusterKeys[jc.JobClusterKey] = true + } + + taskKeys := map[string]bool{} + for _, task := range job.Tasks { + require.NotEmptyf(t, task.TaskKey, "seed %d: task must have a key", seed) + taskKeys[task.TaskKey] = true + + // A task referencing a job cluster must reference one we generated. + if task.JobClusterKey != "" { + assert.Containsf(t, clusterKeys, task.JobClusterKey, + "seed %d: task %q references unknown job cluster %q", seed, task.TaskKey, task.JobClusterKey) + } + } + + // Every dependency must point at a task that exists in this job. + for _, task := range job.Tasks { + for _, dep := range task.DependsOn { + assert.Containsf(t, taskKeys, dep.TaskKey, + "seed %d: task %q depends on unknown task %q", seed, task.TaskKey, dep.TaskKey) + } + } + } +} diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go index f7a797e8f59..1b0acf55b0f 100644 --- a/bundle/fuzz/generate_test.go +++ b/bundle/fuzz/generate_test.go @@ -1,47 +1,345 @@ package fuzz import ( - "testing" + "fmt" + "math/rand/v2" + "strconv" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" ) -func TestGenerateJobIsDeterministic(t *testing.T) { - a := GenerateJob(newRNG(42)) - b := GenerateJob(newRNG(42)) - assert.Equal(t, a, b, "same seed must produce identical job") -} +// Value pools are intentionally small and valid-looking: the goal is to exercise +// the engines' config->payload translation across many field combinations, not to +// stress the API with invalid values (which the testserver would reject before we +// can compare payloads). +var ( + sparkVersions = []string{"13.3.x-scala2.12", "14.3.x-scala2.12", "15.4.x-scala2.12", "16.4.x-scala2.12"} + nodeTypeIDs = []string{"i3.xlarge", "m5.large", "r5.xlarge", "Standard_DS3_v2"} + timezones = []string{"UTC", "America/Los_Angeles", "Europe/Amsterdam"} + cronExprs = []string{"0 0 12 * * ?", "0 15 10 ? * MON-FRI", "0 0/30 * * * ?"} + pauseStatuses = []jobs.PauseStatus{jobs.PauseStatusPaused, jobs.PauseStatusUnpaused} + performance = []jobs.PerformanceTarget{jobs.PerformanceTargetPerformanceOptimized, jobs.PerformanceTargetStandard} + timeUnits = []string{"HOURS", "DAYS", "WEEKS"} + healthMetrics = []string{"RUN_DURATION_SECONDS", "STREAMING_BACKLOG_BYTES", "STREAMING_BACKLOG_RECORDS"} + conditionOps = []string{"EQUAL_TO", "NOT_EQUAL", "GREATER_THAN", "LESS_THAN_OR_EQUAL"} + runIfs = []string{"ALL_SUCCESS", "AT_LEAST_ONE_SUCCESS", "NONE_FAILED", "ALL_DONE"} + gitProviders = []jobs.GitProvider{jobs.GitProviderGitHub, jobs.GitProviderGitLab, jobs.GitProviderAzureDevOpsServices} +) + +// GenerateJob builds a random, well-formed job config driven entirely by rng, so +// the same seed always produces the same job. It deliberately favors fields whose +// translation tends to differ between engines (tasks, clusters, schedules, +// notifications, tags, zero-able scalars). +// +// TODO(DECO-25361): generalize the harness across resource kinds so pipelines, +// apps, etc. get the same create-payload parity coverage as jobs. +func GenerateJob(rng *rand.Rand) *resources.Job { + job := &resources.Job{} + job.Name = randName(rng, "job") + + if chance(rng, 0.5) { + job.Description = randSentence(rng) + } + if chance(rng, 0.4) { + job.MaxConcurrentRuns = rng.IntN(10) + 1 + } + if chance(rng, 0.4) { + job.TimeoutSeconds = rng.IntN(7200) + } + if chance(rng, 0.3) { + job.PerformanceTarget = oneOf(rng, performance) + } + if chance(rng, 0.5) { + job.Tags = randTags(rng) + } + if chance(rng, 0.3) { + job.GitSource = randGitSource(rng) + } -func TestGenerateJobIsWellFormed(t *testing.T) { - for seed := range int64(200) { - job := GenerateJob(newRNG(seed)) - require.NotEmptyf(t, job.Name, "seed %d: job must have a name", seed) - require.NotEmptyf(t, job.Tasks, "seed %d: job must have at least one task", seed) + randScheduling(rng, job) - clusterKeys := map[string]bool{} - for _, jc := range job.JobClusters { - clusterKeys[jc.JobClusterKey] = true + if chance(rng, 0.3) { + job.EmailNotifications = randEmailNotifications(rng) + } + if chance(rng, 0.2) { + job.WebhookNotifications = randWebhookNotifications(rng) + } + if chance(rng, 0.3) { + job.NotificationSettings = &jobs.JobNotificationSettings{ + NoAlertForCanceledRuns: chance(rng, 0.5), + NoAlertForSkippedRuns: chance(rng, 0.5), } + } + if chance(rng, 0.3) { + job.Health = randHealth(rng) + } + if chance(rng, 0.3) { + job.Parameters = randParameters(rng) + } + if chance(rng, 0.3) { + job.Queue = &jobs.QueueSettings{Enabled: chance(rng, 0.5)} + } - taskKeys := map[string]bool{} - for _, task := range job.Tasks { - require.NotEmptyf(t, task.TaskKey, "seed %d: task must have a key", seed) - taskKeys[task.TaskKey] = true + // Generate shared job clusters first so tasks can reference them by key. + var jobClusterKeys []string + if chance(rng, 0.5) { + n := rng.IntN(2) + 1 + for i := range n { + key := fmt.Sprintf("cluster_%d", i) + jobClusterKeys = append(jobClusterKeys, key) + job.JobClusters = append(job.JobClusters, jobs.JobCluster{ + JobClusterKey: key, + NewCluster: randClusterSpec(rng), + }) + } + } - // A task referencing a job cluster must reference one we generated. - if task.JobClusterKey != "" { - assert.Containsf(t, clusterKeys, task.JobClusterKey, - "seed %d: task %q references unknown job cluster %q", seed, task.TaskKey, task.JobClusterKey) + nTasks := rng.IntN(3) + 1 + var taskKeys []string + for i := range nTasks { + task := randTask(rng, i, jobClusterKeys) + // Randomly chain dependencies onto previously generated tasks. + if len(taskKeys) > 0 && chance(rng, 0.4) { + dep := taskKeys[rng.IntN(len(taskKeys))] + task.DependsOn = []jobs.TaskDependency{{TaskKey: dep}} + if chance(rng, 0.5) { + task.RunIf = jobs.RunIf(oneOf(rng, runIfs)) } } + taskKeys = append(taskKeys, task.TaskKey) + job.Tasks = append(job.Tasks, task) + } - // Every dependency must point at a task that exists in this job. - for _, task := range job.Tasks { - for _, dep := range task.DependsOn { - assert.Containsf(t, taskKeys, dep.TaskKey, - "seed %d: task %q depends on unknown task %q", seed, task.TaskKey, dep.TaskKey) - } + return job +} + +// randScheduling sets at most one of schedule/trigger/continuous, which are +// mutually exclusive ways to launch a job. +func randScheduling(rng *rand.Rand, job *resources.Job) { + switch rng.IntN(5) { + case 0: + job.Schedule = &jobs.CronSchedule{ + QuartzCronExpression: oneOf(rng, cronExprs), + TimezoneId: oneOf(rng, timezones), + PauseStatus: oneOf(rng, pauseStatuses), + } + case 1: + job.Trigger = &jobs.TriggerSettings{ + PauseStatus: oneOf(rng, pauseStatuses), + Periodic: &jobs.PeriodicTriggerConfiguration{ + Interval: rng.IntN(12) + 1, + Unit: jobs.PeriodicTriggerConfigurationTimeUnit(oneOf(rng, timeUnits)), + }, + } + case 2: + job.Trigger = &jobs.TriggerSettings{ + PauseStatus: oneOf(rng, pauseStatuses), + FileArrival: &jobs.FileArrivalTriggerConfiguration{ + Url: "s3://" + randWord(rng) + "/" + randWord(rng), + }, + } + case 3: + job.Continuous = &jobs.Continuous{PauseStatus: oneOf(rng, pauseStatuses)} + default: + // no scheduling + } +} + +func randTask(rng *rand.Rand, idx int, jobClusterKeys []string) jobs.Task { + task := jobs.Task{TaskKey: fmt.Sprintf("task_%d", idx)} + + // Use absolute workspace paths with source=WORKSPACE so the generated bundle + // never depends on local files existing on disk (which deploy would reject). + // condition_task needs no compute, so it is handled separately below. + needsCompute := true + switch rng.IntN(4) { + case 0: + task.NotebookTask = &jobs.NotebookTask{ + NotebookPath: "/Workspace/Users/test/" + randName(rng, "nb"), + Source: jobs.SourceWorkspace, + } + case 1: + task.SparkPythonTask = &jobs.SparkPythonTask{ + PythonFile: "/Workspace/Users/test/" + randName(rng, "main") + ".py", + Source: jobs.SourceWorkspace, + } + case 2: + task.PythonWheelTask = &jobs.PythonWheelTask{ + PackageName: randName(rng, "pkg"), + EntryPoint: "main", + } + case 3: + task.ConditionTask = &jobs.ConditionTask{ + Left: randWord(rng), + Op: jobs.ConditionTaskOp(oneOf(rng, conditionOps)), + Right: randWord(rng), + } + needsCompute = false + } + + if needsCompute { + assignCompute(rng, &task, jobClusterKeys) + if chance(rng, 0.4) { + task.Libraries = randLibraries(rng) + } + } + + if chance(rng, 0.3) { + task.TimeoutSeconds = rng.IntN(3600) + } + if chance(rng, 0.3) { + task.MaxRetries = rng.IntN(5) + task.MinRetryIntervalMillis = rng.IntN(60000) + task.RetryOnTimeout = chance(rng, 0.5) + } + return task +} + +// assignCompute attaches exactly one compute source, which notebook/python/wheel +// tasks require: a shared job cluster (when available), a brand-new cluster, or an +// existing cluster id. +func assignCompute(rng *rand.Rand, task *jobs.Task, jobClusterKeys []string) { + const ( + computeNew = iota + computeExisting + computeShared + ) + options := []int{computeNew, computeExisting} + if len(jobClusterKeys) > 0 { + options = append(options, computeShared) + } + switch oneOf(rng, options) { + case computeNew: + spec := randClusterSpec(rng) + task.NewCluster = &spec + case computeExisting: + task.ExistingClusterId = randName(rng, "cluster") + case computeShared: + task.JobClusterKey = oneOf(rng, jobClusterKeys) + } +} + +func randClusterSpec(rng *rand.Rand) compute.ClusterSpec { + spec := compute.ClusterSpec{ + SparkVersion: oneOf(rng, sparkVersions), + NodeTypeId: oneOf(rng, nodeTypeIDs), + } + if chance(rng, 0.5) { + spec.NumWorkers = rng.IntN(8) + } else { + spec.Autoscale = &compute.AutoScale{ + MinWorkers: 1, + MaxWorkers: rng.IntN(8) + 2, + } + } + if chance(rng, 0.4) { + spec.SparkConf = map[string]string{ + "spark.databricks.delta.preview.enabled": "true", + "spark.speculation": strconv.FormatBool(chance(rng, 0.5)), } } + if chance(rng, 0.3) { + spec.CustomTags = randTags(rng) + } + if chance(rng, 0.3) { + spec.SparkEnvVars = map[string]string{"PYSPARK_PYTHON": "/databricks/python3/bin/python3"} + } + if chance(rng, 0.3) { + spec.DriverNodeTypeId = oneOf(rng, nodeTypeIDs) + } + return spec +} + +func randGitSource(rng *rand.Rand) *jobs.GitSource { + src := &jobs.GitSource{ + GitProvider: oneOf(rng, gitProviders), + GitUrl: "https://example.com/" + randWord(rng) + "/" + randWord(rng) + ".git", + } + switch rng.IntN(3) { + case 0: + src.GitBranch = oneOf(rng, []string{"main", "develop", "release"}) + case 1: + src.GitTag = "v" + fmt.Sprintf("%d.%d.0", rng.IntN(5), rng.IntN(10)) + case 2: + src.GitCommit = fmt.Sprintf("%040x", rng.Int64()) + } + return src +} + +func randEmailNotifications(rng *rand.Rand) *jobs.JobEmailNotifications { + email := randWord(rng) + "@example.com" + n := &jobs.JobEmailNotifications{NoAlertForSkippedRuns: chance(rng, 0.5)} + if chance(rng, 0.6) { + n.OnFailure = []string{email} + } + if chance(rng, 0.4) { + n.OnSuccess = []string{email} + } + if chance(rng, 0.3) { + n.OnStart = []string{email} + } + return n +} + +func randWebhookNotifications(rng *rand.Rand) *jobs.WebhookNotifications { + hook := []jobs.Webhook{{Id: randName(rng, "hook")}} + n := &jobs.WebhookNotifications{} + if chance(rng, 0.6) { + n.OnFailure = hook + } + if chance(rng, 0.4) { + n.OnSuccess = hook + } + return n +} + +func randHealth(rng *rand.Rand) *jobs.JobsHealthRules { + return &jobs.JobsHealthRules{ + Rules: []jobs.JobsHealthRule{ + { + Metric: jobs.JobsHealthMetric(oneOf(rng, healthMetrics)), + Op: jobs.JobsHealthOperatorGreaterThan, + Value: int64(rng.IntN(3600) + 1), + }, + }, + } +} + +func randLibraries(rng *rand.Rand) []compute.Library { + n := rng.IntN(2) + 1 + libs := make([]compute.Library, 0, n) + for range n { + switch rng.IntN(3) { + case 0: + libs = append(libs, compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: randWord(rng)}}) + case 1: + libs = append(libs, compute.Library{Maven: &compute.MavenLibrary{Coordinates: "org.example:" + randWord(rng) + ":1.0.0"}}) + case 2: + libs = append(libs, compute.Library{Whl: "/Workspace/Users/test/" + randName(rng, "lib") + ".whl"}) + } + } + return libs +} + +func randParameters(rng *rand.Rand) []jobs.JobParameterDefinition { + n := rng.IntN(3) + 1 + params := make([]jobs.JobParameterDefinition, 0, n) + for i := range n { + params = append(params, jobs.JobParameterDefinition{ + Name: fmt.Sprintf("param_%d", i), + Default: randWord(rng), + }) + } + return params +} + +func randTags(rng *rand.Rand) map[string]string { + n := rng.IntN(3) + 1 + tags := make(map[string]string, n) + for i := range n { + tags[fmt.Sprintf("tag_%d", i)] = randWord(rng) + } + return tags } diff --git a/bundle/fuzz/rand.go b/bundle/fuzz/rand_test.go similarity index 100% rename from bundle/fuzz/rand.go rename to bundle/fuzz/rand_test.go From 11dd172309db416a5383682bc331542401a50ca8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 11:42:04 +0000 Subject: [PATCH 09/53] bundle/fuzz: fix lint (stringsseq, testifylint) in paritySeeds Use strings.SplitSeq instead of ranging over strings.Split (modernize stringsseq) and require.Positivef instead of require.Greaterf(t, n, 0) (testifylint negative-positive). --- bundle/fuzz/fuzz_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 88a3c5a3b6d..79c5b55c18a 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -61,7 +61,7 @@ func TestJobCreateParity(t *testing.T) { func paritySeeds(t *testing.T) []int64 { if v := os.Getenv("FUZZ_SEED"); v != "" { var seeds []int64 - for _, part := range strings.Split(v, ",") { + for part := range strings.SplitSeq(v, ",") { part = strings.TrimSpace(part) if part == "" { continue @@ -78,7 +78,7 @@ func paritySeeds(t *testing.T) []int64 { if v := os.Getenv("FUZZ_SEEDS"); v != "" { n, err := strconv.Atoi(v) require.NoErrorf(t, err, "invalid FUZZ_SEEDS=%q", v) - require.Greaterf(t, n, 0, "FUZZ_SEEDS must be positive, got %d", n) + require.Positivef(t, n, "FUZZ_SEEDS must be positive, got %d", n) count = n } From 36e287f3be3ed34451f902a77e8c70e769efe37e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 11:53:11 +0000 Subject: [PATCH 10/53] bundle/fuzz: fix nightly issue dedup and document paritySeeds test The failure-reporting step used `gh issue list --jq '.[0].number'`, which prints the literal "null" when no open issue exists, so it always took the comment branch and tried to comment on issue "null" instead of creating one. Use `// empty` so the create branch runs on the first divergence. --- .github/workflows/push.yml | 2 +- bundle/fuzz/fuzz_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8029ef6dbb5..007b54b17f6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -475,7 +475,7 @@ jobs: EOF ) - existing=$(gh issue list --state open --label fuzz-nightly --json number --jq '.[0].number') + existing=$(gh issue list --state open --label fuzz-nightly --json number --jq '.[0].number // empty') if [ -n "$existing" ]; then gh issue comment "$existing" --body "$body" else diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 79c5b55c18a..3b15ea5e144 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -107,6 +107,8 @@ func paritySeeds(t *testing.T) []int64 { return seeds } +// TestParitySeeds verifies paritySeeds composes the regression seeds with the +// rotating window, deduplicates overlaps, and lets FUZZ_SEED override both. func TestParitySeeds(t *testing.T) { t.Run("default includes regression seeds then window", func(t *testing.T) { t.Setenv("FUZZ_SEEDS", "3") From 26a29c1664213e59937782817f89d64c83b7f8d6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 17:41:57 +0000 Subject: [PATCH 11/53] bundle/fuzz: document divergences instead of fixing them Revert the num_workers single-node task-cluster fix along with its unit test and acceptance updates so this PR adds only the parity harness. Both terraform/direct divergences the harness found are now documented and suppressed via DefaultIgnorePaths rather than fixed (fixes follow separately): num_workers on single-node task clusters (seed 29) and the spark.databricks.delta.preview.enabled spark conf key. --- .../bundle/deploy/wal/chain-3-jobs/output.txt | 2 - .../deploy/wal/crash-after-create/output.txt | 1 - .../bundle/override/job_tasks/output.txt | 2 - .../missing_map_key/out.validate.direct.json | 3 +- .../out.validate.terraform.json | 3 +- .../mutator/resourcemutator/cluster_fixups.go | 1 - .../resourcemutator/cluster_fixups_test.go | 92 ------------------- bundle/fuzz/compare_test.go | 9 ++ bundle/fuzz/fuzz_test.go | 19 ++-- bundle/fuzz/recorder_test.go | 8 +- 10 files changed, 25 insertions(+), 115 deletions(-) delete mode 100644 bundle/config/mutator/resourcemutator/cluster_fixups_test.go diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index 5cb87e0c097..7cbc9f31a85 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -35,7 +35,6 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { @@ -74,7 +73,6 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index d0a32c78254..e9e2c19c094 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -39,7 +39,6 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt index 59b6fc1c397..2bee9738e33 100644 --- a/acceptance/bundle/override/job_tasks/output.txt +++ b/acceptance/bundle/override/job_tasks/output.txt @@ -18,7 +18,6 @@ }, { "new_cluster": { - "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { @@ -43,7 +42,6 @@ Exit code: 1 "tasks": [ { "new_cluster": { - "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json index 7279aaeba31..cfd1427ce4d 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json @@ -30,8 +30,7 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - }, - "num_workers": 0 + } }, "task_key": "test-task" } diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json index 3bad6f46193..3cdf58f84ea 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json @@ -30,8 +30,7 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - }, - "num_workers": 0 + } }, "task_key": "test-task" } diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups.go b/bundle/config/mutator/resourcemutator/cluster_fixups.go index 04ddef6cc2f..893cd248aa4 100644 --- a/bundle/config/mutator/resourcemutator/cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/cluster_fixups.go @@ -94,7 +94,6 @@ func prepareJobSettingsForUpdate(js *jobs.JobSettings) { for _, task := range js.Tasks { if task.NewCluster != nil { ModifyRequestOnInstancePool(task.NewCluster) - initializeNumWorkers(task.NewCluster) } } for ind := range js.JobClusters { diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups_test.go b/bundle/config/mutator/resourcemutator/cluster_fixups_test.go deleted file mode 100644 index 5cb2e937494..00000000000 --- a/bundle/config/mutator/resourcemutator/cluster_fixups_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package resourcemutator - -import ( - "testing" - - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" -) - -func TestInitializeNumWorkers(t *testing.T) { - tests := []struct { - name string - spec compute.ClusterSpec - wantForceSend bool - }{ - { - name: "single-node cluster force-sends num_workers", - spec: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, - wantForceSend: true, - }, - { - name: "autoscale cluster does not force-send", - spec: compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, - wantForceSend: false, - }, - { - name: "multi-node cluster does not force-send", - spec: compute.ClusterSpec{NumWorkers: 3}, - wantForceSend: false, - }, - { - name: "already force-sent stays force-sent without duplicating", - spec: compute.ClusterSpec{ForceSendFields: []string{"NumWorkers"}}, - wantForceSend: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - spec := tt.spec - initializeNumWorkers(&spec) - - count := 0 - for _, f := range spec.ForceSendFields { - if f == "NumWorkers" { - count++ - } - } - if tt.wantForceSend { - assert.Equal(t, 1, count, "NumWorkers must appear in ForceSendFields exactly once") - } else { - assert.Equal(t, 0, count, "NumWorkers must not be in ForceSendFields") - } - }) - } -} - -// TestPrepareJobSettingsForUpdateForcesNumWorkers locks the DECO-25361 fix: a -// single-node new_cluster must force-send num_workers on task-level clusters too, -// not just shared job_clusters. The terraform provider always sends num_workers:0 -// for such clusters, so missing it on the task side made the direct engine -// produce a divergent create payload. -func TestPrepareJobSettingsForUpdateForcesNumWorkers(t *testing.T) { - js := &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - TaskKey: "single_node_task", - NewCluster: &compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, - }, - { - TaskKey: "autoscale_task", - NewCluster: &compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, - }, - }, - JobClusters: []jobs.JobCluster{ - { - JobClusterKey: "single_node_cluster", - NewCluster: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, - }, - }, - } - - prepareJobSettingsForUpdate(js) - - assert.Contains(t, js.Tasks[0].NewCluster.ForceSendFields, "NumWorkers", - "single-node task cluster must force-send num_workers") - assert.NotContains(t, js.Tasks[1].NewCluster.ForceSendFields, "NumWorkers", - "autoscale task cluster must not force-send num_workers") - assert.Contains(t, js.JobClusters[0].NewCluster.ForceSendFields, "NumWorkers", - "single-node job cluster must force-send num_workers") -} diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go index fd6807b56cc..f53d2f3b30a 100644 --- a/bundle/fuzz/compare_test.go +++ b/bundle/fuzz/compare_test.go @@ -270,4 +270,13 @@ var DefaultIgnorePaths = []string{ // way, so this is a benign provider-side filter rather than a parity bug. `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, + + // For a single-node task-level new_cluster (no autoscale, num_workers unset) + // the terraform provider force-sends num_workers:0 while the direct engine + // omits the field, so the create payloads diverge. This is a real + // terraform/direct divergence the harness found (seed 29); it is documented + // and suppressed here rather than fixed in this PR. Tracked under DECO-25361. + // Shared job_clusters are not affected: resourcemutator already force-sends + // num_workers for them under both engines, so only the task path diverges. + `tasks[*].new_cluster.num_workers`, } diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 3b15ea5e144..7836574d156 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -18,17 +18,18 @@ const defaultParitySeeds = 20 // regressionSeeds are seeds that previously surfaced a terraform/direct create // payload divergence. They are always checked (in addition to the rotating -// nightly window) so a fixed divergence can never silently regress, even though -// the nightly window moves on every run and would otherwise never revisit them. +// nightly window) so the divergence keeps being exercised even though the +// nightly window moves on every run and would otherwise never revisit them. // -// When the nightly job reports a new failing FUZZ_SEED, add it here in the same -// PR that fixes the divergence. +// When the nightly job reports a new failing FUZZ_SEED, add it here. // -// - 29: first seed that generates a single-node task-level new_cluster -// (num_workers 0, no autoscale). The direct engine omitted num_workers on -// task clusters while terraform force-sent num_workers:0, so the create -// payloads diverged. Fixed by applying initializeNumWorkers to task clusters -// in resourcemutator.prepareJobSettingsForUpdate. +// - 29: generates a single-node task-level new_cluster (num_workers 0, no +// autoscale). The direct engine omits num_workers on task clusters while +// terraform force-sends num_workers:0, so the create payloads diverge. This +// divergence is documented and currently suppressed via DefaultIgnorePaths +// (tasks[*].new_cluster.num_workers), not fixed in this PR; tracked under +// DECO-25361. The seed stays here so that once the divergence is fixed and +// its ignore entry removed, this seed guards against regression. var regressionSeeds = []int64{29} // TestJobCreateParity is the first DECO-25361 technique: for many random job diff --git a/bundle/fuzz/recorder_test.go b/bundle/fuzz/recorder_test.go index 244cb81480f..a5e7d4d707f 100644 --- a/bundle/fuzz/recorder_test.go +++ b/bundle/fuzz/recorder_test.go @@ -9,10 +9,10 @@ import ( // jobsCreatePath is the Jobs API route both engines must hit on create. The // direct engine posts here via the SDK and the terraform provider is expected to -// as well. The testserver registers only this exact route, so if an engine ever -// posted to a different version the deploy would 404 and captureJobCreate would -// fail with "did not POST". A version skew therefore surfaces as a capture -// failure, not as a payload diff. +// as well. The testserver registers only this version of the jobs/create route, +// so if an engine ever posted to a different version the deploy would 404 and +// captureJobCreate would fail with "did not POST". A version skew therefore +// surfaces as a capture failure, not as a payload diff. const jobsCreatePath = "/api/2.2/jobs/create" // capturedRequest is a single mutating API request observed by the testserver. From ae9f26d20acaddda52163f28946b608e7459b432 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 26 Jun 2026 07:41:24 +0000 Subject: [PATCH 12/53] bundle/fuzz: narrow num_workers ignore and tidy parity harness Address review feedback on the create-payload parity harness: - Replace the path-only ignore list with value-conditional ignore rules so the documented num_workers divergence (direct omits, terraform force-sends 0) is suppressed only for that exact shape; a real value mismatch at the same path now fails again. - Unexport package-internal identifiers (generateJob, diffPayloads, difference, defaultIgnoreRules) that are only used within the package. - Document why TestCaptureJobCreateDirect is intentionally not opt-in. - Reword the one-sided-deploy failures as deploy/capture differences rather than asserting one engine "rejected" the config. - Make TestParitySeeds hermetic against ambient FUZZ_* env vars. - Correct the seed 29 comment to reflect that the divergence is suppressed. --- bundle/fuzz/compare_cases_test.go | 37 ++++++++-- bundle/fuzz/compare_test.go | 96 ++++++++++++++++--------- bundle/fuzz/deploy_smoke_test.go | 10 ++- bundle/fuzz/fuzz_test.go | 35 +++++---- bundle/fuzz/generate_invariants_test.go | 6 +- bundle/fuzz/generate_test.go | 4 +- 6 files changed, 133 insertions(+), 55 deletions(-) diff --git a/bundle/fuzz/compare_cases_test.go b/bundle/fuzz/compare_cases_test.go index 46e506d75c6..95c732750b9 100644 --- a/bundle/fuzz/compare_cases_test.go +++ b/bundle/fuzz/compare_cases_test.go @@ -13,7 +13,7 @@ func TestDiffPayloads(t *testing.T) { name string direct string terraform string - ignore []string + ignore []ignoreRule want []string }{ { @@ -62,7 +62,7 @@ func TestDiffPayloads(t *testing.T) { name: "ignored path", direct: `{"tasks":[{"timeout_seconds":1}]}`, terraform: `{"tasks":[{"timeout_seconds":2}]}`, - ignore: []string{"tasks[*].timeout_seconds"}, + ignore: []ignoreRule{{Path: "tasks[*].timeout_seconds"}}, want: nil, }, { @@ -75,7 +75,7 @@ func TestDiffPayloads(t *testing.T) { name: "dotted map key can be ignored", direct: `{"c":{"spark_conf":{"spark.x.y":"1"}}}`, terraform: `{"c":{"spark_conf":{}}}`, - ignore: []string{`c.spark_conf["spark.x.y"]`}, + ignore: []ignoreRule{{Path: `c.spark_conf["spark.x.y"]`}}, want: nil, }, { @@ -102,11 +102,29 @@ func TestDiffPayloads(t *testing.T) { terraform: `{"job_clusters":[{"job_cluster_key":"y","new_cluster":{"num_workers":2}},{"job_cluster_key":"x","new_cluster":{"num_workers":1}}]}`, want: nil, }, + { + // The documented single-node divergence: direct omits num_workers, + // terraform force-sends 0. defaultIgnoreRules suppresses exactly this. + name: "task num_workers absent-vs-zero is ignored", + direct: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x"}}]}`, + terraform: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x","num_workers":0}}]}`, + ignore: defaultIgnoreRules, + want: nil, + }, + { + // A real num_workers value mismatch shares the path but is NOT the + // benign shape, so the narrowed rule must still report it. + name: "task num_workers value mismatch still surfaces", + direct: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x","num_workers":3}}]}`, + terraform: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x","num_workers":5}}]}`, + ignore: defaultIgnoreRules, + want: []string{"tasks[0].new_cluster.num_workers"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - diffs, err := DiffPayloads(json.RawMessage(tt.direct), json.RawMessage(tt.terraform), tt.ignore) + diffs, err := diffPayloads(json.RawMessage(tt.direct), json.RawMessage(tt.terraform), tt.ignore) require.NoError(t, err) var paths []string @@ -117,3 +135,14 @@ func TestDiffPayloads(t *testing.T) { }) } } + +func TestIsBenignTaskNumWorkers(t *testing.T) { + assert.True(t, isBenignTaskNumWorkers(difference{Direct: missing{}, Terraform: json.Number("0")}), + "direct absent + terraform 0 is the documented divergence") + assert.False(t, isBenignTaskNumWorkers(difference{Direct: json.Number("3"), Terraform: json.Number("5")}), + "two differing counts is a real divergence") + assert.False(t, isBenignTaskNumWorkers(difference{Direct: missing{}, Terraform: json.Number("2")}), + "direct absent but terraform non-zero is not the benign shape") + assert.False(t, isBenignTaskNumWorkers(difference{Direct: json.Number("0"), Terraform: missing{}}), + "reversed sides are not the benign shape") +} diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go index f53d2f3b30a..de34c18aaf4 100644 --- a/bundle/fuzz/compare_test.go +++ b/bundle/fuzz/compare_test.go @@ -10,15 +10,15 @@ import ( "strings" ) -// Difference is a single mismatch between the two engines' create payloads, +// difference is a single mismatch between the two engines' create payloads, // located by a JSON-ish path (e.g. "tasks[0].new_cluster.num_workers"). -type Difference struct { +type difference struct { Path string Direct any Terraform any } -func (d Difference) String() string { +func (d difference) String() string { return fmt.Sprintf("%s: direct=%s terraform=%s", d.Path, render(d.Direct), render(d.Terraform)) } @@ -36,10 +36,20 @@ func render(v any) string { return string(b) } -// DiffPayloads decodes both create payloads and returns every difference whose -// path is not explicitly ignored. ignorePaths are matched exactly against the -// rendered path, with "[*]" standing in for any slice index. -func DiffPayloads(direct, terraform json.RawMessage, ignorePaths []string) ([]Difference, error) { +// ignoreRule suppresses a known, intentional engine divergence. A rule matches a +// difference when the difference's normalized path equals Path and, if Match is +// non-nil, Match also reports true for the two values. A nil Match ignores any +// difference at Path; a non-nil Match narrows the rule to specific values so a +// genuine mismatch at the same path is still reported. +type ignoreRule struct { + Path string + Match func(d difference) bool +} + +// diffPayloads decodes both create payloads and returns every difference that no +// ignore rule suppresses. Paths are matched with "[*]" standing in for any slice +// index (see normalizePath). +func diffPayloads(direct, terraform json.RawMessage, ignore []ignoreRule) ([]difference, error) { d, err := decode(direct) if err != nil { return nil, fmt.Errorf("decoding direct payload: %w", err) @@ -49,23 +59,32 @@ func DiffPayloads(direct, terraform json.RawMessage, ignorePaths []string) ([]Di return nil, fmt.Errorf("decoding terraform payload: %w", err) } - var diffs []Difference + var diffs []difference diffValue("", d, tf, &diffs) - ignore := make(map[string]bool, len(ignorePaths)) - for _, p := range ignorePaths { - ignore[p] = true - } - filtered := diffs[:0] for _, diff := range diffs { - if !ignore[normalizePath(diff.Path)] { + if !ignored(diff, ignore) { filtered = append(filtered, diff) } } return filtered, nil } +// ignored reports whether any rule suppresses d. +func ignored(d difference, rules []ignoreRule) bool { + norm := normalizePath(d.Path) + for _, r := range rules { + if r.Path != norm { + continue + } + if r.Match == nil || r.Match(d) { + return true + } + } + return false +} + // decode unmarshals JSON using UseNumber so large int64 values (e.g. job ids, // spark_context_id) are not corrupted by float64 rounding. See the encoding rule // in the repo style guide. @@ -82,12 +101,12 @@ func decode(raw json.RawMessage) (any, error) { return v, nil } -func diffValue(path string, a, b any, diffs *[]Difference) { +func diffValue(path string, a, b any, diffs *[]difference) { switch av := a.(type) { case map[string]any: bv, ok := b.(map[string]any) if !ok { - *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) return } keys := unionKeys(av, bv) @@ -99,15 +118,15 @@ func diffValue(path string, a, b any, diffs *[]Difference) { case aok && bok: diffValue(child, achild, bchild, diffs) case aok: - *diffs = append(*diffs, Difference{Path: child, Direct: achild, Terraform: missing{}}) + *diffs = append(*diffs, difference{Path: child, Direct: achild, Terraform: missing{}}) default: - *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bchild}) + *diffs = append(*diffs, difference{Path: child, Direct: missing{}, Terraform: bchild}) } } case []any: bv, ok := b.([]any) if !ok { - *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) return } // Slices whose elements carry a natural identity key (tasks, job clusters) @@ -125,14 +144,14 @@ func diffValue(path string, a, b any, diffs *[]Difference) { case i < len(av) && i < len(bv): diffValue(child, av[i], bv[i], diffs) case i < len(av): - *diffs = append(*diffs, Difference{Path: child, Direct: av[i], Terraform: missing{}}) + *diffs = append(*diffs, difference{Path: child, Direct: av[i], Terraform: missing{}}) default: - *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: bv[i]}) + *diffs = append(*diffs, difference{Path: child, Direct: missing{}, Terraform: bv[i]}) } } default: if !scalarEqual(a, b) { - *diffs = append(*diffs, Difference{Path: path, Direct: a, Terraform: b}) + *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) } } } @@ -174,7 +193,7 @@ func allHaveKey(s []any, field string) bool { // within each slice for tasks/job clusters) and diffs each matched pair, // reporting unmatched elements as present-on-one-side. Paths keep numeric indices // so ignore-path [*] normalization still applies. -func diffKeyedSlice(path, key string, a, b []any, diffs *[]Difference) { +func diffKeyedSlice(path, key string, a, b []any, diffs *[]difference) { // identityFields are unique within a slice by API contract (no two job tasks // share a task_key, no two job_clusters share a job_cluster_key), so keying by // them is unambiguous. If a payload ever repeated a key, last-one-wins here and @@ -193,7 +212,7 @@ func diffKeyedSlice(path, key string, a, b []any, diffs *[]Difference) { if bel, ok := bByKey[k]; ok { diffValue(child, el, bel, diffs) } else { - *diffs = append(*diffs, Difference{Path: child, Direct: el, Terraform: missing{}}) + *diffs = append(*diffs, difference{Path: child, Direct: el, Terraform: missing{}}) } } for j, el := range b { @@ -202,7 +221,7 @@ func diffKeyedSlice(path, key string, a, b []any, diffs *[]Difference) { continue } child := fmt.Sprintf("%s[%d]", path, j) - *diffs = append(*diffs, Difference{Path: child, Direct: missing{}, Terraform: el}) + *diffs = append(*diffs, difference{Path: child, Direct: missing{}, Terraform: el}) } } @@ -260,16 +279,16 @@ func normalizePath(path string) string { return indexRe.ReplaceAllString(path, "[*]") } -// DefaultIgnorePaths lists create-payload paths that legitimately differ between -// the engines and are not parity bugs. Keep this list small and well-justified; -// every entry is a known, intentional divergence. -var DefaultIgnorePaths = []string{ +// defaultIgnoreRules lists create-payload divergences that are known, intentional +// engine differences and not parity bugs. Keep this list small and +// well-justified; every entry is a documented divergence. +var defaultIgnoreRules = []ignoreRule{ // The terraform provider strips the deprecated/ignored spark conf // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while // the direct engine forwards it verbatim. The backend ignores the key either // way, so this is a benign provider-side filter rather than a parity bug. - `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, - `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, + {Path: `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`}, + {Path: `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`}, // For a single-node task-level new_cluster (no autoscale, num_workers unset) // the terraform provider force-sends num_workers:0 while the direct engine @@ -278,5 +297,18 @@ var DefaultIgnorePaths = []string{ // and suppressed here rather than fixed in this PR. Tracked under DECO-25361. // Shared job_clusters are not affected: resourcemutator already force-sends // num_workers for them under both engines, so only the task path diverges. - `tasks[*].new_cluster.num_workers`, + // + // Match narrows this to exactly that shape (direct absent, terraform 0); a + // genuine num_workers value mismatch at the same path is still reported. + {Path: `tasks[*].new_cluster.num_workers`, Match: isBenignTaskNumWorkers}, +} + +// isBenignTaskNumWorkers reports whether d is the single documented num_workers +// divergence: the direct engine omits num_workers while terraform force-sends 0. +// Any other pair of values (in particular two differing non-zero counts) is a +// real divergence and must not be suppressed. +func isBenignTaskNumWorkers(d difference) bool { + _, directAbsent := d.Direct.(missing) + n, ok := d.Terraform.(json.Number) + return directAbsent && ok && n.String() == "0" } diff --git a/bundle/fuzz/deploy_smoke_test.go b/bundle/fuzz/deploy_smoke_test.go index d501ee78089..dd90b359902 100644 --- a/bundle/fuzz/deploy_smoke_test.go +++ b/bundle/fuzz/deploy_smoke_test.go @@ -8,8 +8,14 @@ import ( "github.com/stretchr/testify/require" ) +// TestCaptureJobCreateDirect is intentionally NOT gated behind requireFuzzOptIn, +// unlike the terraform parity suite. The direct engine needs no provisioned +// terraform, and one deterministic direct deploy is cheap, so this runs on every +// `task test` as a smoke test that the capture harness and the direct create path +// still work. The expensive part the opt-in protects against is the terraform +// side (two real deploys per seed), which stays opt-in via requireTerraform. func TestCaptureJobCreateDirect(t *testing.T) { - job := GenerateJob(newRNG(1)) + job := generateJob(newRNG(1)) body, err := captureJobCreate(t.Context(), t, job, "direct") require.NoError(t, err) @@ -23,7 +29,7 @@ func TestCaptureJobCreateDirect(t *testing.T) { func TestCaptureJobCreateTerraform(t *testing.T) { requireTerraform(t) - job := GenerateJob(newRNG(1)) + job := generateJob(newRNG(1)) body, err := captureJobCreate(t.Context(), t, job, "terraform") require.NoError(t, err) diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 7836574d156..596ac99f7db 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -25,11 +25,12 @@ const defaultParitySeeds = 20 // // - 29: generates a single-node task-level new_cluster (num_workers 0, no // autoscale). The direct engine omits num_workers on task clusters while -// terraform force-sends num_workers:0, so the create payloads diverge. This -// divergence is documented and currently suppressed via DefaultIgnorePaths -// (tasks[*].new_cluster.num_workers), not fixed in this PR; tracked under -// DECO-25361. The seed stays here so that once the divergence is fixed and -// its ignore entry removed, this seed guards against regression. +// terraform force-sends num_workers:0, so the create payloads diverge. That +// specific shape is suppressed by defaultIgnoreRules (see +// isBenignTaskNumWorkers), so seed 29 currently asserts only that nothing +// else about this config diverges. Once the divergence is fixed and its +// ignore rule removed, this seed becomes a full guard against it regressing. +// Tracked under DECO-25361. var regressionSeeds = []int64{29} // TestJobCreateParity is the first DECO-25361 technique: for many random job @@ -111,6 +112,14 @@ func paritySeeds(t *testing.T) []int64 { // TestParitySeeds verifies paritySeeds composes the regression seeds with the // rotating window, deduplicates overlaps, and lets FUZZ_SEED override both. func TestParitySeeds(t *testing.T) { + // Isolate from any ambient FUZZ_* in the developer's environment. FUZZ_SEED in + // particular would short-circuit paritySeeds and break the cases below; an + // inherited FUZZ_SEEDS/OFFSET would skew the expected window. paritySeeds + // treats "" as unset, and subtests set only what they need on top. + t.Setenv("FUZZ_SEED", "") + t.Setenv("FUZZ_SEEDS", "") + t.Setenv("FUZZ_SEED_OFFSET", "") + t.Run("default includes regression seeds then window", func(t *testing.T) { t.Setenv("FUZZ_SEEDS", "3") t.Setenv("FUZZ_SEED_OFFSET", "100") @@ -163,13 +172,15 @@ func FuzzJobCreateParity(f *testing.F) { // deploy failure into regressionSeeds (which is only for real payload diffs): // - neither engine deployed: the generator produced a config nothing accepts, // so skip (logging both errors) rather than flag a parity bug. -// - exactly one engine deployed: the engines disagree on whether the config is -// even valid. That is a real divergence worth failing on, but an acceptance -// divergence, not a payload diff, so it is reported as such. +// - exactly one engine deployed: the engines disagree on whether the config +// deploys at all. That is worth failing on, but it is a deploy/capture +// difference rather than a payload diff, so it is reported separately. The +// failing side's error (an API rejection, an unregistered route, etc.) is +// included so triage can tell a true acceptance divergence from a harness gap. // - both deployed: compare the captured create payloads. func checkJobParity(t *testing.T, seed int64) { t.Helper() - job := GenerateJob(newRNG(seed)) + job := generateJob(newRNG(seed)) ctx := t.Context() direct, directErr := captureJobCreate(ctx, t, job, "direct") @@ -179,12 +190,12 @@ func checkJobParity(t *testing.T, seed int64) { case directErr != nil && tfErr != nil: t.Skipf("seed %d: config did not deploy under either engine (not a parity divergence)\ndirect: %v\nterraform: %v", seed, directErr, tfErr) case directErr != nil: - t.Fatalf("seed %d: direct rejected a config terraform accepted (engine acceptance divergence, not a payload diff): %v", seed, directErr) + t.Fatalf("seed %d: terraform deployed but direct did not (deploy/capture difference, not a payload diff): %v", seed, directErr) case tfErr != nil: - t.Fatalf("seed %d: terraform rejected a config direct accepted (engine acceptance divergence, not a payload diff): %v", seed, tfErr) + t.Fatalf("seed %d: direct deployed but terraform did not (deploy/capture difference, not a payload diff): %v", seed, tfErr) } - diffs, err := DiffPayloads(direct, terraform, DefaultIgnorePaths) + diffs, err := diffPayloads(direct, terraform, defaultIgnoreRules) require.NoErrorf(t, err, "seed %d: comparing create payloads", seed) if len(diffs) > 0 { diff --git a/bundle/fuzz/generate_invariants_test.go b/bundle/fuzz/generate_invariants_test.go index f7a797e8f59..9ca3b5cc932 100644 --- a/bundle/fuzz/generate_invariants_test.go +++ b/bundle/fuzz/generate_invariants_test.go @@ -8,14 +8,14 @@ import ( ) func TestGenerateJobIsDeterministic(t *testing.T) { - a := GenerateJob(newRNG(42)) - b := GenerateJob(newRNG(42)) + a := generateJob(newRNG(42)) + b := generateJob(newRNG(42)) assert.Equal(t, a, b, "same seed must produce identical job") } func TestGenerateJobIsWellFormed(t *testing.T) { for seed := range int64(200) { - job := GenerateJob(newRNG(seed)) + job := generateJob(newRNG(seed)) require.NotEmptyf(t, job.Name, "seed %d: job must have a name", seed) require.NotEmptyf(t, job.Tasks, "seed %d: job must have at least one task", seed) diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go index 1b0acf55b0f..6472957b2f4 100644 --- a/bundle/fuzz/generate_test.go +++ b/bundle/fuzz/generate_test.go @@ -28,14 +28,14 @@ var ( gitProviders = []jobs.GitProvider{jobs.GitProviderGitHub, jobs.GitProviderGitLab, jobs.GitProviderAzureDevOpsServices} ) -// GenerateJob builds a random, well-formed job config driven entirely by rng, so +// generateJob builds a random, well-formed job config driven entirely by rng, so // the same seed always produces the same job. It deliberately favors fields whose // translation tends to differ between engines (tasks, clusters, schedules, // notifications, tags, zero-able scalars). // // TODO(DECO-25361): generalize the harness across resource kinds so pipelines, // apps, etc. get the same create-payload parity coverage as jobs. -func GenerateJob(rng *rand.Rand) *resources.Job { +func generateJob(rng *rand.Rand) *resources.Job { job := &resources.Job{} job.Name = randName(rng, "job") From 45cfba67988974ecc2b442aa3255976658530bf1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 26 Jun 2026 08:21:24 +0000 Subject: [PATCH 13/53] bundle: force-send num_workers for single-node task clusters The terraform provider force-sends num_workers:0 for a single-node new_cluster on task-level clusters too, not just shared job_clusters, but prepareJobSettingsForUpdate only applied initializeNumWorkers to job_clusters. The direct engine therefore omitted num_workers on task clusters and the two engines produced divergent create payloads (found by the bundle/fuzz parity harness, seed 29). Apply initializeNumWorkers to task new_cluster too so the direct engine matches terraform, drop the now-obsolete tasks[*].new_cluster.num_workers ignore entry, and simplify the fuzz ignore list to a plain []string now that value-conditional matching is no longer needed. --- .github/workflows/push.yml | 26 ++--- Taskfile.yml | 15 +-- .../bundle/deploy/wal/chain-3-jobs/output.txt | 2 + .../deploy/wal/crash-after-create/output.txt | 1 + .../bundle/override/job_tasks/output.txt | 2 + .../missing_map_key/out.validate.direct.json | 3 +- .../out.validate.terraform.json | 3 +- .../mutator/resourcemutator/cluster_fixups.go | 3 + .../resourcemutator/cluster_fixups_test.go | 92 +++++++++++++++ bundle/fuzz/compare_cases_test.go | 35 +----- bundle/fuzz/compare_test.go | 109 ++++-------------- bundle/fuzz/deploy_smoke_test.go | 9 +- bundle/fuzz/deploy_test.go | 46 +++----- bundle/fuzz/doc.go | 21 +--- bundle/fuzz/fuzz_test.go | 83 +++++-------- bundle/fuzz/generate_test.go | 23 ++-- bundle/fuzz/recorder_test.go | 7 +- 17 files changed, 205 insertions(+), 275 deletions(-) create mode 100644 bundle/config/mutator/resourcemutator/cluster_fixups_test.go diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 007b54b17f6..6644c5cec3f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -397,10 +397,8 @@ jobs: needs: - cleanups - # The terraform/direct create-payload parity tests run two real `bundle deploy` - # invocations per seed, so they are too slow for every PR and too noisy to gate - # the merge queue. Run them on the nightly schedule to catch engine drift; not - # part of test-result for that reason. + # Two real deploys per seed: too slow for every PR, so nightly only and not part + # of test-result. if: ${{ github.event_name == 'schedule' }} name: "task test-fuzz" runs-on: @@ -428,26 +426,16 @@ jobs: - name: Run tests env: - # Shift the seed window by the run number every nightly run so CI - # explores configs it has never tested before instead of re-checking a - # fixed set. The window is kept modest (each seed runs two real deploys) - # since the exploration comes from rotating the window, not its size; - # raise it once nightly timings are known. A divergence prints - # FUZZ_SEED= for one-command reproduction. - # - # offset = GITHUB_RUN_NUMBER * FUZZ_SEEDS. GITHUB_RUN_NUMBER is a - # built-in, monotonically increasing, unique-per-run integer, so as long - # as FUZZ_SEEDS is constant the windows are non-overlapping (gaps from - # non-schedule runs are fine; we only need fresh seeds, not every seed). + # Shift the seed window each nightly run so CI explores new configs. + # offset = GITHUB_RUN_NUMBER * FUZZ_SEEDS keeps windows non-overlapping + # (GITHUB_RUN_NUMBER is monotonic). A divergence prints FUZZ_SEED=. FUZZ_SEEDS: "25" run: | export FUZZ_SEED_OFFSET=$(( GITHUB_RUN_NUMBER * FUZZ_SEEDS )) go tool -modfile=tools/task/go.mod task test-fuzz - # This job is intentionally excluded from test-result, so a failure here is - # invisible unless someone watches the Actions tab. Surface it as a GitHub - # issue instead. Reuse a single open issue (deduped by label) so a recurring - # divergence doesn't open one issue per night. + # Excluded from test-result, so surface failures as a GitHub issue. Reuse one + # open issue (deduped by label) so a recurring divergence doesn't spam nightly. - name: Report failure if: ${{ failure() }} env: diff --git a/Taskfile.yml b/Taskfile.yml index b8d6a139166..c577bc32e1e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -732,19 +732,14 @@ tasks: test-fuzz: desc: Run terraform/direct create-payload parity fuzz tests (provisions terraform) - # No `sources:` fingerprint: the seeds checked are a function of the FUZZ_SEED, - # FUZZ_SEEDS, and FUZZ_SEED_OFFSET env vars, which Task can't see. Skipping on - # an unchanged source checksum would silently no-op a FUZZ_SEED= repro run - # or a shifted nightly window, so always run. + # No `sources:` fingerprint: the seeds depend on FUZZ_* env vars Task can't see, + # so always run rather than no-op a repro or a shifted nightly window. env: - # The terraform parity tests are opt-in (see requireFuzzOptIn): they skip - # unless a FUZZ_* var is set, so a leftover build/ never makes them run as - # part of a plain `task test`. This constant flag opts this target in - # without overriding the FUZZ_SEED(S)/OFFSET tuning knobs. + # Opt this target into the parity suite (see requireFuzzOptIn) without + # overriding the FUZZ_SEED(S)/OFFSET tuning knobs. FUZZ_PARITY: "1" cmds: - # The parity harness expects terraform + the provider mirror at /build; - # requireTerraform skips when it's absent, so provision it first. + # requireTerraform expects terraform + provider mirror at /build. - python3 acceptance/install_terraform.py --targetdir build - | {{.GO_TOOL}} gotestsum \ diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index 7cbc9f31a85..5cb87e0c097 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -35,6 +35,7 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { @@ -73,6 +74,7 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index e9e2c19c094..d0a32c78254 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -39,6 +39,7 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt index 2bee9738e33..59b6fc1c397 100644 --- a/acceptance/bundle/override/job_tasks/output.txt +++ b/acceptance/bundle/override/job_tasks/output.txt @@ -18,6 +18,7 @@ }, { "new_cluster": { + "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { @@ -42,6 +43,7 @@ Exit code: 1 "tasks": [ { "new_cluster": { + "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json index cfd1427ce4d..7279aaeba31 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json @@ -30,7 +30,8 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - } + }, + "num_workers": 0 }, "task_key": "test-task" } diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json index 3cdf58f84ea..3bad6f46193 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json @@ -30,7 +30,8 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - } + }, + "num_workers": 0 }, "task_key": "test-task" } diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups.go b/bundle/config/mutator/resourcemutator/cluster_fixups.go index 893cd248aa4..ee4ee04c8be 100644 --- a/bundle/config/mutator/resourcemutator/cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/cluster_fixups.go @@ -94,6 +94,9 @@ func prepareJobSettingsForUpdate(js *jobs.JobSettings) { for _, task := range js.Tasks { if task.NewCluster != nil { ModifyRequestOnInstancePool(task.NewCluster) + // Match terraform, which force-sends num_workers:0 for single-node + // task clusters too, not just shared job_clusters (DECO-25361). + initializeNumWorkers(task.NewCluster) } } for ind := range js.JobClusters { diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups_test.go b/bundle/config/mutator/resourcemutator/cluster_fixups_test.go new file mode 100644 index 00000000000..5cb2e937494 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/cluster_fixups_test.go @@ -0,0 +1,92 @@ +package resourcemutator + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestInitializeNumWorkers(t *testing.T) { + tests := []struct { + name string + spec compute.ClusterSpec + wantForceSend bool + }{ + { + name: "single-node cluster force-sends num_workers", + spec: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, + wantForceSend: true, + }, + { + name: "autoscale cluster does not force-send", + spec: compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, + wantForceSend: false, + }, + { + name: "multi-node cluster does not force-send", + spec: compute.ClusterSpec{NumWorkers: 3}, + wantForceSend: false, + }, + { + name: "already force-sent stays force-sent without duplicating", + spec: compute.ClusterSpec{ForceSendFields: []string{"NumWorkers"}}, + wantForceSend: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec + initializeNumWorkers(&spec) + + count := 0 + for _, f := range spec.ForceSendFields { + if f == "NumWorkers" { + count++ + } + } + if tt.wantForceSend { + assert.Equal(t, 1, count, "NumWorkers must appear in ForceSendFields exactly once") + } else { + assert.Equal(t, 0, count, "NumWorkers must not be in ForceSendFields") + } + }) + } +} + +// TestPrepareJobSettingsForUpdateForcesNumWorkers locks the DECO-25361 fix: a +// single-node new_cluster must force-send num_workers on task-level clusters too, +// not just shared job_clusters. The terraform provider always sends num_workers:0 +// for such clusters, so missing it on the task side made the direct engine +// produce a divergent create payload. +func TestPrepareJobSettingsForUpdateForcesNumWorkers(t *testing.T) { + js := &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "single_node_task", + NewCluster: &compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, + }, + { + TaskKey: "autoscale_task", + NewCluster: &compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, + }, + }, + JobClusters: []jobs.JobCluster{ + { + JobClusterKey: "single_node_cluster", + NewCluster: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, + }, + }, + } + + prepareJobSettingsForUpdate(js) + + assert.Contains(t, js.Tasks[0].NewCluster.ForceSendFields, "NumWorkers", + "single-node task cluster must force-send num_workers") + assert.NotContains(t, js.Tasks[1].NewCluster.ForceSendFields, "NumWorkers", + "autoscale task cluster must not force-send num_workers") + assert.Contains(t, js.JobClusters[0].NewCluster.ForceSendFields, "NumWorkers", + "single-node job cluster must force-send num_workers") +} diff --git a/bundle/fuzz/compare_cases_test.go b/bundle/fuzz/compare_cases_test.go index 95c732750b9..1549b3de23a 100644 --- a/bundle/fuzz/compare_cases_test.go +++ b/bundle/fuzz/compare_cases_test.go @@ -13,7 +13,7 @@ func TestDiffPayloads(t *testing.T) { name string direct string terraform string - ignore []ignoreRule + ignore []string want []string }{ { @@ -62,7 +62,7 @@ func TestDiffPayloads(t *testing.T) { name: "ignored path", direct: `{"tasks":[{"timeout_seconds":1}]}`, terraform: `{"tasks":[{"timeout_seconds":2}]}`, - ignore: []ignoreRule{{Path: "tasks[*].timeout_seconds"}}, + ignore: []string{"tasks[*].timeout_seconds"}, want: nil, }, { @@ -75,7 +75,7 @@ func TestDiffPayloads(t *testing.T) { name: "dotted map key can be ignored", direct: `{"c":{"spark_conf":{"spark.x.y":"1"}}}`, terraform: `{"c":{"spark_conf":{}}}`, - ignore: []ignoreRule{{Path: `c.spark_conf["spark.x.y"]`}}, + ignore: []string{`c.spark_conf["spark.x.y"]`}, want: nil, }, { @@ -102,24 +102,6 @@ func TestDiffPayloads(t *testing.T) { terraform: `{"job_clusters":[{"job_cluster_key":"y","new_cluster":{"num_workers":2}},{"job_cluster_key":"x","new_cluster":{"num_workers":1}}]}`, want: nil, }, - { - // The documented single-node divergence: direct omits num_workers, - // terraform force-sends 0. defaultIgnoreRules suppresses exactly this. - name: "task num_workers absent-vs-zero is ignored", - direct: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x"}}]}`, - terraform: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x","num_workers":0}}]}`, - ignore: defaultIgnoreRules, - want: nil, - }, - { - // A real num_workers value mismatch shares the path but is NOT the - // benign shape, so the narrowed rule must still report it. - name: "task num_workers value mismatch still surfaces", - direct: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x","num_workers":3}}]}`, - terraform: `{"tasks":[{"task_key":"t","new_cluster":{"spark_version":"x","num_workers":5}}]}`, - ignore: defaultIgnoreRules, - want: []string{"tasks[0].new_cluster.num_workers"}, - }, } for _, tt := range tests { @@ -135,14 +117,3 @@ func TestDiffPayloads(t *testing.T) { }) } } - -func TestIsBenignTaskNumWorkers(t *testing.T) { - assert.True(t, isBenignTaskNumWorkers(difference{Direct: missing{}, Terraform: json.Number("0")}), - "direct absent + terraform 0 is the documented divergence") - assert.False(t, isBenignTaskNumWorkers(difference{Direct: json.Number("3"), Terraform: json.Number("5")}), - "two differing counts is a real divergence") - assert.False(t, isBenignTaskNumWorkers(difference{Direct: missing{}, Terraform: json.Number("2")}), - "direct absent but terraform non-zero is not the benign shape") - assert.False(t, isBenignTaskNumWorkers(difference{Direct: json.Number("0"), Terraform: missing{}}), - "reversed sides are not the benign shape") -} diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go index de34c18aaf4..1681e171799 100644 --- a/bundle/fuzz/compare_test.go +++ b/bundle/fuzz/compare_test.go @@ -36,20 +36,10 @@ func render(v any) string { return string(b) } -// ignoreRule suppresses a known, intentional engine divergence. A rule matches a -// difference when the difference's normalized path equals Path and, if Match is -// non-nil, Match also reports true for the two values. A nil Match ignores any -// difference at Path; a non-nil Match narrows the rule to specific values so a -// genuine mismatch at the same path is still reported. -type ignoreRule struct { - Path string - Match func(d difference) bool -} - -// diffPayloads decodes both create payloads and returns every difference that no -// ignore rule suppresses. Paths are matched with "[*]" standing in for any slice -// index (see normalizePath). -func diffPayloads(direct, terraform json.RawMessage, ignore []ignoreRule) ([]difference, error) { +// diffPayloads decodes both create payloads and returns every difference whose +// normalized path is not in ignore ("[*]" stands in for any slice index, see +// normalizePath). +func diffPayloads(direct, terraform json.RawMessage, ignore []string) ([]difference, error) { d, err := decode(direct) if err != nil { return nil, fmt.Errorf("decoding direct payload: %w", err) @@ -64,30 +54,15 @@ func diffPayloads(direct, terraform json.RawMessage, ignore []ignoreRule) ([]dif filtered := diffs[:0] for _, diff := range diffs { - if !ignored(diff, ignore) { + if !slices.Contains(ignore, normalizePath(diff.Path)) { filtered = append(filtered, diff) } } return filtered, nil } -// ignored reports whether any rule suppresses d. -func ignored(d difference, rules []ignoreRule) bool { - norm := normalizePath(d.Path) - for _, r := range rules { - if r.Path != norm { - continue - } - if r.Match == nil || r.Match(d) { - return true - } - } - return false -} - -// decode unmarshals JSON using UseNumber so large int64 values (e.g. job ids, -// spark_context_id) are not corrupted by float64 rounding. See the encoding rule -// in the repo style guide. +// decode unmarshals JSON with UseNumber so large int64 values (job ids, +// spark_context_id) aren't corrupted by float64 rounding. func decode(raw json.RawMessage) (any, error) { if len(raw) == 0 { return nil, nil @@ -129,10 +104,8 @@ func diffValue(path string, a, b any, diffs *[]difference) { *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) return } - // Slices whose elements carry a natural identity key (tasks, job clusters) - // are matched by that key so an engine emitting the same elements in a - // different order is not reported as a difference. Everything else is - // compared positionally. + // Match keyed slices (tasks, job clusters) by identity so a different emit + // order isn't a difference; everything else is compared positionally. if key := identityKey(av, bv); key != "" { diffKeyedSlice(path, key, av, bv, diffs) return @@ -157,13 +130,11 @@ func diffValue(path string, a, b any, diffs *[]difference) { } // identityFields are the keys, in priority order, that uniquely identify the -// elements of a payload slice. Job tasks and shared job clusters are the slices -// whose order is not significant but which the engines may emit differently. +// elements of order-insensitive payload slices (job tasks, shared job clusters). var identityFields = []string{"task_key", "job_cluster_key"} // identityKey returns the field that identifies every element of both slices, or -// "" if the elements are not uniformly keyed objects (in which case the caller -// falls back to positional comparison). +// "" if they are not uniformly keyed objects (caller then compares positionally). func identityKey(a, b []any) string { for _, field := range identityFields { if allHaveKey(a, field) && allHaveKey(b, field) { @@ -189,16 +160,11 @@ func allHaveKey(s []any, field string) bool { return true } -// diffKeyedSlice matches elements of a and b by the value of key (which is unique -// within each slice for tasks/job clusters) and diffs each matched pair, -// reporting unmatched elements as present-on-one-side. Paths keep numeric indices -// so ignore-path [*] normalization still applies. +// diffKeyedSlice matches elements of a and b by key (unique within each slice for +// tasks/job clusters by API contract) and diffs each matched pair, reporting +// unmatched elements as present-on-one-side. Paths keep numeric indices so [*] +// normalization still applies. Duplicate keys would be last-one-wins. func diffKeyedSlice(path, key string, a, b []any, diffs *[]difference) { - // identityFields are unique within a slice by API contract (no two job tasks - // share a task_key, no two job_clusters share a job_cluster_key), so keying by - // them is unambiguous. If a payload ever repeated a key, last-one-wins here and - // the duplicate would be mismatched rather than reported precisely; callers - // outside the job-create harness must not rely on this for non-unique keys. bByKey := make(map[string]any, len(b)) for _, el := range b { bByKey[el.(map[string]any)[key].(string)] = el @@ -256,10 +222,8 @@ func unionKeys(a, b map[string]any) []string { } func joinKey(path, key string) string { - // Map keys can themselves contain dots or brackets (e.g. spark_conf entries - // like "spark.databricks.delta.preview.enabled"). Render those as bracketed, - // quoted segments so the path stays unambiguous and ignore entries can target - // a single key. + // Map keys can contain dots/brackets (e.g. spark_conf keys), so render those as + // bracketed quoted segments to keep the path unambiguous. if key == "" || strings.ContainsAny(key, `.[]"`) { return path + "[" + strconv.Quote(key) + "]" } @@ -279,36 +243,11 @@ func normalizePath(path string) string { return indexRe.ReplaceAllString(path, "[*]") } -// defaultIgnoreRules lists create-payload divergences that are known, intentional -// engine differences and not parity bugs. Keep this list small and -// well-justified; every entry is a documented divergence. -var defaultIgnoreRules = []ignoreRule{ - // The terraform provider strips the deprecated/ignored spark conf - // "spark.databricks.delta.preview.enabled" from new_cluster.spark_conf, while - // the direct engine forwards it verbatim. The backend ignores the key either - // way, so this is a benign provider-side filter rather than a parity bug. - {Path: `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`}, - {Path: `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`}, - - // For a single-node task-level new_cluster (no autoscale, num_workers unset) - // the terraform provider force-sends num_workers:0 while the direct engine - // omits the field, so the create payloads diverge. This is a real - // terraform/direct divergence the harness found (seed 29); it is documented - // and suppressed here rather than fixed in this PR. Tracked under DECO-25361. - // Shared job_clusters are not affected: resourcemutator already force-sends - // num_workers for them under both engines, so only the task path diverges. - // - // Match narrows this to exactly that shape (direct absent, terraform 0); a - // genuine num_workers value mismatch at the same path is still reported. - {Path: `tasks[*].new_cluster.num_workers`, Match: isBenignTaskNumWorkers}, -} - -// isBenignTaskNumWorkers reports whether d is the single documented num_workers -// divergence: the direct engine omits num_workers while terraform force-sends 0. -// Any other pair of values (in particular two differing non-zero counts) is a -// real divergence and must not be suppressed. -func isBenignTaskNumWorkers(d difference) bool { - _, directAbsent := d.Direct.(missing) - n, ok := d.Terraform.(json.Number) - return directAbsent && ok && n.String() == "0" +// defaultIgnorePaths lists known, intentional engine divergences. Keep it small; +// every entry is a documented difference, not a parity bug. +var defaultIgnorePaths = []string{ + // Terraform strips the deprecated "spark.databricks.delta.preview.enabled" from + // spark_conf while direct forwards it. The backend ignores it either way. + `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, + `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, } diff --git a/bundle/fuzz/deploy_smoke_test.go b/bundle/fuzz/deploy_smoke_test.go index dd90b359902..f6f9e5ea39f 100644 --- a/bundle/fuzz/deploy_smoke_test.go +++ b/bundle/fuzz/deploy_smoke_test.go @@ -8,12 +8,9 @@ import ( "github.com/stretchr/testify/require" ) -// TestCaptureJobCreateDirect is intentionally NOT gated behind requireFuzzOptIn, -// unlike the terraform parity suite. The direct engine needs no provisioned -// terraform, and one deterministic direct deploy is cheap, so this runs on every -// `task test` as a smoke test that the capture harness and the direct create path -// still work. The expensive part the opt-in protects against is the terraform -// side (two real deploys per seed), which stays opt-in via requireTerraform. +// TestCaptureJobCreateDirect is intentionally NOT opt-in gated: a single direct +// deploy is cheap, so it runs on every `task test` as a smoke test of the capture +// harness. The expensive terraform side stays opt-in via requireTerraform. func TestCaptureJobCreateDirect(t *testing.T) { job := generateJob(newRNG(1)) diff --git a/bundle/fuzz/deploy_test.go b/bundle/fuzz/deploy_test.go index 2328e0354e2..3a738b9cfaf 100644 --- a/bundle/fuzz/deploy_test.go +++ b/bundle/fuzz/deploy_test.go @@ -20,18 +20,10 @@ const ( ) // captureJobCreate deploys a bundle containing job through the given engine -// ("direct" or "terraform") and returns the create request body sent to the -// Jobs API. -// -// Both engines run the full `bundle deploy` pipeline against an in-process -// testserver, so the only difference between two captures with different engines -// is the engine itself. That is what makes the resulting payloads directly -// comparable: shared mutators (deployment metadata, presets, ...) are applied -// identically on both sides and cancel out in the diff. -// -// The terraform engine additionally requires DATABRICKS_TF_EXEC_PATH and -// DATABRICKS_TF_CLI_CONFIG_FILE to point at a provisioned terraform binary and -// provider mirror; see requireTerraform. +// ("direct" or "terraform") and returns the create request body sent to the Jobs +// API. Both engines run the full `bundle deploy` against an in-process testserver, +// so shared mutators cancel out and the only difference in the payloads is the +// engine itself. Terraform additionally needs the env from requireTerraform. func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, engine string) (json.RawMessage, error) { rec := &recorder{} server := testserver.New(t) @@ -61,9 +53,8 @@ func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, eng return body, nil } -// writeJobBundle writes a minimal databricks.yml describing a single job. The -// document is emitted as JSON, which is valid YAML, so we can reuse the job's -// own JSON marshaling (which honors ForceSendFields) without a YAML dependency. +// writeJobBundle writes a minimal databricks.yml for a single job. It emits JSON +// (valid YAML) to reuse the job's own marshaling, which honors ForceSendFields. func writeJobBundle(dir, host string, job *resources.Job) error { jobJSON, err := json.Marshal(job) if err != nil { @@ -91,16 +82,12 @@ func writeJobBundle(dir, host string, job *resources.Job) error { return os.WriteFile(filepath.Join(dir, "databricks.yml"), data, 0o600) } -// fuzzOptInVars are the environment variables that opt a run into the -// terraform-backed parity suite. FUZZ_SEED / FUZZ_SEEDS / FUZZ_SEED_OFFSET double -// as the tuning knobs (see paritySeeds), so setting any of them implies opt-in; -// FUZZ_PARITY is a no-tuning switch used by `task test-fuzz`. +// fuzzOptInVars opt a run into the terraform parity suite. FUZZ_SEED(S)/OFFSET also +// tune it (see paritySeeds); FUZZ_PARITY is a no-tuning switch for `task test-fuzz`. var fuzzOptInVars = []string{"FUZZ_PARITY", "FUZZ_SEED", "FUZZ_SEEDS", "FUZZ_SEED_OFFSET"} -// requireFuzzOptIn skips unless the run explicitly opted into the terraform -// parity suite. Gating on an env var rather than on the presence of build/ keeps -// a leftover terraform install (from a prior `task test-fuzz` or acceptance run) -// from silently turning a plain `task test` into dozens of real deploys. +// requireFuzzOptIn skips unless a FUZZ_* var is set. Gating on an env var rather +// than on a leftover build/ keeps a plain `task test` from running real deploys. func requireFuzzOptIn(t testing.TB) { for _, name := range fuzzOptInVars { if os.Getenv(name) != "" { @@ -111,9 +98,8 @@ func requireFuzzOptIn(t testing.TB) { } // requireTerraform opts in via requireFuzzOptIn, then points the terraform engine -// at the binary and provider mirror provisioned by acceptance/install_terraform.py -// into /build, skipping when they are absent so the suite still skips -// cleanly where terraform is not set up. +// at the binary and provider mirror that acceptance/install_terraform.py provisions +// into /build, skipping cleanly when they are absent. func requireTerraform(t testing.TB) { requireFuzzOptIn(t) @@ -121,9 +107,8 @@ func requireTerraform(t testing.TB) { execPath := filepath.Join(buildDir, "terraform") cfgFile := filepath.Join(buildDir, ".terraformrc") - // install_terraform.py provisions all three together; a partial build/ (e.g. - // the binary without the provider mirror or .terraformrc) would otherwise fail - // mid-deploy with a confusing error instead of skipping cleanly. + // Require all three together; a partial build/ would otherwise fail mid-deploy + // instead of skipping cleanly. tfpluginsDir := filepath.Join(buildDir, "tfplugins") for _, p := range []string{execPath, cfgFile, tfpluginsDir} { if _, err := os.Stat(p); err != nil { @@ -134,8 +119,7 @@ func requireTerraform(t testing.TB) { t.Setenv("DATABRICKS_TF_EXEC_PATH", execPath) t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", cfgFile) t.Setenv("TF_CLI_CONFIG_FILE", cfgFile) - // Terraform phones home to checkpoint-api.hashicorp.com otherwise; disable it - // so the testserver/network isn't hit. See acceptance_test.go. + // Disable terraform's checkpoint-api.hashicorp.com phone-home. See acceptance_test.go. t.Setenv("CHECKPOINT_DISABLE", "1") } diff --git a/bundle/fuzz/doc.go b/bundle/fuzz/doc.go index cf898d3ec14..10608ae2489 100644 --- a/bundle/fuzz/doc.go +++ b/bundle/fuzz/doc.go @@ -1,17 +1,8 @@ -// Package fuzz provides randomized generators and harnesses that compare how the -// terraform and direct deploy engines translate the same bundle resource into an -// API create payload. See DECO-25361. +// Package fuzz compares how the terraform and direct deploy engines translate the +// same bundle resource into an API create payload, catching divergences during the +// migration off terraform. Generators are seeded so any divergence reproduces from +// the printed seed. Jobs only for now (DECO-25361). // -// The first technique implemented here generates a random resource config and -// checks for differences in the create payload between the terraform and direct -// engines. Generators are seeded so that any divergence found by the fuzz driver -// can be reproduced from the printed seed. -// -// Only jobs are covered for now. Extending the harness to other resource kinds -// (pipelines, apps, ...) is tracked as follow-up work under DECO-25361. -// -// Everything else in the package lives in _test.go files: the package is a -// test-only utility and nothing in the product imports it, so keeping the logic -// out of the regular build avoids shipping dead code. This file exists only to -// carry the package documentation in a non-test file. +// Everything lives in _test.go files: the package is test-only and nothing in the +// product imports it. This file exists only to carry the package doc. package fuzz diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 596ac99f7db..33c0b3963e0 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -11,32 +11,21 @@ import ( "github.com/stretchr/testify/require" ) -// defaultParitySeeds is the number of random jobs TestJobCreateParity checks by -// default. Each seed runs two real deploys (direct + terraform), so the count is -// kept modest; override with FUZZ_SEEDS for a deeper local run. +// defaultParitySeeds is how many random jobs TestJobCreateParity checks by default. +// Each seed runs two real deploys, so keep it modest; override with FUZZ_SEEDS. const defaultParitySeeds = 20 -// regressionSeeds are seeds that previously surfaced a terraform/direct create -// payload divergence. They are always checked (in addition to the rotating -// nightly window) so the divergence keeps being exercised even though the -// nightly window moves on every run and would otherwise never revisit them. +// regressionSeeds are seeds that previously surfaced a divergence. They are always +// checked (on top of the rotating nightly window, which never revisits them) so a +// fixed divergence can't silently regress. When the nightly job reports a new +// failing FUZZ_SEED, add it here in the PR that fixes the divergence. // -// When the nightly job reports a new failing FUZZ_SEED, add it here. -// -// - 29: generates a single-node task-level new_cluster (num_workers 0, no -// autoscale). The direct engine omits num_workers on task clusters while -// terraform force-sends num_workers:0, so the create payloads diverge. That -// specific shape is suppressed by defaultIgnoreRules (see -// isBenignTaskNumWorkers), so seed 29 currently asserts only that nothing -// else about this config diverges. Once the divergence is fixed and its -// ignore rule removed, this seed becomes a full guard against it regressing. -// Tracked under DECO-25361. +// - 29: single-node task new_cluster; direct omitted num_workers while terraform +// force-sent 0. Fixed by initializeNumWorkers on task clusters (DECO-25361). var regressionSeeds = []int64{29} -// TestJobCreateParity is the first DECO-25361 technique: for many random job -// configs, assert the terraform and direct engines produce equivalent create -// payloads. On divergence it prints the seed and the generated job so the failure -// can be reproduced and inspected. +// TestJobCreateParity asserts the terraform and direct engines produce equivalent +// create payloads for many random jobs, printing the seed on divergence. func TestJobCreateParity(t *testing.T) { requireTerraform(t) @@ -49,17 +38,11 @@ func TestJobCreateParity(t *testing.T) { // paritySeeds returns the seeds TestJobCreateParity should check. // -// FUZZ_SEED (comma-separated list) runs exactly those seeds and overrides -// everything else. This is the knob the failure message prints so a single -// reported divergence can be reproduced with one command, without re-running -// every seed before it. -// -// Otherwise the test runs the regressionSeeds plus FUZZ_SEEDS seeds (default -// defaultParitySeeds) starting at FUZZ_SEED_OFFSET. The offset lets the nightly -// job shift the window every run (push.yml derives it from the run number) so CI -// explores configs it has never tested before instead of re-checking the same -// fixed set forever; the regressionSeeds are always included on top so known -// past divergences keep being verified. +// FUZZ_SEED (comma-separated) runs exactly those seeds and overrides everything, +// so a reported divergence reproduces with one command. Otherwise it runs +// regressionSeeds plus FUZZ_SEEDS seeds (default defaultParitySeeds) from +// FUZZ_SEED_OFFSET; the nightly job shifts the offset each run so CI keeps +// exploring new configs. func paritySeeds(t *testing.T) []int64 { if v := os.Getenv("FUZZ_SEED"); v != "" { var seeds []int64 @@ -112,10 +95,8 @@ func paritySeeds(t *testing.T) []int64 { // TestParitySeeds verifies paritySeeds composes the regression seeds with the // rotating window, deduplicates overlaps, and lets FUZZ_SEED override both. func TestParitySeeds(t *testing.T) { - // Isolate from any ambient FUZZ_* in the developer's environment. FUZZ_SEED in - // particular would short-circuit paritySeeds and break the cases below; an - // inherited FUZZ_SEEDS/OFFSET would skew the expected window. paritySeeds - // treats "" as unset, and subtests set only what they need on top. + // Isolate from ambient FUZZ_* in the dev environment (paritySeeds treats "" as + // unset); subtests set only what they need. t.Setenv("FUZZ_SEED", "") t.Setenv("FUZZ_SEEDS", "") t.Setenv("FUZZ_SEED_OFFSET", "") @@ -146,16 +127,14 @@ func TestParitySeeds(t *testing.T) { }) } -// FuzzJobCreateParity exposes the same parity check to Go's native fuzzer -// (`go test -fuzz=FuzzJobCreateParity`). Note each input runs two real deploys, -// so this is intended for ad-hoc deep runs, not the default `go test` path. +// FuzzJobCreateParity exposes the parity check to Go's native fuzzer. Each input +// runs two real deploys, so it's for ad-hoc deep runs, not the default test path. func FuzzJobCreateParity(f *testing.F) { requireTerraform(f) for seed := range int64(5) { f.Add(seed) } - // Seed the corpus with known past divergences so the fuzzer always starts - // from inputs that previously exposed a bug. + // Seed the corpus with known past divergences. for _, seed := range regressionSeeds { f.Add(seed) } @@ -164,20 +143,12 @@ func FuzzJobCreateParity(f *testing.F) { }) } -// checkJobParity generates the job for seed, deploys it under both engines, and -// fails the test with reproduction details if the create payloads diverge. -// -// A deploy/capture failure is not a create-payload divergence, so the three -// outcomes are handled distinctly to keep nightly triage from misdirecting a -// deploy failure into regressionSeeds (which is only for real payload diffs): -// - neither engine deployed: the generator produced a config nothing accepts, -// so skip (logging both errors) rather than flag a parity bug. -// - exactly one engine deployed: the engines disagree on whether the config -// deploys at all. That is worth failing on, but it is a deploy/capture -// difference rather than a payload diff, so it is reported separately. The -// failing side's error (an API rejection, an unregistered route, etc.) is -// included so triage can tell a true acceptance divergence from a harness gap. -// - both deployed: compare the captured create payloads. +// checkJobParity deploys the seed's job under both engines and fails if the create +// payloads diverge. A deploy/capture failure is not a payload divergence, so the +// outcomes are kept distinct: +// - neither deployed: skip (the config is unacceptable to both engines). +// - one deployed: fail separately as a deploy/capture difference, not a diff. +// - both deployed: compare the captured payloads. func checkJobParity(t *testing.T, seed int64) { t.Helper() job := generateJob(newRNG(seed)) @@ -195,7 +166,7 @@ func checkJobParity(t *testing.T, seed int64) { t.Fatalf("seed %d: direct deployed but terraform did not (deploy/capture difference, not a payload diff): %v", seed, tfErr) } - diffs, err := diffPayloads(direct, terraform, defaultIgnoreRules) + diffs, err := diffPayloads(direct, terraform, defaultIgnorePaths) require.NoErrorf(t, err, "seed %d: comparing create payloads", seed) if len(diffs) > 0 { diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go index 6472957b2f4..7a96c1868cd 100644 --- a/bundle/fuzz/generate_test.go +++ b/bundle/fuzz/generate_test.go @@ -11,9 +11,8 @@ import ( ) // Value pools are intentionally small and valid-looking: the goal is to exercise -// the engines' config->payload translation across many field combinations, not to -// stress the API with invalid values (which the testserver would reject before we -// can compare payloads). +// config->payload translation across many field combinations, not to stress the +// API with invalid values the testserver would reject. var ( sparkVersions = []string{"13.3.x-scala2.12", "14.3.x-scala2.12", "15.4.x-scala2.12", "16.4.x-scala2.12"} nodeTypeIDs = []string{"i3.xlarge", "m5.large", "r5.xlarge", "Standard_DS3_v2"} @@ -29,12 +28,10 @@ var ( ) // generateJob builds a random, well-formed job config driven entirely by rng, so -// the same seed always produces the same job. It deliberately favors fields whose -// translation tends to differ between engines (tasks, clusters, schedules, -// notifications, tags, zero-able scalars). +// the same seed always produces the same job. It favors fields whose translation +// tends to differ between engines. // -// TODO(DECO-25361): generalize the harness across resource kinds so pipelines, -// apps, etc. get the same create-payload parity coverage as jobs. +// TODO(DECO-25361): generalize the harness across resource kinds. func generateJob(rng *rand.Rand) *resources.Job { job := &resources.Job{} job.Name = randName(rng, "job") @@ -150,9 +147,8 @@ func randScheduling(rng *rand.Rand, job *resources.Job) { func randTask(rng *rand.Rand, idx int, jobClusterKeys []string) jobs.Task { task := jobs.Task{TaskKey: fmt.Sprintf("task_%d", idx)} - // Use absolute workspace paths with source=WORKSPACE so the generated bundle - // never depends on local files existing on disk (which deploy would reject). - // condition_task needs no compute, so it is handled separately below. + // Use absolute workspace paths so deploy never depends on local files. + // condition_task needs no compute, handled separately below. needsCompute := true switch rng.IntN(4) { case 0: @@ -197,9 +193,8 @@ func randTask(rng *rand.Rand, idx int, jobClusterKeys []string) jobs.Task { return task } -// assignCompute attaches exactly one compute source, which notebook/python/wheel -// tasks require: a shared job cluster (when available), a brand-new cluster, or an -// existing cluster id. +// assignCompute attaches exactly one compute source: a shared job cluster (when +// available), a new cluster, or an existing cluster id. func assignCompute(rng *rand.Rand, task *jobs.Task, jobClusterKeys []string) { const ( computeNew = iota diff --git a/bundle/fuzz/recorder_test.go b/bundle/fuzz/recorder_test.go index a5e7d4d707f..73620d00e19 100644 --- a/bundle/fuzz/recorder_test.go +++ b/bundle/fuzz/recorder_test.go @@ -8,11 +8,8 @@ import ( ) // jobsCreatePath is the Jobs API route both engines must hit on create. The -// direct engine posts here via the SDK and the terraform provider is expected to -// as well. The testserver registers only this version of the jobs/create route, -// so if an engine ever posted to a different version the deploy would 404 and -// captureJobCreate would fail with "did not POST". A version skew therefore -// surfaces as a capture failure, not as a payload diff. +// testserver registers only this version, so an engine posting to a different one +// surfaces as a capture failure ("did not POST"), not a payload diff. const jobsCreatePath = "/api/2.2/jobs/create" // capturedRequest is a single mutating API request observed by the testserver. From 6cb3060cdc6d20bca7f42cec5e70e51db2e1e31c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 26 Jun 2026 15:44:13 +0000 Subject: [PATCH 14/53] bundle/fuzz: replace terraform/direct parity with invariant testing Switch the fuzz suite from comparing terraform and direct create payloads to asserting invariants on the direct engine's payload. Terraform and direct can disagree for legitimate reasons, so a payload diff is noisy; an invariant has no legitimate reason to fail, so a failure is a real bug. This drops the payload diff and its ignore-list of documented divergences, and removes terraform from the harness (each seed is now one in-process direct deploy). Gate on `bundle validate` so the suite distinguishes the two fuzzing outcomes: an invalid config skips (it can't violate an invariant), while a validated config that fails to deploy or breaks an invariant fails. This is the distinction a looser, schema-driven generator will rely on. Revert the num_workers:0 force-send for single-node task clusters (and its acceptance goldens): it only matched terraform's payload, with no demonstrated behavior benefit, and direct has shipped without it. If a real backend requirement is confirmed, it can return as a standalone change. --- .github/workflows/push.yml | 17 +- .gitignore | 4 - Taskfile.yml | 8 +- .../bundle/deploy/wal/chain-3-jobs/output.txt | 2 - .../deploy/wal/crash-after-create/output.txt | 1 - .../bundle/override/job_tasks/output.txt | 2 - .../missing_map_key/out.validate.direct.json | 3 +- .../out.validate.terraform.json | 3 +- .../mutator/resourcemutator/cluster_fixups.go | 3 - .../resourcemutator/cluster_fixups_test.go | 92 ------- bundle/fuzz/compare_cases_test.go | 119 -------- bundle/fuzz/compare_test.go | 253 ------------------ bundle/fuzz/deploy_smoke_test.go | 25 +- bundle/fuzz/deploy_test.go | 90 +++---- bundle/fuzz/doc.go | 11 +- bundle/fuzz/fuzz_test.go | 120 ++++----- bundle/fuzz/generate_test.go | 6 +- bundle/fuzz/invariants_cases_test.go | 93 +++++++ bundle/fuzz/invariants_test.go | 175 ++++++++++++ bundle/fuzz/recorder_test.go | 9 +- 20 files changed, 377 insertions(+), 659 deletions(-) delete mode 100644 bundle/config/mutator/resourcemutator/cluster_fixups_test.go delete mode 100644 bundle/fuzz/compare_cases_test.go delete mode 100644 bundle/fuzz/compare_test.go create mode 100644 bundle/fuzz/invariants_cases_test.go create mode 100644 bundle/fuzz/invariants_test.go diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6644c5cec3f..75af3ffbfaa 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -397,8 +397,9 @@ jobs: needs: - cleanups - # Two real deploys per seed: too slow for every PR, so nightly only and not part - # of test-result. + # A real deploy per seed across a wide rotating window: too slow for every PR, + # so nightly only and not part of test-result. (The package's un-gated smoke + # test still checks the invariants on one seed on every PR.) if: ${{ github.event_name == 'schedule' }} name: "task test-fuzz" runs-on: @@ -428,14 +429,14 @@ jobs: env: # Shift the seed window each nightly run so CI explores new configs. # offset = GITHUB_RUN_NUMBER * FUZZ_SEEDS keeps windows non-overlapping - # (GITHUB_RUN_NUMBER is monotonic). A divergence prints FUZZ_SEED=. + # (GITHUB_RUN_NUMBER is monotonic). A failure prints FUZZ_SEED=. FUZZ_SEEDS: "25" run: | export FUZZ_SEED_OFFSET=$(( GITHUB_RUN_NUMBER * FUZZ_SEEDS )) go tool -modfile=tools/task/go.mod task test-fuzz # Excluded from test-result, so surface failures as a GitHub issue. Reuse one - # open issue (deduped by label) so a recurring divergence doesn't spam nightly. + # open issue (deduped by label) so a recurring failure doesn't spam nightly. - name: Report failure if: ${{ failure() }} env: @@ -443,11 +444,11 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | gh label create fuzz-nightly \ - --description "Nightly terraform/direct create-payload parity failures" \ + --description "Nightly create-payload invariant failures" \ --color FBCA04 2>/dev/null || true body=$(cat </build. - - python3 acceptance/install_terraform.py --targetdir build - | {{.GO_TOOL}} gotestsum \ --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index 5cb87e0c097..7cbc9f31a85 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -35,7 +35,6 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { @@ -74,7 +73,6 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index d0a32c78254..e9e2c19c094 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -39,7 +39,6 @@ Exit code: [KILLED] { "new_cluster": { "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt index 59b6fc1c397..2bee9738e33 100644 --- a/acceptance/bundle/override/job_tasks/output.txt +++ b/acceptance/bundle/override/job_tasks/output.txt @@ -18,7 +18,6 @@ }, { "new_cluster": { - "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { @@ -43,7 +42,6 @@ Exit code: 1 "tasks": [ { "new_cluster": { - "num_workers": 0, "spark_version": "13.3.x-scala2.12" }, "spark_python_task": { diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json index 7279aaeba31..cfd1427ce4d 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.direct.json @@ -30,8 +30,7 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - }, - "num_workers": 0 + } }, "task_key": "test-task" } diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json index 3bad6f46193..3cdf58f84ea 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json +++ b/acceptance/bundle/resource_deps/missing_map_key/out.validate.terraform.json @@ -30,8 +30,7 @@ "new_cluster": { "custom_tags": { "ResourceClass": "SingleNode" - }, - "num_workers": 0 + } }, "task_key": "test-task" } diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups.go b/bundle/config/mutator/resourcemutator/cluster_fixups.go index ee4ee04c8be..893cd248aa4 100644 --- a/bundle/config/mutator/resourcemutator/cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/cluster_fixups.go @@ -94,9 +94,6 @@ func prepareJobSettingsForUpdate(js *jobs.JobSettings) { for _, task := range js.Tasks { if task.NewCluster != nil { ModifyRequestOnInstancePool(task.NewCluster) - // Match terraform, which force-sends num_workers:0 for single-node - // task clusters too, not just shared job_clusters (DECO-25361). - initializeNumWorkers(task.NewCluster) } } for ind := range js.JobClusters { diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups_test.go b/bundle/config/mutator/resourcemutator/cluster_fixups_test.go deleted file mode 100644 index 5cb2e937494..00000000000 --- a/bundle/config/mutator/resourcemutator/cluster_fixups_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package resourcemutator - -import ( - "testing" - - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" -) - -func TestInitializeNumWorkers(t *testing.T) { - tests := []struct { - name string - spec compute.ClusterSpec - wantForceSend bool - }{ - { - name: "single-node cluster force-sends num_workers", - spec: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, - wantForceSend: true, - }, - { - name: "autoscale cluster does not force-send", - spec: compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, - wantForceSend: false, - }, - { - name: "multi-node cluster does not force-send", - spec: compute.ClusterSpec{NumWorkers: 3}, - wantForceSend: false, - }, - { - name: "already force-sent stays force-sent without duplicating", - spec: compute.ClusterSpec{ForceSendFields: []string{"NumWorkers"}}, - wantForceSend: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - spec := tt.spec - initializeNumWorkers(&spec) - - count := 0 - for _, f := range spec.ForceSendFields { - if f == "NumWorkers" { - count++ - } - } - if tt.wantForceSend { - assert.Equal(t, 1, count, "NumWorkers must appear in ForceSendFields exactly once") - } else { - assert.Equal(t, 0, count, "NumWorkers must not be in ForceSendFields") - } - }) - } -} - -// TestPrepareJobSettingsForUpdateForcesNumWorkers locks the DECO-25361 fix: a -// single-node new_cluster must force-send num_workers on task-level clusters too, -// not just shared job_clusters. The terraform provider always sends num_workers:0 -// for such clusters, so missing it on the task side made the direct engine -// produce a divergent create payload. -func TestPrepareJobSettingsForUpdateForcesNumWorkers(t *testing.T) { - js := &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - TaskKey: "single_node_task", - NewCluster: &compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, - }, - { - TaskKey: "autoscale_task", - NewCluster: &compute.ClusterSpec{Autoscale: &compute.AutoScale{MinWorkers: 1, MaxWorkers: 4}}, - }, - }, - JobClusters: []jobs.JobCluster{ - { - JobClusterKey: "single_node_cluster", - NewCluster: compute.ClusterSpec{SparkVersion: "15.4.x-scala2.12", NodeTypeId: "i3.xlarge"}, - }, - }, - } - - prepareJobSettingsForUpdate(js) - - assert.Contains(t, js.Tasks[0].NewCluster.ForceSendFields, "NumWorkers", - "single-node task cluster must force-send num_workers") - assert.NotContains(t, js.Tasks[1].NewCluster.ForceSendFields, "NumWorkers", - "autoscale task cluster must not force-send num_workers") - assert.Contains(t, js.JobClusters[0].NewCluster.ForceSendFields, "NumWorkers", - "single-node job cluster must force-send num_workers") -} diff --git a/bundle/fuzz/compare_cases_test.go b/bundle/fuzz/compare_cases_test.go deleted file mode 100644 index 1549b3de23a..00000000000 --- a/bundle/fuzz/compare_cases_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package fuzz - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDiffPayloads(t *testing.T) { - tests := []struct { - name string - direct string - terraform string - ignore []string - want []string - }{ - { - name: "identical", - direct: `{"name":"a","tasks":[{"task_key":"t"}]}`, - terraform: `{"name":"a","tasks":[{"task_key":"t"}]}`, - want: nil, - }, - { - name: "scalar mismatch", - direct: `{"name":"a"}`, - terraform: `{"name":"b"}`, - want: []string{"name"}, - }, - { - name: "missing on terraform", - direct: `{"name":"a","queue":{"enabled":true}}`, - terraform: `{"name":"a"}`, - want: []string{"queue"}, - }, - { - name: "missing on direct", - direct: `{"name":"a"}`, - terraform: `{"name":"a","max_concurrent_runs":1}`, - want: []string{"max_concurrent_runs"}, - }, - { - name: "nested slice element mismatch", - direct: `{"tasks":[{"task_key":"t","timeout_seconds":1}]}`, - terraform: `{"tasks":[{"task_key":"t","timeout_seconds":2}]}`, - want: []string{"tasks[0].timeout_seconds"}, - }, - { - name: "slice length mismatch", - direct: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - terraform: `{"tasks":[{"task_key":"a"}]}`, - want: []string{"tasks[1]"}, - }, - { - name: "number 1 vs 1.0 differ", - direct: `{"n":1}`, - terraform: `{"n":1.0}`, - want: []string{"n"}, - }, - { - name: "ignored path", - direct: `{"tasks":[{"timeout_seconds":1}]}`, - terraform: `{"tasks":[{"timeout_seconds":2}]}`, - ignore: []string{"tasks[*].timeout_seconds"}, - want: nil, - }, - { - name: "dotted map key is bracket-quoted", - direct: `{"spark_conf":{"spark.x.y":"1"}}`, - terraform: `{"spark_conf":{}}`, - want: []string{`spark_conf["spark.x.y"]`}, - }, - { - name: "dotted map key can be ignored", - direct: `{"c":{"spark_conf":{"spark.x.y":"1"}}}`, - terraform: `{"c":{"spark_conf":{}}}`, - ignore: []string{`c.spark_conf["spark.x.y"]`}, - want: nil, - }, - { - name: "tasks matched by key ignore order", - direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, - terraform: `{"tasks":[{"task_key":"b","timeout_seconds":2},{"task_key":"a","timeout_seconds":1}]}`, - want: nil, - }, - { - name: "tasks matched by key surface real diff at direct index", - direct: `{"tasks":[{"task_key":"a","timeout_seconds":1},{"task_key":"b","timeout_seconds":2}]}`, - terraform: `{"tasks":[{"task_key":"b","timeout_seconds":9},{"task_key":"a","timeout_seconds":1}]}`, - want: []string{"tasks[1].timeout_seconds"}, - }, - { - name: "task only on terraform reported at its index", - direct: `{"tasks":[{"task_key":"a"}]}`, - terraform: `{"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - want: []string{"tasks[1]"}, - }, - { - name: "job_clusters matched by key ignore order", - direct: `{"job_clusters":[{"job_cluster_key":"x","new_cluster":{"num_workers":1}},{"job_cluster_key":"y","new_cluster":{"num_workers":2}}]}`, - terraform: `{"job_clusters":[{"job_cluster_key":"y","new_cluster":{"num_workers":2}},{"job_cluster_key":"x","new_cluster":{"num_workers":1}}]}`, - want: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - diffs, err := diffPayloads(json.RawMessage(tt.direct), json.RawMessage(tt.terraform), tt.ignore) - require.NoError(t, err) - - var paths []string - for _, d := range diffs { - paths = append(paths, d.Path) - } - assert.ElementsMatch(t, tt.want, paths) - }) - } -} diff --git a/bundle/fuzz/compare_test.go b/bundle/fuzz/compare_test.go deleted file mode 100644 index 1681e171799..00000000000 --- a/bundle/fuzz/compare_test.go +++ /dev/null @@ -1,253 +0,0 @@ -package fuzz - -import ( - "bytes" - "encoding/json" - "fmt" - "regexp" - "slices" - "strconv" - "strings" -) - -// difference is a single mismatch between the two engines' create payloads, -// located by a JSON-ish path (e.g. "tasks[0].new_cluster.num_workers"). -type difference struct { - Path string - Direct any - Terraform any -} - -func (d difference) String() string { - return fmt.Sprintf("%s: direct=%s terraform=%s", d.Path, render(d.Direct), render(d.Terraform)) -} - -// missing marks a value that is absent on one side. -type missing struct{} - -func render(v any) string { - if _, ok := v.(missing); ok { - return "" - } - b, err := json.Marshal(v) - if err != nil { - return fmt.Sprintf("%v", v) - } - return string(b) -} - -// diffPayloads decodes both create payloads and returns every difference whose -// normalized path is not in ignore ("[*]" stands in for any slice index, see -// normalizePath). -func diffPayloads(direct, terraform json.RawMessage, ignore []string) ([]difference, error) { - d, err := decode(direct) - if err != nil { - return nil, fmt.Errorf("decoding direct payload: %w", err) - } - tf, err := decode(terraform) - if err != nil { - return nil, fmt.Errorf("decoding terraform payload: %w", err) - } - - var diffs []difference - diffValue("", d, tf, &diffs) - - filtered := diffs[:0] - for _, diff := range diffs { - if !slices.Contains(ignore, normalizePath(diff.Path)) { - filtered = append(filtered, diff) - } - } - return filtered, nil -} - -// decode unmarshals JSON with UseNumber so large int64 values (job ids, -// spark_context_id) aren't corrupted by float64 rounding. -func decode(raw json.RawMessage) (any, error) { - if len(raw) == 0 { - return nil, nil - } - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - var v any - if err := dec.Decode(&v); err != nil { - return nil, err - } - return v, nil -} - -func diffValue(path string, a, b any, diffs *[]difference) { - switch av := a.(type) { - case map[string]any: - bv, ok := b.(map[string]any) - if !ok { - *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) - return - } - keys := unionKeys(av, bv) - for _, k := range keys { - achild, aok := av[k] - bchild, bok := bv[k] - child := joinKey(path, k) - switch { - case aok && bok: - diffValue(child, achild, bchild, diffs) - case aok: - *diffs = append(*diffs, difference{Path: child, Direct: achild, Terraform: missing{}}) - default: - *diffs = append(*diffs, difference{Path: child, Direct: missing{}, Terraform: bchild}) - } - } - case []any: - bv, ok := b.([]any) - if !ok { - *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) - return - } - // Match keyed slices (tasks, job clusters) by identity so a different emit - // order isn't a difference; everything else is compared positionally. - if key := identityKey(av, bv); key != "" { - diffKeyedSlice(path, key, av, bv, diffs) - return - } - n := max(len(av), len(bv)) - for i := range n { - child := fmt.Sprintf("%s[%d]", path, i) - switch { - case i < len(av) && i < len(bv): - diffValue(child, av[i], bv[i], diffs) - case i < len(av): - *diffs = append(*diffs, difference{Path: child, Direct: av[i], Terraform: missing{}}) - default: - *diffs = append(*diffs, difference{Path: child, Direct: missing{}, Terraform: bv[i]}) - } - } - default: - if !scalarEqual(a, b) { - *diffs = append(*diffs, difference{Path: path, Direct: a, Terraform: b}) - } - } -} - -// identityFields are the keys, in priority order, that uniquely identify the -// elements of order-insensitive payload slices (job tasks, shared job clusters). -var identityFields = []string{"task_key", "job_cluster_key"} - -// identityKey returns the field that identifies every element of both slices, or -// "" if they are not uniformly keyed objects (caller then compares positionally). -func identityKey(a, b []any) string { - for _, field := range identityFields { - if allHaveKey(a, field) && allHaveKey(b, field) { - return field - } - } - return "" -} - -func allHaveKey(s []any, field string) bool { - if len(s) == 0 { - return false - } - for _, el := range s { - m, ok := el.(map[string]any) - if !ok { - return false - } - if _, ok := m[field].(string); !ok { - return false - } - } - return true -} - -// diffKeyedSlice matches elements of a and b by key (unique within each slice for -// tasks/job clusters by API contract) and diffs each matched pair, reporting -// unmatched elements as present-on-one-side. Paths keep numeric indices so [*] -// normalization still applies. Duplicate keys would be last-one-wins. -func diffKeyedSlice(path, key string, a, b []any, diffs *[]difference) { - bByKey := make(map[string]any, len(b)) - for _, el := range b { - bByKey[el.(map[string]any)[key].(string)] = el - } - - matched := make(map[string]bool, len(a)) - for i, el := range a { - child := fmt.Sprintf("%s[%d]", path, i) - k := el.(map[string]any)[key].(string) - matched[k] = true - if bel, ok := bByKey[k]; ok { - diffValue(child, el, bel, diffs) - } else { - *diffs = append(*diffs, difference{Path: child, Direct: el, Terraform: missing{}}) - } - } - for j, el := range b { - k := el.(map[string]any)[key].(string) - if matched[k] { - continue - } - child := fmt.Sprintf("%s[%d]", path, j) - *diffs = append(*diffs, difference{Path: child, Direct: missing{}, Terraform: el}) - } -} - -// scalarEqual compares two JSON scalars. json.Number is compared by its string -// form so 1 and 1.0 don't masquerade as equal across engines. -func scalarEqual(a, b any) bool { - an, aok := a.(json.Number) - bn, bok := b.(json.Number) - if aok && bok { - return an.String() == bn.String() - } - return a == b -} - -func unionKeys(a, b map[string]any) []string { - seen := map[string]bool{} - var keys []string - for k := range a { - if !seen[k] { - seen[k] = true - keys = append(keys, k) - } - } - for k := range b { - if !seen[k] { - seen[k] = true - keys = append(keys, k) - } - } - slices.Sort(keys) - return keys -} - -func joinKey(path, key string) string { - // Map keys can contain dots/brackets (e.g. spark_conf keys), so render those as - // bracketed quoted segments to keep the path unambiguous. - if key == "" || strings.ContainsAny(key, `.[]"`) { - return path + "[" + strconv.Quote(key) + "]" - } - if path == "" { - return key - } - return path + "." + key -} - -// indexRe matches numeric slice indices like "[12]" but not quoted string keys -// like ["spark.x"]. -var indexRe = regexp.MustCompile(`\[\d+\]`) - -// normalizePath replaces concrete slice indices with [*] so a single ignore -// entry can cover every element of a slice. -func normalizePath(path string) string { - return indexRe.ReplaceAllString(path, "[*]") -} - -// defaultIgnorePaths lists known, intentional engine divergences. Keep it small; -// every entry is a documented difference, not a parity bug. -var defaultIgnorePaths = []string{ - // Terraform strips the deprecated "spark.databricks.delta.preview.enabled" from - // spark_conf while direct forwards it. The backend ignores it either way. - `tasks[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, - `job_clusters[*].new_cluster.spark_conf["spark.databricks.delta.preview.enabled"]`, -} diff --git a/bundle/fuzz/deploy_smoke_test.go b/bundle/fuzz/deploy_smoke_test.go index f6f9e5ea39f..0121c7468ec 100644 --- a/bundle/fuzz/deploy_smoke_test.go +++ b/bundle/fuzz/deploy_smoke_test.go @@ -1,38 +1,21 @@ package fuzz import ( - "encoding/json" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestCaptureJobCreateDirect is intentionally NOT opt-in gated: a single direct // deploy is cheap, so it runs on every `task test` as a smoke test of the capture -// harness. The expensive terraform side stays opt-in via requireTerraform. +// harness and the invariants. The wider seed sweep stays opt-in via +// requireFuzzOptIn. func TestCaptureJobCreateDirect(t *testing.T) { job := generateJob(newRNG(1)) - body, err := captureJobCreate(t.Context(), t, job, "direct") + body, err := captureJobCreate(t.Context(), t, job) require.NoError(t, err) require.NotEmpty(t, body) - var payload map[string]any - require.NoError(t, json.Unmarshal(body, &payload)) - assert.Equal(t, job.Name, payload["name"]) - assert.Contains(t, payload, "tasks") -} - -func TestCaptureJobCreateTerraform(t *testing.T) { - requireTerraform(t) - job := generateJob(newRNG(1)) - - body, err := captureJobCreate(t.Context(), t, job, "terraform") - require.NoError(t, err) - require.NotEmpty(t, body) - - var payload map[string]any - require.NoError(t, json.Unmarshal(body, &payload)) - assert.Equal(t, job.Name, payload["name"]) + checkJobInvariants(t, 1, job, body) } diff --git a/bundle/fuzz/deploy_test.go b/bundle/fuzz/deploy_test.go index 3a738b9cfaf..ddf8d4342b5 100644 --- a/bundle/fuzz/deploy_test.go +++ b/bundle/fuzz/deploy_test.go @@ -3,6 +3,7 @@ package fuzz import ( "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -19,12 +20,18 @@ const ( fakeToken = "testtoken" ) -// captureJobCreate deploys a bundle containing job through the given engine -// ("direct" or "terraform") and returns the create request body sent to the Jobs -// API. Both engines run the full `bundle deploy` against an in-process testserver, -// so shared mutators cancel out and the only difference in the payloads is the -// engine itself. Terraform additionally needs the env from requireTerraform. -func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, engine string) (json.RawMessage, error) { +// errInvalidConfig marks a generated config that `bundle validate` rejects. The +// caller skips on it: an invalid config can't violate an invariant, so it is not a +// bug. This is the distinction that makes the suite safe to point at a looser +// (e.g. schema-driven) generator, which will produce invalid configs by design. +var errInvalidConfig = errors.New("config did not validate") + +// captureJobCreate validates then deploys a bundle containing job via the direct +// engine against an in-process testserver, returning the create request body sent +// to the Jobs API. A validation failure is wrapped as errInvalidConfig. The +// invariant suite asserts properties of the payload; the terraform engine is not +// involved (we assert fundamental properties rather than compare engines). +func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job) (json.RawMessage, error) { rec := &recorder{} server := testserver.New(t) server.RequestCallback = rec.callback @@ -37,18 +44,24 @@ func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job, eng t.Setenv("DATABRICKS_HOST", server.URL) t.Setenv("DATABRICKS_TOKEN", fakeToken) - t.Setenv("DATABRICKS_BUNDLE_ENGINE", engine) + t.Setenv("DATABRICKS_BUNDLE_ENGINE", "direct") t.Chdir(dir) + // Validate first so an invalid config is reported as errInvalidConfig (caller + // skips) rather than a deploy failure (caller fails). + if _, stderr, err := testcli.NewRunner(t, ctx, "bundle", "validate").Run(); err != nil { + return nil, fmt.Errorf("%w: %v\nstderr:\n%s", errInvalidConfig, err, stderr.String()) + } + stdout, stderr, err := testcli.NewRunner(t, ctx, "bundle", "deploy").Run() if err != nil { - return nil, fmt.Errorf("bundle deploy (engine=%s) failed: %w\nstdout:\n%s\nstderr:\n%s", - engine, err, stdout.String(), stderr.String()) + return nil, fmt.Errorf("bundle deploy failed: %w\nstdout:\n%s\nstderr:\n%s", + err, stdout.String(), stderr.String()) } body, ok := rec.find("POST", jobsCreatePath) if !ok { - return nil, fmt.Errorf("engine=%s did not POST %s during deploy", engine, jobsCreatePath) + return nil, fmt.Errorf("deploy did not POST %s", jobsCreatePath) } return body, nil } @@ -82,61 +95,18 @@ func writeJobBundle(dir, host string, job *resources.Job) error { return os.WriteFile(filepath.Join(dir, "databricks.yml"), data, 0o600) } -// fuzzOptInVars opt a run into the terraform parity suite. FUZZ_SEED(S)/OFFSET also -// tune it (see paritySeeds); FUZZ_PARITY is a no-tuning switch for `task test-fuzz`. -var fuzzOptInVars = []string{"FUZZ_PARITY", "FUZZ_SEED", "FUZZ_SEEDS", "FUZZ_SEED_OFFSET"} +// fuzzOptInVars opt a run into the invariant suite. FUZZ_SEED(S)/OFFSET also tune +// it (see invariantSeeds); FUZZ_INVARIANTS is a no-tuning switch for `task test-fuzz`. +var fuzzOptInVars = []string{"FUZZ_INVARIANTS", "FUZZ_SEED", "FUZZ_SEEDS", "FUZZ_SEED_OFFSET"} -// requireFuzzOptIn skips unless a FUZZ_* var is set. Gating on an env var rather -// than on a leftover build/ keeps a plain `task test` from running real deploys. +// requireFuzzOptIn skips unless a FUZZ_* var is set. Each seed runs a real +// in-process deploy, so gating keeps a plain `task test` fast (the single +// un-gated direct smoke test still exercises the harness on every run). func requireFuzzOptIn(t testing.TB) { for _, name := range fuzzOptInVars { if os.Getenv(name) != "" { return } } - t.Skip("terraform parity suite is opt-in; run `task test-fuzz` or set FUZZ_SEED= to reproduce a single seed") -} - -// requireTerraform opts in via requireFuzzOptIn, then points the terraform engine -// at the binary and provider mirror that acceptance/install_terraform.py provisions -// into /build, skipping cleanly when they are absent. -func requireTerraform(t testing.TB) { - requireFuzzOptIn(t) - - buildDir := filepath.Join(repoRoot(t), "build") - execPath := filepath.Join(buildDir, "terraform") - cfgFile := filepath.Join(buildDir, ".terraformrc") - - // Require all three together; a partial build/ would otherwise fail mid-deploy - // instead of skipping cleanly. - tfpluginsDir := filepath.Join(buildDir, "tfplugins") - for _, p := range []string{execPath, cfgFile, tfpluginsDir} { - if _, err := os.Stat(p); err != nil { - t.Skipf("terraform not fully provisioned (%s); run: python3 acceptance/install_terraform.py --targetdir build", p) - } - } - - t.Setenv("DATABRICKS_TF_EXEC_PATH", execPath) - t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", cfgFile) - t.Setenv("TF_CLI_CONFIG_FILE", cfgFile) - // Disable terraform's checkpoint-api.hashicorp.com phone-home. See acceptance_test.go. - t.Setenv("CHECKPOINT_DISABLE", "1") -} - -// repoRoot returns the repository root by walking up from the current directory. -func repoRoot(t testing.TB) string { - dir, err := os.Getwd() - if err != nil { - t.Fatalf("getwd: %s", err) - } - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - return dir - } - parent := filepath.Dir(dir) - if parent == dir { - t.Fatal("could not locate repo root (go.mod not found)") - } - dir = parent - } + t.Skip("invariant fuzz suite is opt-in; run `task test-fuzz` or set FUZZ_SEED= to reproduce a single seed") } diff --git a/bundle/fuzz/doc.go b/bundle/fuzz/doc.go index 10608ae2489..59b04170963 100644 --- a/bundle/fuzz/doc.go +++ b/bundle/fuzz/doc.go @@ -1,7 +1,10 @@ -// Package fuzz compares how the terraform and direct deploy engines translate the -// same bundle resource into an API create payload, catching divergences during the -// migration off terraform. Generators are seeded so any divergence reproduces from -// the printed seed. Jobs only for now (DECO-25361). +// Package fuzz deploys randomly generated bundle resources through the direct +// engine and asserts invariants that any valid config's API create payload must +// satisfy (e.g. task keys are preserved, references resolve, a new_cluster is +// sized by autoscale or num_workers but not both). Unlike a terraform/direct +// payload comparison, an invariant has no legitimate reason to fail, so a failure +// is a real bug. Generators are seeded so any failure reproduces from the printed +// seed. Jobs only for now. // // Everything lives in _test.go files: the package is test-only and nothing in the // product imports it. This file exists only to carry the package doc. diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go index 33c0b3963e0..f8d758e88ae 100644 --- a/bundle/fuzz/fuzz_test.go +++ b/bundle/fuzz/fuzz_test.go @@ -2,6 +2,7 @@ package fuzz import ( "encoding/json" + "errors" "os" "strconv" "strings" @@ -11,39 +12,38 @@ import ( "github.com/stretchr/testify/require" ) -// defaultParitySeeds is how many random jobs TestJobCreateParity checks by default. -// Each seed runs two real deploys, so keep it modest; override with FUZZ_SEEDS. -const defaultParitySeeds = 20 +// defaultInvariantSeeds is how many random jobs TestJobInvariants checks by +// default. Each seed runs a real deploy, so keep it modest; override with +// FUZZ_SEEDS. +const defaultInvariantSeeds = 20 -// regressionSeeds are seeds that previously surfaced a divergence. They are always +// regressionSeeds are seeds that previously broke an invariant. They are always // checked (on top of the rotating nightly window, which never revisits them) so a -// fixed divergence can't silently regress. When the nightly job reports a new -// failing FUZZ_SEED, add it here in the PR that fixes the divergence. -// -// - 29: single-node task new_cluster; direct omitted num_workers while terraform -// force-sent 0. Fixed by initializeNumWorkers on task clusters (DECO-25361). -var regressionSeeds = []int64{29} +// fixed bug can't silently regress. When the nightly job reports a new failing +// FUZZ_SEED, add it here in the PR that fixes it. Empty until the first such bug. +var regressionSeeds = []int64{} -// TestJobCreateParity asserts the terraform and direct engines produce equivalent -// create payloads for many random jobs, printing the seed on divergence. -func TestJobCreateParity(t *testing.T) { - requireTerraform(t) +// TestJobInvariants asserts the engine produces a create payload satisfying the +// invariants in checkJobInvariants for many random jobs, printing the seed on +// failure. +func TestJobInvariants(t *testing.T) { + requireFuzzOptIn(t) - for _, seed := range paritySeeds(t) { + for _, seed := range invariantSeeds(t) { t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { - checkJobParity(t, seed) + checkJob(t, seed) }) } } -// paritySeeds returns the seeds TestJobCreateParity should check. +// invariantSeeds returns the seeds TestJobInvariants should check. // // FUZZ_SEED (comma-separated) runs exactly those seeds and overrides everything, -// so a reported divergence reproduces with one command. Otherwise it runs -// regressionSeeds plus FUZZ_SEEDS seeds (default defaultParitySeeds) from +// so a reported failure reproduces with one command. Otherwise it runs +// regressionSeeds plus FUZZ_SEEDS seeds (default defaultInvariantSeeds) from // FUZZ_SEED_OFFSET; the nightly job shifts the offset each run so CI keeps // exploring new configs. -func paritySeeds(t *testing.T) []int64 { +func invariantSeeds(t *testing.T) []int64 { if v := os.Getenv("FUZZ_SEED"); v != "" { var seeds []int64 for part := range strings.SplitSeq(v, ",") { @@ -59,7 +59,7 @@ func paritySeeds(t *testing.T) []int64 { return seeds } - count := defaultParitySeeds + count := defaultInvariantSeeds if v := os.Getenv("FUZZ_SEEDS"); v != "" { n, err := strconv.Atoi(v) require.NoErrorf(t, err, "invalid FUZZ_SEEDS=%q", v) @@ -92,89 +92,63 @@ func paritySeeds(t *testing.T) []int64 { return seeds } -// TestParitySeeds verifies paritySeeds composes the regression seeds with the -// rotating window, deduplicates overlaps, and lets FUZZ_SEED override both. -func TestParitySeeds(t *testing.T) { - // Isolate from ambient FUZZ_* in the dev environment (paritySeeds treats "" as - // unset); subtests set only what they need. +// TestInvariantSeeds verifies invariantSeeds composes the regression seeds with +// the rotating window, deduplicates overlaps, and lets FUZZ_SEED override both. +func TestInvariantSeeds(t *testing.T) { + // Isolate from ambient FUZZ_* in the dev environment (invariantSeeds treats "" + // as unset); subtests set only what they need. t.Setenv("FUZZ_SEED", "") t.Setenv("FUZZ_SEEDS", "") t.Setenv("FUZZ_SEED_OFFSET", "") - t.Run("default includes regression seeds then window", func(t *testing.T) { + t.Run("default is regression seeds then the window", func(t *testing.T) { t.Setenv("FUZZ_SEEDS", "3") t.Setenv("FUZZ_SEED_OFFSET", "100") want := append(append([]int64{}, regressionSeeds...), 100, 101, 102) - assert.Equal(t, want, paritySeeds(t)) - }) - - t.Run("window overlapping a regression seed is deduplicated", func(t *testing.T) { - t.Setenv("FUZZ_SEEDS", "5") - t.Setenv("FUZZ_SEED_OFFSET", "27") - seeds := paritySeeds(t) - count := 0 - for _, s := range seeds { - if s == 29 { - count++ - } - } - assert.Equal(t, 1, count, "seed 29 must appear once even though it is both a regression seed and inside the window") + assert.Equal(t, want, invariantSeeds(t)) }) t.Run("FUZZ_SEED override ignores regression seeds", func(t *testing.T) { t.Setenv("FUZZ_SEED", "7, 8") - assert.Equal(t, []int64{7, 8}, paritySeeds(t)) + assert.Equal(t, []int64{7, 8}, invariantSeeds(t)) }) } -// FuzzJobCreateParity exposes the parity check to Go's native fuzzer. Each input -// runs two real deploys, so it's for ad-hoc deep runs, not the default test path. -func FuzzJobCreateParity(f *testing.F) { - requireTerraform(f) +// FuzzJobInvariants exposes the invariant check to Go's native fuzzer. Each input +// runs a real deploy, so it's for ad-hoc deep runs, not the default test path. +func FuzzJobInvariants(f *testing.F) { + requireFuzzOptIn(f) for seed := range int64(5) { f.Add(seed) } - // Seed the corpus with known past divergences. + // Seed the corpus with known past failures. for _, seed := range regressionSeeds { f.Add(seed) } f.Fuzz(func(t *testing.T, seed int64) { - checkJobParity(t, seed) + checkJob(t, seed) }) } -// checkJobParity deploys the seed's job under both engines and fails if the create -// payloads diverge. A deploy/capture failure is not a payload divergence, so the -// outcomes are kept distinct: -// - neither deployed: skip (the config is unacceptable to both engines). -// - one deployed: fail separately as a deploy/capture difference, not a diff. -// - both deployed: compare the captured payloads. -func checkJobParity(t *testing.T, seed int64) { +// checkJob validates and deploys the seed's job, then asserts its create payload +// satisfies the invariants. It separates the two fuzzing outcomes: +// - the config doesn't validate: skip, since invalid input can't be a bug. +// - a validated config that fails to deploy or breaks an invariant: fail (a +// config the CLI accepted must deploy and produce a sound payload). +func checkJob(t *testing.T, seed int64) { t.Helper() job := generateJob(newRNG(seed)) - ctx := t.Context() - direct, directErr := captureJobCreate(ctx, t, job, "direct") - terraform, tfErr := captureJobCreate(ctx, t, job, "terraform") - - switch { - case directErr != nil && tfErr != nil: - t.Skipf("seed %d: config did not deploy under either engine (not a parity divergence)\ndirect: %v\nterraform: %v", seed, directErr, tfErr) - case directErr != nil: - t.Fatalf("seed %d: terraform deployed but direct did not (deploy/capture difference, not a payload diff): %v", seed, directErr) - case tfErr != nil: - t.Fatalf("seed %d: direct deployed but terraform did not (deploy/capture difference, not a payload diff): %v", seed, tfErr) + payload, err := captureJobCreate(t.Context(), t, job) + if errors.Is(err, errInvalidConfig) { + t.Skipf("seed %d: config did not validate, so it can't violate an invariant: %v", seed, err) } + require.NoErrorf(t, err, "seed %d: validated config failed to deploy", seed) - diffs, err := diffPayloads(direct, terraform, defaultIgnorePaths) - require.NoErrorf(t, err, "seed %d: comparing create payloads", seed) + checkJobInvariants(t, seed, job, payload) - if len(diffs) > 0 { + if t.Failed() { jobJSON, _ := json.MarshalIndent(job, "", " ") - t.Errorf("seed %d: terraform/direct create payloads diverge (%d differences):", seed, len(diffs)) - for _, d := range diffs { - t.Errorf(" %s", d) - } t.Logf("reproduce with: FUZZ_SEED=%d task test-fuzz\nonce fixed, add %d to regressionSeeds in bundle/fuzz/fuzz_test.go\n%s", seed, seed, jobJSON) } } diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go index 7a96c1868cd..47abd7b7e91 100644 --- a/bundle/fuzz/generate_test.go +++ b/bundle/fuzz/generate_test.go @@ -28,10 +28,10 @@ var ( ) // generateJob builds a random, well-formed job config driven entirely by rng, so -// the same seed always produces the same job. It favors fields whose translation -// tends to differ between engines. +// the same seed always produces the same job. It favors fields whose +// config->payload translation is non-trivial (clusters, scheduling, references). // -// TODO(DECO-25361): generalize the harness across resource kinds. +// TODO: generalize the harness across resource kinds. func generateJob(rng *rand.Rand) *resources.Job { job := &resources.Job{} job.Name = randName(rng, "job") diff --git a/bundle/fuzz/invariants_cases_test.go b/bundle/fuzz/invariants_cases_test.go new file mode 100644 index 00000000000..ea14c45f508 --- /dev/null +++ b/bundle/fuzz/invariants_cases_test.go @@ -0,0 +1,93 @@ +package fuzz + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +// recordingT captures whether the invariant assertions failed, so the table below +// can check that a bad payload is rejected and a good one is accepted without a +// real deploy. +type recordingT struct{ failed bool } + +func (r *recordingT) Errorf(string, ...any) { r.failed = true } + +// FailNow is only reached if decodePayload errors; every case here is valid JSON, +// so record and stop the goroutine the way require would. +func (r *recordingT) FailNow() { panic("unexpected FailNow") } + +func TestCheckJobInvariants(t *testing.T) { + job := &resources.Job{ + JobSettings: jobs.JobSettings{ + Name: "j", + JobClusters: []jobs.JobCluster{ + {JobClusterKey: "shared", NewCluster: compute.ClusterSpec{}}, + }, + Tasks: []jobs.Task{ + {TaskKey: "a"}, + {TaskKey: "b"}, + }, + }, + } + + tests := []struct { + name string + payload string + wantFailed bool + }{ + { + name: "valid payload", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"num_workers":0}}],"tasks":[{"task_key":"a","job_cluster_key":"shared"},{"task_key":"b","depends_on":[{"task_key":"a"}]}]}`, + }, + { + name: "renamed job", + payload: `{"name":"other","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + wantFailed: true, + }, + { + name: "dropped task", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a"}]}`, + wantFailed: true, + }, + { + name: "dangling dependency", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a"},{"task_key":"b","depends_on":[{"task_key":"ghost"}]}]}`, + wantFailed: true, + }, + { + name: "dangling job cluster reference", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a","job_cluster_key":"missing"},{"task_key":"b"}]}`, + wantFailed: true, + }, + { + name: "new_cluster without explicit size is a valid single node", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"spark_version":"x"}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + }, + { + name: "single-node new_cluster with num_workers 0", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"num_workers":0}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + }, + { + name: "autoscale new_cluster", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"autoscale":{"min_workers":1,"max_workers":3}}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + }, + { + name: "new_cluster sets both autoscale and num_workers", + payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"autoscale":{"min_workers":1,"max_workers":3},"num_workers":2}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, + wantFailed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := &recordingT{} + checkJobInvariants(rec, 0, job, json.RawMessage(tt.payload)) + assert.Equal(t, tt.wantFailed, rec.failed) + }) + } +} diff --git a/bundle/fuzz/invariants_test.go b/bundle/fuzz/invariants_test.go new file mode 100644 index 00000000000..055390236a7 --- /dev/null +++ b/bundle/fuzz/invariants_test.go @@ -0,0 +1,175 @@ +package fuzz + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// checkJobInvariants asserts the properties that any valid job's create payload +// must satisfy, independent of deploy engine. Unlike a terraform/direct payload +// diff, an invariant has no legitimate reason to fail, so a failure is a real bug +// and the seed reproduces it. Each invariant is checked separately so a failure +// points at the property that broke. +func checkJobInvariants(t require.TestingT, seed int64, job *resources.Job, payload json.RawMessage) { + p, err := decodePayload(payload) + require.NoErrorf(t, err, "seed %d: decoding create payload", seed) + + nameMatchesConfig(t, seed, job, p) + taskKeysMatchConfig(t, seed, job, p) + dependenciesResolve(t, seed, p) + jobClusterKeysMatchConfig(t, seed, job, p) + taskClusterRefsResolve(t, seed, p) + newClustersSizedExclusively(t, seed, p) +} + +// nameMatchesConfig: the engine must not rename the job. +func nameMatchesConfig(t require.TestingT, seed int64, job *resources.Job, p map[string]any) { + assert.Equalf(t, job.Name, p["name"], "seed %d: payload name must match config", seed) +} + +// taskKeysMatchConfig: the payload must carry exactly the tasks from config, no +// more and no fewer, identified by task_key. +func taskKeysMatchConfig(t require.TestingT, seed int64, job *resources.Job, p map[string]any) { + want := make([]string, 0, len(job.Tasks)) + for _, task := range job.Tasks { + want = append(want, task.TaskKey) + } + assert.ElementsMatchf(t, want, taskKeys(p), "seed %d: payload task keys must match config", seed) +} + +// dependenciesResolve: every depends_on must point at a task in the same payload. +func dependenciesResolve(t require.TestingT, seed int64, p map[string]any) { + keys := sliceToSet(taskKeys(p)) + for _, task := range payloadTasks(p) { + for _, dep := range slice(task["depends_on"]) { + d, ok := dep.(map[string]any) + if !ok { + continue + } + assert.Containsf(t, keys, d["task_key"], + "seed %d: task %v depends on unknown task %v", seed, task["task_key"], d["task_key"]) + } + } +} + +// jobClusterKeysMatchConfig: the payload's shared job clusters must match config. +func jobClusterKeysMatchConfig(t require.TestingT, seed int64, job *resources.Job, p map[string]any) { + want := make([]string, 0, len(job.JobClusters)) + for _, jc := range job.JobClusters { + want = append(want, jc.JobClusterKey) + } + assert.ElementsMatchf(t, want, jobClusterKeys(p), "seed %d: payload job cluster keys must match config", seed) +} + +// taskClusterRefsResolve: a task referencing a shared cluster must reference one +// declared in job_clusters. +func taskClusterRefsResolve(t require.TestingT, seed int64, p map[string]any) { + keys := sliceToSet(jobClusterKeys(p)) + for _, task := range payloadTasks(p) { + ref, ok := task["job_cluster_key"].(string) + if !ok || ref == "" { + continue + } + assert.Containsf(t, keys, ref, + "seed %d: task %v references unknown job cluster %q", seed, task["task_key"], ref) + } +} + +// newClustersSizedExclusively: a new_cluster is sized either by autoscale or by a +// fixed num_workers, never both. The two are mutually exclusive cluster shapes, so +// an engine emitting both (e.g. force-sending num_workers onto an autoscale +// cluster) produces a payload the backend rejects. +func newClustersSizedExclusively(t require.TestingT, seed int64, p map[string]any) { + for _, c := range newClusters(p) { + _, hasAutoscale := c["autoscale"] + _, hasNumWorkers := c["num_workers"] + assert.Falsef(t, hasAutoscale && hasNumWorkers, + "seed %d: new_cluster must not set both autoscale and num_workers, got %v", seed, c) + } +} + +// decodePayload unmarshals the create body with UseNumber so large int64 values +// (job ids, spark_context_id) aren't corrupted by float64 rounding. +func decodePayload(raw json.RawMessage) (map[string]any, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var p map[string]any + if err := dec.Decode(&p); err != nil { + return nil, fmt.Errorf("decoding payload: %w", err) + } + return p, nil +} + +// payloadTasks returns the payload's task objects. +func payloadTasks(p map[string]any) []map[string]any { + tasks := make([]map[string]any, 0, len(slice(p["tasks"]))) + for _, el := range slice(p["tasks"]) { + if m, ok := el.(map[string]any); ok { + tasks = append(tasks, m) + } + } + return tasks +} + +func taskKeys(p map[string]any) []string { + var keys []string + for _, task := range payloadTasks(p) { + if k, ok := task["task_key"].(string); ok { + keys = append(keys, k) + } + } + return keys +} + +func jobClusterKeys(p map[string]any) []string { + var keys []string + for _, el := range slice(p["job_clusters"]) { + jc, ok := el.(map[string]any) + if !ok { + continue + } + if k, ok := jc["job_cluster_key"].(string); ok { + keys = append(keys, k) + } + } + return keys +} + +// newClusters returns every new_cluster spec in the payload: one per task that +// defines its own cluster plus one per shared job cluster. +func newClusters(p map[string]any) []map[string]any { + var specs []map[string]any + for _, task := range payloadTasks(p) { + if c, ok := task["new_cluster"].(map[string]any); ok { + specs = append(specs, c) + } + } + for _, el := range slice(p["job_clusters"]) { + jc, ok := el.(map[string]any) + if !ok { + continue + } + if c, ok := jc["new_cluster"].(map[string]any); ok { + specs = append(specs, c) + } + } + return specs +} + +func slice(v any) []any { + s, _ := v.([]any) + return s +} + +func sliceToSet(s []string) map[string]bool { + set := make(map[string]bool, len(s)) + for _, v := range s { + set[v] = true + } + return set +} diff --git a/bundle/fuzz/recorder_test.go b/bundle/fuzz/recorder_test.go index 73620d00e19..cfabf227219 100644 --- a/bundle/fuzz/recorder_test.go +++ b/bundle/fuzz/recorder_test.go @@ -7,9 +7,9 @@ import ( "github.com/databricks/cli/libs/testserver" ) -// jobsCreatePath is the Jobs API route both engines must hit on create. The -// testserver registers only this version, so an engine posting to a different one -// surfaces as a capture failure ("did not POST"), not a payload diff. +// jobsCreatePath is the Jobs API route the deploy must hit on create. The +// testserver registers only this version, so posting to a different one surfaces +// as a capture failure ("did not POST"). const jobsCreatePath = "/api/2.2/jobs/create" // capturedRequest is a single mutating API request observed by the testserver. @@ -20,8 +20,7 @@ type capturedRequest struct { } // recorder collects request bodies sent to a testserver. It is safe for -// concurrent use because the SDK and terraform may issue requests from multiple -// goroutines. +// concurrent use because the deploy may issue requests from multiple goroutines. type recorder struct { mu sync.Mutex requests []capturedRequest From 63258264e26404a9f743eabf65083428cbc456e3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 29 Jun 2026 07:58:49 +0000 Subject: [PATCH 15/53] acceptance: replace bundle/fuzz parity with schema-driven invariant fuzzing Drop the terraform/direct create-payload parity package in favor of fuzzing the existing acceptance/bundle/invariant framework, which already checks invariants across all resource types and is prepped for fuzzing via its INPUT_CONFIG_OK contract. - add acceptance/bin/gen_fuzz_config.py: a seeded generator that walks the bundle schema and emits a random databricks.yml for any resource type - add acceptance/bundle/invariant/fuzz: generates configs over a seed window and asserts the CLI never panics; the no-drift invariant is opt-in (FUZZ_CHECK_DRIFT) for the nightly wide-window run - point task test-fuzz and the nightly job at the new variant - remove bundle/fuzz and its parity harness --- .github/workflows/push.yml | 26 +- Taskfile.yml | 18 +- acceptance/bin/gen_fuzz_config.py | 207 +++++++++++ .../bundle/invariant/fuzz/out.test.toml | 5 + acceptance/bundle/invariant/fuzz/output.txt | 0 acceptance/bundle/invariant/fuzz/script | 62 ++++ acceptance/bundle/invariant/fuzz/test.toml | 5 + bundle/fuzz/deploy_smoke_test.go | 21 -- bundle/fuzz/deploy_test.go | 112 ------ bundle/fuzz/doc.go | 11 - bundle/fuzz/fuzz_test.go | 154 -------- bundle/fuzz/generate_invariants_test.go | 47 --- bundle/fuzz/generate_test.go | 340 ------------------ bundle/fuzz/invariants_cases_test.go | 93 ----- bundle/fuzz/invariants_test.go | 175 --------- bundle/fuzz/rand_test.go | 47 --- bundle/fuzz/recorder_test.go | 57 --- 17 files changed, 300 insertions(+), 1080 deletions(-) create mode 100755 acceptance/bin/gen_fuzz_config.py create mode 100644 acceptance/bundle/invariant/fuzz/out.test.toml create mode 100644 acceptance/bundle/invariant/fuzz/output.txt create mode 100644 acceptance/bundle/invariant/fuzz/script create mode 100644 acceptance/bundle/invariant/fuzz/test.toml delete mode 100644 bundle/fuzz/deploy_smoke_test.go delete mode 100644 bundle/fuzz/deploy_test.go delete mode 100644 bundle/fuzz/doc.go delete mode 100644 bundle/fuzz/fuzz_test.go delete mode 100644 bundle/fuzz/generate_invariants_test.go delete mode 100644 bundle/fuzz/generate_test.go delete mode 100644 bundle/fuzz/invariants_cases_test.go delete mode 100644 bundle/fuzz/invariants_test.go delete mode 100644 bundle/fuzz/rand_test.go delete mode 100644 bundle/fuzz/recorder_test.go diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 75af3ffbfaa..3b60837cb98 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -397,9 +397,10 @@ jobs: needs: - cleanups - # A real deploy per seed across a wide rotating window: too slow for every PR, - # so nightly only and not part of test-result. (The package's un-gated smoke - # test still checks the invariants on one seed on every PR.) + # A real deploy per seed across a wide rotating window, with the no-drift + # invariant on: too slow for every PR, so nightly only and not part of + # test-result. (The committed acceptance fuzz test still checks the no-panic + # invariant on a small fixed seed window on every PR.) if: ${{ github.event_name == 'schedule' }} name: "task test-fuzz" runs-on: @@ -428,11 +429,11 @@ jobs: - name: Run tests env: # Shift the seed window each nightly run so CI explores new configs. - # offset = GITHUB_RUN_NUMBER * FUZZ_SEEDS keeps windows non-overlapping - # (GITHUB_RUN_NUMBER is monotonic). A failure prints FUZZ_SEED=. - FUZZ_SEEDS: "25" + # start = GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT keeps windows non-overlapping + # (GITHUB_RUN_NUMBER is monotonic). A failure prints the failing seed. + FUZZ_SEED_COUNT: "25" run: | - export FUZZ_SEED_OFFSET=$(( GITHUB_RUN_NUMBER * FUZZ_SEEDS )) + export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) go tool -modfile=tools/task/go.mod task test-fuzz # Excluded from test-result, so surface failures as a GitHub issue. Reuse one @@ -444,23 +445,20 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | gh label create fuzz-nightly \ - --description "Nightly create-payload invariant failures" \ + --description "Nightly schema fuzz invariant failures" \ --color FBCA04 2>/dev/null || true body=$(cat <\`. + The failing seed is printed in the job log as \`reproduce with: ...\`. Reproduce locally with: \`\`\` - FUZZ_SEED= task test-fuzz + FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz \`\`\` - - Once fixed, add the seed to \`regressionSeeds\` in \`bundle/fuzz/fuzz_test.go\` - in the same PR so the bug can never silently regress. EOF ) diff --git a/Taskfile.yml b/Taskfile.yml index 0ce5ae35031..152af84e4e4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -731,20 +731,20 @@ tasks: -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox" test-fuzz: - desc: Run create-payload invariant fuzz tests (random jobs, direct engine) - # No `sources:` fingerprint: the seeds depend on FUZZ_* env vars Task can't see, - # so always run rather than no-op a repro or a shifted nightly window. - env: - # Opt this target into the invariant suite (see requireFuzzOptIn) without - # overriding the FUZZ_SEED(S)/OFFSET tuning knobs. - FUZZ_INVARIANTS: "1" + desc: Run schema fuzz invariant tests (random configs, direct engine) + # No `sources:` fingerprint: the seed window depends on FUZZ_* env vars Task + # can't see, so always run rather than no-op a repro or a shifted nightly window. cmds: - | + # Sweep a wider window than the committed acceptance run and turn on the + # no-drift invariant; a repro can narrow it with FUZZ_SEED_START/COUNT. + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" + export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" {{.GO_TOOL}} gotestsum \ --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ --no-summary=skipped \ - --packages ./bundle/fuzz/... \ - -- -timeout=${LOCAL_TIMEOUT:-30m} + --packages ./acceptance/... \ + -- -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz" # --- Integration tests --- diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py new file mode 100755 index 00000000000..85909bb03fb --- /dev/null +++ b/acceptance/bin/gen_fuzz_config.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +Generate a random bundle config from the bundle JSON schema. + +The generator walks the schema (`databricks bundle schema`), resolving $ref and +picking concrete branches of oneOf/anyOf, and emits a single random resource as a +databricks.yml. It is seeded so a failing run can be reproduced with the same --seed. + +This feeds the invariant tests (see acceptance/bundle/invariant/): the harness +deploys the generated config and asserts invariants such as no-drift. Configs the +CLI rejects are filtered out by the harness before invariants are checked, so the +generator is free to produce structurally-random-but-sometimes-invalid configs. +""" + +import argparse +import json +import random +import sys + +# Maximum object/array nesting depth. The schema is recursive (e.g. job tasks -> +# for_each_task -> task), so without a cap the walk would not terminate. +MAX_DEPTH = 6 + +# A string branch whose pattern matches a ${...} reference. These exist because the +# schema generator wraps every concrete field in a oneOf with interpolation-string +# alternatives (see bundle/internal/schema/main.go addInterpolationPatterns). We +# generate concrete values, not references, so these branches are skipped. +INTERPOLATION_MARKER = "\\$\\{" + + +class Generator: + def __init__(self, schema, rng, unique): + self.root = schema + self.rng = rng + self.unique = unique + + def resolve(self, schema): + # Follow $ref chains. A ref looks like "#/$defs/github.com/.../resources.Job"; + # definitions are nested under $defs by the "/"-separated path segments. + while isinstance(schema, dict) and "$ref" in schema: + cur = self.root["$defs"] + for part in schema["$ref"].split("/")[2:]: + cur = cur[part] + schema = cur + return schema + + def is_interpolation(self, branch): + return branch.get("type") == "string" and INTERPOLATION_MARKER in branch.get("pattern", "") + + def choose_branch(self, branches): + # Prefer concrete branches over the ${...} interpolation-string alternatives. + concrete = [b for b in branches if not self.is_interpolation(b)] + return self.rng.choice(concrete or branches) + + def gen(self, schema, depth, name=""): + schema = self.resolve(schema) + if not isinstance(schema, dict) or not schema: + return self.gen_scalar({"type": "string"}, name) + + if "const" in schema: + return schema["const"] + if schema.get("enum"): + return self.rng.choice(schema["enum"]) + + for key in ("oneOf", "anyOf"): + if schema.get(key): + return self.gen(self.choose_branch(schema[key]), depth, name) + + t = schema.get("type") + if t == "object" or "properties" in schema or self.is_map(schema): + return self.gen_object(schema, depth) + if t == "array": + return self.gen_array(schema, depth, name) + return self.gen_scalar(schema, name) + + def is_map(self, schema): + return isinstance(schema.get("additionalProperties"), dict) and not schema.get("properties") + + def gen_object(self, schema, depth): + props = schema.get("properties", {}) + required = set(schema.get("required", [])) + result = {} + + for prop_name, prop_schema in props.items(): + # Always emit required fields; emit optional ones with decreasing + # probability as we go deeper to keep configs from exploding. + keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) + if not keep: + continue + value = self.gen(prop_schema, depth + 1, prop_name) + if value is not None: + result[prop_name] = value + + # Map type (additionalProperties schema, no fixed properties): synthesize a + # few random keys, e.g. resources. or string maps like tags. + if self.is_map(schema): + for _ in range(self.rng.randint(1, 2)): + key = self.token() + result[key] = self.gen(schema["additionalProperties"], depth + 1, key) + + return result + + def gen_array(self, schema, depth, name): + items = schema.get("items") + if not items or depth >= MAX_DEPTH: + return [] + return [self.gen(items, depth + 1, name) for _ in range(self.rng.randint(1, 3))] + + def gen_scalar(self, schema, name): + t = schema.get("type") + if t == "boolean": + return self.rng.choice([True, False]) + if t == "integer": + return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) + if t == "number": + return round(self.rng.uniform(0, 1000), 2) + # string (default) + if name in ("name", "display_name"): + return f"fuzz-{name}-{self.unique}" + return self.token() + + def token(self): + return "fuzz_" + "".join(self.rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def resource_types(schema, gen): + # resources is `oneOf[{object with one property per resource type}]`. + resources = gen.resolve(schema["properties"]["resources"]) + obj = next(b for b in resources["oneOf"] if b.get("type") == "object") + return obj["properties"] + + +def gen_config(schema, seed, unique, allowed): + rng = random.Random(seed) + gen = Generator(schema, rng, unique) + + types = resource_types(schema, gen) + candidates = [t for t in types if not allowed or t in allowed] + if not candidates: + sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") + rtype = rng.choice(sorted(candidates)) + + # Each resource type is a map ref; its element schema lives under the object + # branch's additionalProperties. + map_schema = gen.resolve(types[rtype]) + obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") + element = obj["additionalProperties"] + + key = f"fuzz_{rtype}_{seed}" + instance = gen.gen(element, 0, "name") + return { + "bundle": {"name": f"fuzz-{unique}"}, + "resources": {rtype: {key: instance}}, + } + + +def to_yaml(obj, indent=0, list_item=False): + pad = " " * indent + if isinstance(obj, dict): + if not obj: + return f"{pad}{{}}\n" if not list_item else f"{pad}- {{}}\n" + out = "" + first = True + for k, v in obj.items(): + prefix = pad + "- " if list_item and first else (pad + " " if list_item else pad) + child_indent = indent + 2 if list_item else indent + 1 + if isinstance(v, (dict, list)) and v: + out += f"{prefix}{k}:\n" + to_yaml(v, child_indent) + else: + out += f"{prefix}{k}: {json.dumps(v)}\n" + first = False + return out + if isinstance(obj, list): + if not obj: + return f"{pad}[]\n" + out = "" + for item in obj: + if isinstance(item, (dict, list)): + out += to_yaml(item, indent, list_item=True) + else: + out += f"{pad}- {json.dumps(item)}\n" + return out + return f"{pad}{json.dumps(obj)}\n" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--schema", required=True, help="Path to bundle JSON schema") + parser.add_argument("--seed", type=int, required=True, help="RNG seed (for reproducibility)") + parser.add_argument("--unique", default="local", help="Unique suffix for resource names") + parser.add_argument( + "--resources", + default="", + help="Comma-separated allow-list of resource types (default: all)", + ) + args = parser.parse_args() + + with open(args.schema) as f: + schema = json.load(f) + + allowed = {r.strip() for r in args.resources.split(",") if r.strip()} + config = gen_config(schema, args.seed, args.unique, allowed) + sys.stdout.write(to_yaml(config)) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml new file mode 100644 index 00000000000..789aa10c799 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/output.txt b/acceptance/bundle/invariant/fuzz/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script new file mode 100644 index 00000000000..f7751dde581 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/script @@ -0,0 +1,62 @@ +# Invariant to test: the CLI never panics or hits an internal error on any config +# generated from the bundle schema, and a config that deploys cleanly has no drift. +# +# gen_fuzz_config.py walks the schema emitted by the CLI under test and produces a +# random-but-schema-valid config. Most invariant work is shared with the no_drift +# test; the difference is the input is generated, not a curated template. +# +# Seeds form a window [START, START+COUNT). The window is env-driven so the nightly +# job can sweep a wide, non-overlapping range (see Taskfile.yml test-fuzz) while this +# committed test stays small and deterministic. Everything is routed to LOG.* / *.json +# so output.txt stays empty regardless of the window: a violation fails via exit code, +# not via output diff, which is what lets the same test run under any seed window. +# +# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a freshly deployed random config can +# legitimately differ from the fake server's state, so the local/PR run asserts only +# the cheap no-panic invariant. The nightly job enables drift on a real workspace. + +START="${FUZZ_SEED_START:-0}" +COUNT="${FUZZ_SEED_COUNT:-5}" + +# Emit the schema from the CLI under test so the generator always matches it. +$CLI bundle schema > schema.json 2>LOG.schema.err +cat LOG.schema.err | contains.py '!panic' '!internal error' > /dev/null + +for ((offset = 0; offset < COUNT; offset++)); do + seed=$((START + offset)) + dir="seed-$seed" + mkdir -p "$dir" + + gen_fuzz_config.py --schema schema.json --seed "$seed" --unique "$UNIQUE_NAME-$seed" --resources "${FUZZ_RESOURCES:-}" > "$dir/databricks.yml" 2>"$dir/LOG.gen.err" + cat "$dir/LOG.gen.err" | contains.py '!Traceback' > /dev/null + + ( + cd "$dir" + + # The CLI is allowed to reject a generated config, but never to crash. + set +e + $CLI bundle validate &> LOG.validate + $CLI bundle deploy &> LOG.deploy + deploy_rc=$? + set -e + cat LOG.validate LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + + # Deploy failed => config was rejected (not a bug). This is the negative of + # the no_drift test's INPUT_CONFIG_OK marker: nothing more to assert. + if [ "$deploy_rc" -ne 0 ]; then + exit 0 + fi + + if [ -n "${FUZZ_CHECK_DRIFT:-}" ]; then + $CLI bundle plan -o json > plan.json 2>LOG.plan.err + cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null + verify_no_drift.py plan.json + fi + + $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + ) || { + echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 task test-fuzz" >&2 + exit 1 + } +done diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml new file mode 100644 index 00000000000..019d2dc6494 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -0,0 +1,5 @@ +# Schema fuzzing: generate random configs from the bundle schema and assert +# invariants (see script). Unlike the curated-corpus invariant tests (no_drift, +# migrate), the fuzzer generates its own configs, so drop the inherited +# INPUT_CONFIG matrix. +EnvMatrix.INPUT_CONFIG = [] diff --git a/bundle/fuzz/deploy_smoke_test.go b/bundle/fuzz/deploy_smoke_test.go deleted file mode 100644 index 0121c7468ec..00000000000 --- a/bundle/fuzz/deploy_smoke_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package fuzz - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -// TestCaptureJobCreateDirect is intentionally NOT opt-in gated: a single direct -// deploy is cheap, so it runs on every `task test` as a smoke test of the capture -// harness and the invariants. The wider seed sweep stays opt-in via -// requireFuzzOptIn. -func TestCaptureJobCreateDirect(t *testing.T) { - job := generateJob(newRNG(1)) - - body, err := captureJobCreate(t.Context(), t, job) - require.NoError(t, err) - require.NotEmpty(t, body) - - checkJobInvariants(t, 1, job, body) -} diff --git a/bundle/fuzz/deploy_test.go b/bundle/fuzz/deploy_test.go deleted file mode 100644 index ddf8d4342b5..00000000000 --- a/bundle/fuzz/deploy_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package fuzz - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/internal/testcli" - "github.com/databricks/cli/libs/testserver" -) - -const ( - // bundleResourceKey is the map key the generated job is registered under. - bundleResourceKey = "fuzz_job" - fakeToken = "testtoken" -) - -// errInvalidConfig marks a generated config that `bundle validate` rejects. The -// caller skips on it: an invalid config can't violate an invariant, so it is not a -// bug. This is the distinction that makes the suite safe to point at a looser -// (e.g. schema-driven) generator, which will produce invalid configs by design. -var errInvalidConfig = errors.New("config did not validate") - -// captureJobCreate validates then deploys a bundle containing job via the direct -// engine against an in-process testserver, returning the create request body sent -// to the Jobs API. A validation failure is wrapped as errInvalidConfig. The -// invariant suite asserts properties of the payload; the terraform engine is not -// involved (we assert fundamental properties rather than compare engines). -func captureJobCreate(ctx context.Context, t *testing.T, job *resources.Job) (json.RawMessage, error) { - rec := &recorder{} - server := testserver.New(t) - server.RequestCallback = rec.callback - testserver.AddDefaultHandlers(server) - - dir := t.TempDir() - if err := writeJobBundle(dir, server.URL, job); err != nil { - return nil, err - } - - t.Setenv("DATABRICKS_HOST", server.URL) - t.Setenv("DATABRICKS_TOKEN", fakeToken) - t.Setenv("DATABRICKS_BUNDLE_ENGINE", "direct") - t.Chdir(dir) - - // Validate first so an invalid config is reported as errInvalidConfig (caller - // skips) rather than a deploy failure (caller fails). - if _, stderr, err := testcli.NewRunner(t, ctx, "bundle", "validate").Run(); err != nil { - return nil, fmt.Errorf("%w: %v\nstderr:\n%s", errInvalidConfig, err, stderr.String()) - } - - stdout, stderr, err := testcli.NewRunner(t, ctx, "bundle", "deploy").Run() - if err != nil { - return nil, fmt.Errorf("bundle deploy failed: %w\nstdout:\n%s\nstderr:\n%s", - err, stdout.String(), stderr.String()) - } - - body, ok := rec.find("POST", jobsCreatePath) - if !ok { - return nil, fmt.Errorf("deploy did not POST %s", jobsCreatePath) - } - return body, nil -} - -// writeJobBundle writes a minimal databricks.yml for a single job. It emits JSON -// (valid YAML) to reuse the job's own marshaling, which honors ForceSendFields. -func writeJobBundle(dir, host string, job *resources.Job) error { - jobJSON, err := json.Marshal(job) - if err != nil { - return fmt.Errorf("marshaling job: %w", err) - } - - var jobMap map[string]any - if err := json.Unmarshal(jobJSON, &jobMap); err != nil { - return fmt.Errorf("unmarshaling job: %w", err) - } - - doc := map[string]any{ - "bundle": map[string]any{"name": "fuzz"}, - "workspace": map[string]any{"host": host}, - "resources": map[string]any{ - "jobs": map[string]any{bundleResourceKey: jobMap}, - }, - } - - data, err := json.MarshalIndent(doc, "", " ") - if err != nil { - return fmt.Errorf("marshaling bundle: %w", err) - } - - return os.WriteFile(filepath.Join(dir, "databricks.yml"), data, 0o600) -} - -// fuzzOptInVars opt a run into the invariant suite. FUZZ_SEED(S)/OFFSET also tune -// it (see invariantSeeds); FUZZ_INVARIANTS is a no-tuning switch for `task test-fuzz`. -var fuzzOptInVars = []string{"FUZZ_INVARIANTS", "FUZZ_SEED", "FUZZ_SEEDS", "FUZZ_SEED_OFFSET"} - -// requireFuzzOptIn skips unless a FUZZ_* var is set. Each seed runs a real -// in-process deploy, so gating keeps a plain `task test` fast (the single -// un-gated direct smoke test still exercises the harness on every run). -func requireFuzzOptIn(t testing.TB) { - for _, name := range fuzzOptInVars { - if os.Getenv(name) != "" { - return - } - } - t.Skip("invariant fuzz suite is opt-in; run `task test-fuzz` or set FUZZ_SEED= to reproduce a single seed") -} diff --git a/bundle/fuzz/doc.go b/bundle/fuzz/doc.go deleted file mode 100644 index 59b04170963..00000000000 --- a/bundle/fuzz/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -// Package fuzz deploys randomly generated bundle resources through the direct -// engine and asserts invariants that any valid config's API create payload must -// satisfy (e.g. task keys are preserved, references resolve, a new_cluster is -// sized by autoscale or num_workers but not both). Unlike a terraform/direct -// payload comparison, an invariant has no legitimate reason to fail, so a failure -// is a real bug. Generators are seeded so any failure reproduces from the printed -// seed. Jobs only for now. -// -// Everything lives in _test.go files: the package is test-only and nothing in the -// product imports it. This file exists only to carry the package doc. -package fuzz diff --git a/bundle/fuzz/fuzz_test.go b/bundle/fuzz/fuzz_test.go deleted file mode 100644 index f8d758e88ae..00000000000 --- a/bundle/fuzz/fuzz_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package fuzz - -import ( - "encoding/json" - "errors" - "os" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// defaultInvariantSeeds is how many random jobs TestJobInvariants checks by -// default. Each seed runs a real deploy, so keep it modest; override with -// FUZZ_SEEDS. -const defaultInvariantSeeds = 20 - -// regressionSeeds are seeds that previously broke an invariant. They are always -// checked (on top of the rotating nightly window, which never revisits them) so a -// fixed bug can't silently regress. When the nightly job reports a new failing -// FUZZ_SEED, add it here in the PR that fixes it. Empty until the first such bug. -var regressionSeeds = []int64{} - -// TestJobInvariants asserts the engine produces a create payload satisfying the -// invariants in checkJobInvariants for many random jobs, printing the seed on -// failure. -func TestJobInvariants(t *testing.T) { - requireFuzzOptIn(t) - - for _, seed := range invariantSeeds(t) { - t.Run("seed="+strconv.FormatInt(seed, 10), func(t *testing.T) { - checkJob(t, seed) - }) - } -} - -// invariantSeeds returns the seeds TestJobInvariants should check. -// -// FUZZ_SEED (comma-separated) runs exactly those seeds and overrides everything, -// so a reported failure reproduces with one command. Otherwise it runs -// regressionSeeds plus FUZZ_SEEDS seeds (default defaultInvariantSeeds) from -// FUZZ_SEED_OFFSET; the nightly job shifts the offset each run so CI keeps -// exploring new configs. -func invariantSeeds(t *testing.T) []int64 { - if v := os.Getenv("FUZZ_SEED"); v != "" { - var seeds []int64 - for part := range strings.SplitSeq(v, ",") { - part = strings.TrimSpace(part) - if part == "" { - continue - } - n, err := strconv.ParseInt(part, 10, 64) - require.NoErrorf(t, err, "invalid FUZZ_SEED entry %q", part) - seeds = append(seeds, n) - } - require.NotEmptyf(t, seeds, "FUZZ_SEED=%q contained no seeds", v) - return seeds - } - - count := defaultInvariantSeeds - if v := os.Getenv("FUZZ_SEEDS"); v != "" { - n, err := strconv.Atoi(v) - require.NoErrorf(t, err, "invalid FUZZ_SEEDS=%q", v) - require.Positivef(t, n, "FUZZ_SEEDS must be positive, got %d", n) - count = n - } - - var offset int64 - if v := os.Getenv("FUZZ_SEED_OFFSET"); v != "" { - n, err := strconv.ParseInt(v, 10, 64) - require.NoErrorf(t, err, "invalid FUZZ_SEED_OFFSET=%q", v) - offset = n - } - - seeds := make([]int64, 0, len(regressionSeeds)+count) - seen := make(map[int64]bool, len(regressionSeeds)+count) - for _, s := range regressionSeeds { - if !seen[s] { - seen[s] = true - seeds = append(seeds, s) - } - } - for i := range int64(count) { - s := offset + i - if !seen[s] { - seen[s] = true - seeds = append(seeds, s) - } - } - return seeds -} - -// TestInvariantSeeds verifies invariantSeeds composes the regression seeds with -// the rotating window, deduplicates overlaps, and lets FUZZ_SEED override both. -func TestInvariantSeeds(t *testing.T) { - // Isolate from ambient FUZZ_* in the dev environment (invariantSeeds treats "" - // as unset); subtests set only what they need. - t.Setenv("FUZZ_SEED", "") - t.Setenv("FUZZ_SEEDS", "") - t.Setenv("FUZZ_SEED_OFFSET", "") - - t.Run("default is regression seeds then the window", func(t *testing.T) { - t.Setenv("FUZZ_SEEDS", "3") - t.Setenv("FUZZ_SEED_OFFSET", "100") - want := append(append([]int64{}, regressionSeeds...), 100, 101, 102) - assert.Equal(t, want, invariantSeeds(t)) - }) - - t.Run("FUZZ_SEED override ignores regression seeds", func(t *testing.T) { - t.Setenv("FUZZ_SEED", "7, 8") - assert.Equal(t, []int64{7, 8}, invariantSeeds(t)) - }) -} - -// FuzzJobInvariants exposes the invariant check to Go's native fuzzer. Each input -// runs a real deploy, so it's for ad-hoc deep runs, not the default test path. -func FuzzJobInvariants(f *testing.F) { - requireFuzzOptIn(f) - for seed := range int64(5) { - f.Add(seed) - } - // Seed the corpus with known past failures. - for _, seed := range regressionSeeds { - f.Add(seed) - } - f.Fuzz(func(t *testing.T, seed int64) { - checkJob(t, seed) - }) -} - -// checkJob validates and deploys the seed's job, then asserts its create payload -// satisfies the invariants. It separates the two fuzzing outcomes: -// - the config doesn't validate: skip, since invalid input can't be a bug. -// - a validated config that fails to deploy or breaks an invariant: fail (a -// config the CLI accepted must deploy and produce a sound payload). -func checkJob(t *testing.T, seed int64) { - t.Helper() - job := generateJob(newRNG(seed)) - - payload, err := captureJobCreate(t.Context(), t, job) - if errors.Is(err, errInvalidConfig) { - t.Skipf("seed %d: config did not validate, so it can't violate an invariant: %v", seed, err) - } - require.NoErrorf(t, err, "seed %d: validated config failed to deploy", seed) - - checkJobInvariants(t, seed, job, payload) - - if t.Failed() { - jobJSON, _ := json.MarshalIndent(job, "", " ") - t.Logf("reproduce with: FUZZ_SEED=%d task test-fuzz\nonce fixed, add %d to regressionSeeds in bundle/fuzz/fuzz_test.go\n%s", seed, seed, jobJSON) - } -} diff --git a/bundle/fuzz/generate_invariants_test.go b/bundle/fuzz/generate_invariants_test.go deleted file mode 100644 index 9ca3b5cc932..00000000000 --- a/bundle/fuzz/generate_invariants_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package fuzz - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGenerateJobIsDeterministic(t *testing.T) { - a := generateJob(newRNG(42)) - b := generateJob(newRNG(42)) - assert.Equal(t, a, b, "same seed must produce identical job") -} - -func TestGenerateJobIsWellFormed(t *testing.T) { - for seed := range int64(200) { - job := generateJob(newRNG(seed)) - require.NotEmptyf(t, job.Name, "seed %d: job must have a name", seed) - require.NotEmptyf(t, job.Tasks, "seed %d: job must have at least one task", seed) - - clusterKeys := map[string]bool{} - for _, jc := range job.JobClusters { - clusterKeys[jc.JobClusterKey] = true - } - - taskKeys := map[string]bool{} - for _, task := range job.Tasks { - require.NotEmptyf(t, task.TaskKey, "seed %d: task must have a key", seed) - taskKeys[task.TaskKey] = true - - // A task referencing a job cluster must reference one we generated. - if task.JobClusterKey != "" { - assert.Containsf(t, clusterKeys, task.JobClusterKey, - "seed %d: task %q references unknown job cluster %q", seed, task.TaskKey, task.JobClusterKey) - } - } - - // Every dependency must point at a task that exists in this job. - for _, task := range job.Tasks { - for _, dep := range task.DependsOn { - assert.Containsf(t, taskKeys, dep.TaskKey, - "seed %d: task %q depends on unknown task %q", seed, task.TaskKey, dep.TaskKey) - } - } - } -} diff --git a/bundle/fuzz/generate_test.go b/bundle/fuzz/generate_test.go deleted file mode 100644 index 47abd7b7e91..00000000000 --- a/bundle/fuzz/generate_test.go +++ /dev/null @@ -1,340 +0,0 @@ -package fuzz - -import ( - "fmt" - "math/rand/v2" - "strconv" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" -) - -// Value pools are intentionally small and valid-looking: the goal is to exercise -// config->payload translation across many field combinations, not to stress the -// API with invalid values the testserver would reject. -var ( - sparkVersions = []string{"13.3.x-scala2.12", "14.3.x-scala2.12", "15.4.x-scala2.12", "16.4.x-scala2.12"} - nodeTypeIDs = []string{"i3.xlarge", "m5.large", "r5.xlarge", "Standard_DS3_v2"} - timezones = []string{"UTC", "America/Los_Angeles", "Europe/Amsterdam"} - cronExprs = []string{"0 0 12 * * ?", "0 15 10 ? * MON-FRI", "0 0/30 * * * ?"} - pauseStatuses = []jobs.PauseStatus{jobs.PauseStatusPaused, jobs.PauseStatusUnpaused} - performance = []jobs.PerformanceTarget{jobs.PerformanceTargetPerformanceOptimized, jobs.PerformanceTargetStandard} - timeUnits = []string{"HOURS", "DAYS", "WEEKS"} - healthMetrics = []string{"RUN_DURATION_SECONDS", "STREAMING_BACKLOG_BYTES", "STREAMING_BACKLOG_RECORDS"} - conditionOps = []string{"EQUAL_TO", "NOT_EQUAL", "GREATER_THAN", "LESS_THAN_OR_EQUAL"} - runIfs = []string{"ALL_SUCCESS", "AT_LEAST_ONE_SUCCESS", "NONE_FAILED", "ALL_DONE"} - gitProviders = []jobs.GitProvider{jobs.GitProviderGitHub, jobs.GitProviderGitLab, jobs.GitProviderAzureDevOpsServices} -) - -// generateJob builds a random, well-formed job config driven entirely by rng, so -// the same seed always produces the same job. It favors fields whose -// config->payload translation is non-trivial (clusters, scheduling, references). -// -// TODO: generalize the harness across resource kinds. -func generateJob(rng *rand.Rand) *resources.Job { - job := &resources.Job{} - job.Name = randName(rng, "job") - - if chance(rng, 0.5) { - job.Description = randSentence(rng) - } - if chance(rng, 0.4) { - job.MaxConcurrentRuns = rng.IntN(10) + 1 - } - if chance(rng, 0.4) { - job.TimeoutSeconds = rng.IntN(7200) - } - if chance(rng, 0.3) { - job.PerformanceTarget = oneOf(rng, performance) - } - if chance(rng, 0.5) { - job.Tags = randTags(rng) - } - if chance(rng, 0.3) { - job.GitSource = randGitSource(rng) - } - - randScheduling(rng, job) - - if chance(rng, 0.3) { - job.EmailNotifications = randEmailNotifications(rng) - } - if chance(rng, 0.2) { - job.WebhookNotifications = randWebhookNotifications(rng) - } - if chance(rng, 0.3) { - job.NotificationSettings = &jobs.JobNotificationSettings{ - NoAlertForCanceledRuns: chance(rng, 0.5), - NoAlertForSkippedRuns: chance(rng, 0.5), - } - } - if chance(rng, 0.3) { - job.Health = randHealth(rng) - } - if chance(rng, 0.3) { - job.Parameters = randParameters(rng) - } - if chance(rng, 0.3) { - job.Queue = &jobs.QueueSettings{Enabled: chance(rng, 0.5)} - } - - // Generate shared job clusters first so tasks can reference them by key. - var jobClusterKeys []string - if chance(rng, 0.5) { - n := rng.IntN(2) + 1 - for i := range n { - key := fmt.Sprintf("cluster_%d", i) - jobClusterKeys = append(jobClusterKeys, key) - job.JobClusters = append(job.JobClusters, jobs.JobCluster{ - JobClusterKey: key, - NewCluster: randClusterSpec(rng), - }) - } - } - - nTasks := rng.IntN(3) + 1 - var taskKeys []string - for i := range nTasks { - task := randTask(rng, i, jobClusterKeys) - // Randomly chain dependencies onto previously generated tasks. - if len(taskKeys) > 0 && chance(rng, 0.4) { - dep := taskKeys[rng.IntN(len(taskKeys))] - task.DependsOn = []jobs.TaskDependency{{TaskKey: dep}} - if chance(rng, 0.5) { - task.RunIf = jobs.RunIf(oneOf(rng, runIfs)) - } - } - taskKeys = append(taskKeys, task.TaskKey) - job.Tasks = append(job.Tasks, task) - } - - return job -} - -// randScheduling sets at most one of schedule/trigger/continuous, which are -// mutually exclusive ways to launch a job. -func randScheduling(rng *rand.Rand, job *resources.Job) { - switch rng.IntN(5) { - case 0: - job.Schedule = &jobs.CronSchedule{ - QuartzCronExpression: oneOf(rng, cronExprs), - TimezoneId: oneOf(rng, timezones), - PauseStatus: oneOf(rng, pauseStatuses), - } - case 1: - job.Trigger = &jobs.TriggerSettings{ - PauseStatus: oneOf(rng, pauseStatuses), - Periodic: &jobs.PeriodicTriggerConfiguration{ - Interval: rng.IntN(12) + 1, - Unit: jobs.PeriodicTriggerConfigurationTimeUnit(oneOf(rng, timeUnits)), - }, - } - case 2: - job.Trigger = &jobs.TriggerSettings{ - PauseStatus: oneOf(rng, pauseStatuses), - FileArrival: &jobs.FileArrivalTriggerConfiguration{ - Url: "s3://" + randWord(rng) + "/" + randWord(rng), - }, - } - case 3: - job.Continuous = &jobs.Continuous{PauseStatus: oneOf(rng, pauseStatuses)} - default: - // no scheduling - } -} - -func randTask(rng *rand.Rand, idx int, jobClusterKeys []string) jobs.Task { - task := jobs.Task{TaskKey: fmt.Sprintf("task_%d", idx)} - - // Use absolute workspace paths so deploy never depends on local files. - // condition_task needs no compute, handled separately below. - needsCompute := true - switch rng.IntN(4) { - case 0: - task.NotebookTask = &jobs.NotebookTask{ - NotebookPath: "/Workspace/Users/test/" + randName(rng, "nb"), - Source: jobs.SourceWorkspace, - } - case 1: - task.SparkPythonTask = &jobs.SparkPythonTask{ - PythonFile: "/Workspace/Users/test/" + randName(rng, "main") + ".py", - Source: jobs.SourceWorkspace, - } - case 2: - task.PythonWheelTask = &jobs.PythonWheelTask{ - PackageName: randName(rng, "pkg"), - EntryPoint: "main", - } - case 3: - task.ConditionTask = &jobs.ConditionTask{ - Left: randWord(rng), - Op: jobs.ConditionTaskOp(oneOf(rng, conditionOps)), - Right: randWord(rng), - } - needsCompute = false - } - - if needsCompute { - assignCompute(rng, &task, jobClusterKeys) - if chance(rng, 0.4) { - task.Libraries = randLibraries(rng) - } - } - - if chance(rng, 0.3) { - task.TimeoutSeconds = rng.IntN(3600) - } - if chance(rng, 0.3) { - task.MaxRetries = rng.IntN(5) - task.MinRetryIntervalMillis = rng.IntN(60000) - task.RetryOnTimeout = chance(rng, 0.5) - } - return task -} - -// assignCompute attaches exactly one compute source: a shared job cluster (when -// available), a new cluster, or an existing cluster id. -func assignCompute(rng *rand.Rand, task *jobs.Task, jobClusterKeys []string) { - const ( - computeNew = iota - computeExisting - computeShared - ) - options := []int{computeNew, computeExisting} - if len(jobClusterKeys) > 0 { - options = append(options, computeShared) - } - switch oneOf(rng, options) { - case computeNew: - spec := randClusterSpec(rng) - task.NewCluster = &spec - case computeExisting: - task.ExistingClusterId = randName(rng, "cluster") - case computeShared: - task.JobClusterKey = oneOf(rng, jobClusterKeys) - } -} - -func randClusterSpec(rng *rand.Rand) compute.ClusterSpec { - spec := compute.ClusterSpec{ - SparkVersion: oneOf(rng, sparkVersions), - NodeTypeId: oneOf(rng, nodeTypeIDs), - } - if chance(rng, 0.5) { - spec.NumWorkers = rng.IntN(8) - } else { - spec.Autoscale = &compute.AutoScale{ - MinWorkers: 1, - MaxWorkers: rng.IntN(8) + 2, - } - } - if chance(rng, 0.4) { - spec.SparkConf = map[string]string{ - "spark.databricks.delta.preview.enabled": "true", - "spark.speculation": strconv.FormatBool(chance(rng, 0.5)), - } - } - if chance(rng, 0.3) { - spec.CustomTags = randTags(rng) - } - if chance(rng, 0.3) { - spec.SparkEnvVars = map[string]string{"PYSPARK_PYTHON": "/databricks/python3/bin/python3"} - } - if chance(rng, 0.3) { - spec.DriverNodeTypeId = oneOf(rng, nodeTypeIDs) - } - return spec -} - -func randGitSource(rng *rand.Rand) *jobs.GitSource { - src := &jobs.GitSource{ - GitProvider: oneOf(rng, gitProviders), - GitUrl: "https://example.com/" + randWord(rng) + "/" + randWord(rng) + ".git", - } - switch rng.IntN(3) { - case 0: - src.GitBranch = oneOf(rng, []string{"main", "develop", "release"}) - case 1: - src.GitTag = "v" + fmt.Sprintf("%d.%d.0", rng.IntN(5), rng.IntN(10)) - case 2: - src.GitCommit = fmt.Sprintf("%040x", rng.Int64()) - } - return src -} - -func randEmailNotifications(rng *rand.Rand) *jobs.JobEmailNotifications { - email := randWord(rng) + "@example.com" - n := &jobs.JobEmailNotifications{NoAlertForSkippedRuns: chance(rng, 0.5)} - if chance(rng, 0.6) { - n.OnFailure = []string{email} - } - if chance(rng, 0.4) { - n.OnSuccess = []string{email} - } - if chance(rng, 0.3) { - n.OnStart = []string{email} - } - return n -} - -func randWebhookNotifications(rng *rand.Rand) *jobs.WebhookNotifications { - hook := []jobs.Webhook{{Id: randName(rng, "hook")}} - n := &jobs.WebhookNotifications{} - if chance(rng, 0.6) { - n.OnFailure = hook - } - if chance(rng, 0.4) { - n.OnSuccess = hook - } - return n -} - -func randHealth(rng *rand.Rand) *jobs.JobsHealthRules { - return &jobs.JobsHealthRules{ - Rules: []jobs.JobsHealthRule{ - { - Metric: jobs.JobsHealthMetric(oneOf(rng, healthMetrics)), - Op: jobs.JobsHealthOperatorGreaterThan, - Value: int64(rng.IntN(3600) + 1), - }, - }, - } -} - -func randLibraries(rng *rand.Rand) []compute.Library { - n := rng.IntN(2) + 1 - libs := make([]compute.Library, 0, n) - for range n { - switch rng.IntN(3) { - case 0: - libs = append(libs, compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: randWord(rng)}}) - case 1: - libs = append(libs, compute.Library{Maven: &compute.MavenLibrary{Coordinates: "org.example:" + randWord(rng) + ":1.0.0"}}) - case 2: - libs = append(libs, compute.Library{Whl: "/Workspace/Users/test/" + randName(rng, "lib") + ".whl"}) - } - } - return libs -} - -func randParameters(rng *rand.Rand) []jobs.JobParameterDefinition { - n := rng.IntN(3) + 1 - params := make([]jobs.JobParameterDefinition, 0, n) - for i := range n { - params = append(params, jobs.JobParameterDefinition{ - Name: fmt.Sprintf("param_%d", i), - Default: randWord(rng), - }) - } - return params -} - -func randTags(rng *rand.Rand) map[string]string { - n := rng.IntN(3) + 1 - tags := make(map[string]string, n) - for i := range n { - tags[fmt.Sprintf("tag_%d", i)] = randWord(rng) - } - return tags -} diff --git a/bundle/fuzz/invariants_cases_test.go b/bundle/fuzz/invariants_cases_test.go deleted file mode 100644 index ea14c45f508..00000000000 --- a/bundle/fuzz/invariants_cases_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package fuzz - -import ( - "encoding/json" - "testing" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" -) - -// recordingT captures whether the invariant assertions failed, so the table below -// can check that a bad payload is rejected and a good one is accepted without a -// real deploy. -type recordingT struct{ failed bool } - -func (r *recordingT) Errorf(string, ...any) { r.failed = true } - -// FailNow is only reached if decodePayload errors; every case here is valid JSON, -// so record and stop the goroutine the way require would. -func (r *recordingT) FailNow() { panic("unexpected FailNow") } - -func TestCheckJobInvariants(t *testing.T) { - job := &resources.Job{ - JobSettings: jobs.JobSettings{ - Name: "j", - JobClusters: []jobs.JobCluster{ - {JobClusterKey: "shared", NewCluster: compute.ClusterSpec{}}, - }, - Tasks: []jobs.Task{ - {TaskKey: "a"}, - {TaskKey: "b"}, - }, - }, - } - - tests := []struct { - name string - payload string - wantFailed bool - }{ - { - name: "valid payload", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"num_workers":0}}],"tasks":[{"task_key":"a","job_cluster_key":"shared"},{"task_key":"b","depends_on":[{"task_key":"a"}]}]}`, - }, - { - name: "renamed job", - payload: `{"name":"other","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - wantFailed: true, - }, - { - name: "dropped task", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a"}]}`, - wantFailed: true, - }, - { - name: "dangling dependency", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a"},{"task_key":"b","depends_on":[{"task_key":"ghost"}]}]}`, - wantFailed: true, - }, - { - name: "dangling job cluster reference", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared"}],"tasks":[{"task_key":"a","job_cluster_key":"missing"},{"task_key":"b"}]}`, - wantFailed: true, - }, - { - name: "new_cluster without explicit size is a valid single node", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"spark_version":"x"}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - }, - { - name: "single-node new_cluster with num_workers 0", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"num_workers":0}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - }, - { - name: "autoscale new_cluster", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"autoscale":{"min_workers":1,"max_workers":3}}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - }, - { - name: "new_cluster sets both autoscale and num_workers", - payload: `{"name":"j","job_clusters":[{"job_cluster_key":"shared","new_cluster":{"autoscale":{"min_workers":1,"max_workers":3},"num_workers":2}}],"tasks":[{"task_key":"a"},{"task_key":"b"}]}`, - wantFailed: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - rec := &recordingT{} - checkJobInvariants(rec, 0, job, json.RawMessage(tt.payload)) - assert.Equal(t, tt.wantFailed, rec.failed) - }) - } -} diff --git a/bundle/fuzz/invariants_test.go b/bundle/fuzz/invariants_test.go deleted file mode 100644 index 055390236a7..00000000000 --- a/bundle/fuzz/invariants_test.go +++ /dev/null @@ -1,175 +0,0 @@ -package fuzz - -import ( - "bytes" - "encoding/json" - "fmt" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// checkJobInvariants asserts the properties that any valid job's create payload -// must satisfy, independent of deploy engine. Unlike a terraform/direct payload -// diff, an invariant has no legitimate reason to fail, so a failure is a real bug -// and the seed reproduces it. Each invariant is checked separately so a failure -// points at the property that broke. -func checkJobInvariants(t require.TestingT, seed int64, job *resources.Job, payload json.RawMessage) { - p, err := decodePayload(payload) - require.NoErrorf(t, err, "seed %d: decoding create payload", seed) - - nameMatchesConfig(t, seed, job, p) - taskKeysMatchConfig(t, seed, job, p) - dependenciesResolve(t, seed, p) - jobClusterKeysMatchConfig(t, seed, job, p) - taskClusterRefsResolve(t, seed, p) - newClustersSizedExclusively(t, seed, p) -} - -// nameMatchesConfig: the engine must not rename the job. -func nameMatchesConfig(t require.TestingT, seed int64, job *resources.Job, p map[string]any) { - assert.Equalf(t, job.Name, p["name"], "seed %d: payload name must match config", seed) -} - -// taskKeysMatchConfig: the payload must carry exactly the tasks from config, no -// more and no fewer, identified by task_key. -func taskKeysMatchConfig(t require.TestingT, seed int64, job *resources.Job, p map[string]any) { - want := make([]string, 0, len(job.Tasks)) - for _, task := range job.Tasks { - want = append(want, task.TaskKey) - } - assert.ElementsMatchf(t, want, taskKeys(p), "seed %d: payload task keys must match config", seed) -} - -// dependenciesResolve: every depends_on must point at a task in the same payload. -func dependenciesResolve(t require.TestingT, seed int64, p map[string]any) { - keys := sliceToSet(taskKeys(p)) - for _, task := range payloadTasks(p) { - for _, dep := range slice(task["depends_on"]) { - d, ok := dep.(map[string]any) - if !ok { - continue - } - assert.Containsf(t, keys, d["task_key"], - "seed %d: task %v depends on unknown task %v", seed, task["task_key"], d["task_key"]) - } - } -} - -// jobClusterKeysMatchConfig: the payload's shared job clusters must match config. -func jobClusterKeysMatchConfig(t require.TestingT, seed int64, job *resources.Job, p map[string]any) { - want := make([]string, 0, len(job.JobClusters)) - for _, jc := range job.JobClusters { - want = append(want, jc.JobClusterKey) - } - assert.ElementsMatchf(t, want, jobClusterKeys(p), "seed %d: payload job cluster keys must match config", seed) -} - -// taskClusterRefsResolve: a task referencing a shared cluster must reference one -// declared in job_clusters. -func taskClusterRefsResolve(t require.TestingT, seed int64, p map[string]any) { - keys := sliceToSet(jobClusterKeys(p)) - for _, task := range payloadTasks(p) { - ref, ok := task["job_cluster_key"].(string) - if !ok || ref == "" { - continue - } - assert.Containsf(t, keys, ref, - "seed %d: task %v references unknown job cluster %q", seed, task["task_key"], ref) - } -} - -// newClustersSizedExclusively: a new_cluster is sized either by autoscale or by a -// fixed num_workers, never both. The two are mutually exclusive cluster shapes, so -// an engine emitting both (e.g. force-sending num_workers onto an autoscale -// cluster) produces a payload the backend rejects. -func newClustersSizedExclusively(t require.TestingT, seed int64, p map[string]any) { - for _, c := range newClusters(p) { - _, hasAutoscale := c["autoscale"] - _, hasNumWorkers := c["num_workers"] - assert.Falsef(t, hasAutoscale && hasNumWorkers, - "seed %d: new_cluster must not set both autoscale and num_workers, got %v", seed, c) - } -} - -// decodePayload unmarshals the create body with UseNumber so large int64 values -// (job ids, spark_context_id) aren't corrupted by float64 rounding. -func decodePayload(raw json.RawMessage) (map[string]any, error) { - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - var p map[string]any - if err := dec.Decode(&p); err != nil { - return nil, fmt.Errorf("decoding payload: %w", err) - } - return p, nil -} - -// payloadTasks returns the payload's task objects. -func payloadTasks(p map[string]any) []map[string]any { - tasks := make([]map[string]any, 0, len(slice(p["tasks"]))) - for _, el := range slice(p["tasks"]) { - if m, ok := el.(map[string]any); ok { - tasks = append(tasks, m) - } - } - return tasks -} - -func taskKeys(p map[string]any) []string { - var keys []string - for _, task := range payloadTasks(p) { - if k, ok := task["task_key"].(string); ok { - keys = append(keys, k) - } - } - return keys -} - -func jobClusterKeys(p map[string]any) []string { - var keys []string - for _, el := range slice(p["job_clusters"]) { - jc, ok := el.(map[string]any) - if !ok { - continue - } - if k, ok := jc["job_cluster_key"].(string); ok { - keys = append(keys, k) - } - } - return keys -} - -// newClusters returns every new_cluster spec in the payload: one per task that -// defines its own cluster plus one per shared job cluster. -func newClusters(p map[string]any) []map[string]any { - var specs []map[string]any - for _, task := range payloadTasks(p) { - if c, ok := task["new_cluster"].(map[string]any); ok { - specs = append(specs, c) - } - } - for _, el := range slice(p["job_clusters"]) { - jc, ok := el.(map[string]any) - if !ok { - continue - } - if c, ok := jc["new_cluster"].(map[string]any); ok { - specs = append(specs, c) - } - } - return specs -} - -func slice(v any) []any { - s, _ := v.([]any) - return s -} - -func sliceToSet(s []string) map[string]bool { - set := make(map[string]bool, len(s)) - for _, v := range s { - set[v] = true - } - return set -} diff --git a/bundle/fuzz/rand_test.go b/bundle/fuzz/rand_test.go deleted file mode 100644 index 529e4da1153..00000000000 --- a/bundle/fuzz/rand_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package fuzz - -import ( - "fmt" - "math/rand/v2" - "strings" -) - -var words = []string{ - "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", - "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa", -} - -// newRNG returns a deterministic RNG for the given seed, so any job the fuzzer -// flags can be regenerated from the printed seed alone. -func newRNG(seed int64) *rand.Rand { - return rand.New(rand.NewPCG(uint64(seed), 0)) -} - -// chance returns true with probability p (0..1). -func chance(rng *rand.Rand, p float64) bool { - return rng.Float64() < p -} - -// oneOf returns a random element of s. s must be non-empty. -func oneOf[T any](rng *rand.Rand, s []T) T { - return s[rng.IntN(len(s))] -} - -func randWord(rng *rand.Rand) string { - return oneOf(rng, words) -} - -// randName returns a deterministic-but-varied identifier with the given prefix, -// e.g. "job_alpha_4271". -func randName(rng *rand.Rand, prefix string) string { - return fmt.Sprintf("%s_%s_%d", prefix, randWord(rng), rng.IntN(10000)) -} - -func randSentence(rng *rand.Rand) string { - n := rng.IntN(4) + 2 - parts := make([]string, 0, n) - for range n { - parts = append(parts, randWord(rng)) - } - return strings.Join(parts, " ") -} diff --git a/bundle/fuzz/recorder_test.go b/bundle/fuzz/recorder_test.go deleted file mode 100644 index cfabf227219..00000000000 --- a/bundle/fuzz/recorder_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package fuzz - -import ( - "encoding/json" - "sync" - - "github.com/databricks/cli/libs/testserver" -) - -// jobsCreatePath is the Jobs API route the deploy must hit on create. The -// testserver registers only this version, so posting to a different one surfaces -// as a capture failure ("did not POST"). -const jobsCreatePath = "/api/2.2/jobs/create" - -// capturedRequest is a single mutating API request observed by the testserver. -type capturedRequest struct { - Method string - Path string - Body json.RawMessage -} - -// recorder collects request bodies sent to a testserver. It is safe for -// concurrent use because the deploy may issue requests from multiple goroutines. -type recorder struct { - mu sync.Mutex - requests []capturedRequest -} - -func (r *recorder) callback(req *testserver.Request) { - r.mu.Lock() - defer r.mu.Unlock() - - var body json.RawMessage - if json.Valid(req.Body) { - // Copy: testserver reuses the underlying buffer across requests. - body = append(json.RawMessage(nil), req.Body...) - } - - r.requests = append(r.requests, capturedRequest{ - Method: req.Method, - Path: req.URL.Path, - Body: body, - }) -} - -// find returns the body of the first recorded request matching method and path. -func (r *recorder) find(method, path string) (json.RawMessage, bool) { - r.mu.Lock() - defer r.mu.Unlock() - - for _, req := range r.requests { - if req.Method == method && req.Path == path { - return req.Body, true - } - } - return nil, false -} From ead1515b9fa3cea19eb9fe168bf7667a78450dc1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 29 Jun 2026 08:52:47 +0000 Subject: [PATCH 16/53] acceptance/fuzz: clarify comments and tidy schema fuzz harness - Correct misleading comments: the nightly test-fuzz job runs the same local harness against the fake server (wider seed window + drift on), not a real workspace. - Run config generation inside the per-seed subshell so a generator crash also prints the "reproduce with" hint. - Document the schema-driven fuzz subdir in the invariant README, including that a failure is a real CLI bug and how to reproduce it. - Drop the unused name hint in gen_config (objects ignore it). --- .github/workflows/push.yml | 2 +- acceptance/bin/gen_fuzz_config.py | 2 +- acceptance/bundle/invariant/README.md | 6 ++++++ acceptance/bundle/invariant/fuzz/script | 10 ++++++---- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3b60837cb98..db40b4174da 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -397,7 +397,7 @@ jobs: needs: - cleanups - # A real deploy per seed across a wide rotating window, with the no-drift + # Sweeps a wide rotating seed window against the fake server with the no-drift # invariant on: too slow for every PR, so nightly only and not part of # test-result. (The committed acceptance fuzz test still checks the no-panic # invariant on a small fixed seed window on every PR.) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 85909bb03fb..f016690725e 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -147,7 +147,7 @@ def gen_config(schema, seed, unique, allowed): element = obj["additionalProperties"] key = f"fuzz_{rtype}_{seed}" - instance = gen.gen(element, 0, "name") + instance = gen.gen(element, 0) return { "bundle": {"name": f"fuzz-{unique}"}, "resources": {rtype: {key: instance}}, diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 184d3f541c4..12b87902dd6 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -4,3 +4,9 @@ no_drift test checks that there are no actions planned after successful deploy. test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. + +The fuzz/ test is different: instead of a curated config it generates random configs +from the live `databricks bundle schema` (see fuzz/script). Because the schema is read +from the CLI under test, an unrelated change to a resource struct can shift a seed onto +a new config. A failure there is a real CLI bug (a panic, internal error, or drift), not +test flakiness; reproduce it with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index f7751dde581..ce12ddce64f 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -13,7 +13,8 @@ # # Drift checking is opt-in (FUZZ_CHECK_DRIFT): a freshly deployed random config can # legitimately differ from the fake server's state, so the local/PR run asserts only -# the cheap no-panic invariant. The nightly job enables drift on a real workspace. +# the cheap no-panic invariant. The nightly job runs this same harness against the +# fake server with a wider seed window and drift on (see Taskfile.yml test-fuzz). START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" @@ -27,12 +28,13 @@ for ((offset = 0; offset < COUNT; offset++)); do dir="seed-$seed" mkdir -p "$dir" - gen_fuzz_config.py --schema schema.json --seed "$seed" --unique "$UNIQUE_NAME-$seed" --resources "${FUZZ_RESOURCES:-}" > "$dir/databricks.yml" 2>"$dir/LOG.gen.err" - cat "$dir/LOG.gen.err" | contains.py '!Traceback' > /dev/null - + # Run inside the subshell so a generator crash also prints the repro hint below. ( cd "$dir" + gen_fuzz_config.py --schema ../schema.json --seed "$seed" --unique "$UNIQUE_NAME-$seed" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + # The CLI is allowed to reject a generated config, but never to crash. set +e $CLI bundle validate &> LOG.validate From 3e106cec72987fca1821552b98e0558878a3054d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 29 Jun 2026 09:11:28 +0000 Subject: [PATCH 17/53] acceptance/fuzz: shorten and tighten comments Make the comments across the schema fuzz harness more concise while keeping the non-obvious "why" context. --- .github/workflows/push.yml | 18 ++++----- Taskfile.yml | 8 ++-- acceptance/bin/gen_fuzz_config.py | 46 +++++++++++----------- acceptance/bundle/invariant/README.md | 10 ++--- acceptance/bundle/invariant/fuzz/script | 29 ++++++-------- acceptance/bundle/invariant/fuzz/test.toml | 6 +-- 6 files changed, 53 insertions(+), 64 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index db40b4174da..e0e103b276c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -397,10 +397,9 @@ jobs: needs: - cleanups - # Sweeps a wide rotating seed window against the fake server with the no-drift - # invariant on: too slow for every PR, so nightly only and not part of - # test-result. (The committed acceptance fuzz test still checks the no-panic - # invariant on a small fixed seed window on every PR.) + # Wide rotating seed window with drift checking on: too slow for every PR, so + # nightly only and not part of test-result. The committed acceptance test still + # checks the no-panic invariant on a small fixed window per PR. if: ${{ github.event_name == 'schedule' }} name: "task test-fuzz" runs-on: @@ -414,7 +413,7 @@ jobs: permissions: id-token: write contents: read - # Needed by the failure-reporting step below to open/comment a tracking issue. + # Failure-reporting step opens/comments a tracking issue. issues: write steps: @@ -428,16 +427,15 @@ jobs: - name: Run tests env: - # Shift the seed window each nightly run so CI explores new configs. - # start = GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT keeps windows non-overlapping - # (GITHUB_RUN_NUMBER is monotonic). A failure prints the failing seed. + # start = monotonic GITHUB_RUN_NUMBER * COUNT keeps each nightly window + # non-overlapping, so CI explores new configs every run. FUZZ_SEED_COUNT: "25" run: | export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) go tool -modfile=tools/task/go.mod task test-fuzz - # Excluded from test-result, so surface failures as a GitHub issue. Reuse one - # open issue (deduped by label) so a recurring failure doesn't spam nightly. + # Not in test-result, so surface failures as an issue. Reuse one open issue + # (deduped by label) so a recurring failure doesn't spam nightly. - name: Report failure if: ${{ failure() }} env: diff --git a/Taskfile.yml b/Taskfile.yml index 152af84e4e4..d5e96842b1e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -732,12 +732,12 @@ tasks: test-fuzz: desc: Run schema fuzz invariant tests (random configs, direct engine) - # No `sources:` fingerprint: the seed window depends on FUZZ_* env vars Task - # can't see, so always run rather than no-op a repro or a shifted nightly window. + # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't + # see, so always run rather than no-op a repro or shifted nightly window. cmds: - | - # Sweep a wider window than the committed acceptance run and turn on the - # no-drift invariant; a repro can narrow it with FUZZ_SEED_START/COUNT. + # Wider window than the committed run, with drift checking on; a repro can + # narrow it via FUZZ_SEED_START/COUNT. export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" {{.GO_TOOL}} gotestsum \ diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index f016690725e..1c3f53d046d 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -2,14 +2,13 @@ """ Generate a random bundle config from the bundle JSON schema. -The generator walks the schema (`databricks bundle schema`), resolving $ref and -picking concrete branches of oneOf/anyOf, and emits a single random resource as a -databricks.yml. It is seeded so a failing run can be reproduced with the same --seed. - -This feeds the invariant tests (see acceptance/bundle/invariant/): the harness -deploys the generated config and asserts invariants such as no-drift. Configs the -CLI rejects are filtered out by the harness before invariants are checked, so the -generator is free to produce structurally-random-but-sometimes-invalid configs. +Walks the schema (`databricks bundle schema`), resolving $ref and picking concrete +branches of oneOf/anyOf, and emits one random resource as a databricks.yml. Seeded +so a failing run reproduces with the same --seed. + +Feeds the invariant tests (see acceptance/bundle/invariant/). The harness filters out +configs the CLI rejects, so the generator may emit structurally-random-but-sometimes- +invalid configs. """ import argparse @@ -17,14 +16,13 @@ import random import sys -# Maximum object/array nesting depth. The schema is recursive (e.g. job tasks -> -# for_each_task -> task), so without a cap the walk would not terminate. +# Cap nesting depth: the schema is recursive (e.g. task -> for_each_task -> task), +# so without a cap the walk would not terminate. MAX_DEPTH = 6 -# A string branch whose pattern matches a ${...} reference. These exist because the -# schema generator wraps every concrete field in a oneOf with interpolation-string -# alternatives (see bundle/internal/schema/main.go addInterpolationPatterns). We -# generate concrete values, not references, so these branches are skipped. +# Matches the ${...} interpolation-string branches the schema wraps every concrete +# field in (see bundle/internal/schema/main.go addInterpolationPatterns). We emit +# concrete values, so these branches are skipped. INTERPOLATION_MARKER = "\\$\\{" @@ -35,8 +33,8 @@ def __init__(self, schema, rng, unique): self.unique = unique def resolve(self, schema): - # Follow $ref chains. A ref looks like "#/$defs/github.com/.../resources.Job"; - # definitions are nested under $defs by the "/"-separated path segments. + # Follow $ref chains, e.g. "#/$defs/github.com/.../resources.Job", nested + # under $defs by "/"-separated path segments. while isinstance(schema, dict) and "$ref" in schema: cur = self.root["$defs"] for part in schema["$ref"].split("/")[2:]: @@ -48,7 +46,7 @@ def is_interpolation(self, branch): return branch.get("type") == "string" and INTERPOLATION_MARKER in branch.get("pattern", "") def choose_branch(self, branches): - # Prefer concrete branches over the ${...} interpolation-string alternatives. + # Prefer concrete branches over the ${...} alternatives. concrete = [b for b in branches if not self.is_interpolation(b)] return self.rng.choice(concrete or branches) @@ -82,8 +80,8 @@ def gen_object(self, schema, depth): result = {} for prop_name, prop_schema in props.items(): - # Always emit required fields; emit optional ones with decreasing - # probability as we go deeper to keep configs from exploding. + # Always emit required fields; emit optional ones less often as we go + # deeper to keep configs from exploding. keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) if not keep: continue @@ -91,8 +89,8 @@ def gen_object(self, schema, depth): if value is not None: result[prop_name] = value - # Map type (additionalProperties schema, no fixed properties): synthesize a - # few random keys, e.g. resources. or string maps like tags. + # Map type (additionalProperties, no fixed properties): synthesize a few + # random keys, e.g. resources. or string maps like tags. if self.is_map(schema): for _ in range(self.rng.randint(1, 2)): key = self.token() @@ -124,7 +122,7 @@ def token(self): def resource_types(schema, gen): - # resources is `oneOf[{object with one property per resource type}]`. + # resources is oneOf[{ object with one property per resource type }]. resources = gen.resolve(schema["properties"]["resources"]) obj = next(b for b in resources["oneOf"] if b.get("type") == "object") return obj["properties"] @@ -140,8 +138,8 @@ def gen_config(schema, seed, unique, allowed): sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") rtype = rng.choice(sorted(candidates)) - # Each resource type is a map ref; its element schema lives under the object - # branch's additionalProperties. + # Each resource type is a map ref; the element schema is the object branch's + # additionalProperties. map_schema = gen.resolve(types[rtype]) obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") element = obj["additionalProperties"] diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 12b87902dd6..defcafcf35d 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -5,8 +5,8 @@ test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. -The fuzz/ test is different: instead of a curated config it generates random configs -from the live `databricks bundle schema` (see fuzz/script). Because the schema is read -from the CLI under test, an unrelated change to a resource struct can shift a seed onto -a new config. A failure there is a real CLI bug (a panic, internal error, or drift), not -test flakiness; reproduce it with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. +The fuzz/ test instead generates random configs from the live `databricks bundle +schema` (see fuzz/script). Since the schema comes from the CLI under test, an unrelated +struct change can shift a seed onto a new config. A failure is a real CLI bug (panic, +internal error, or drift), not flakiness; reproduce with +`FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index ce12ddce64f..84994b9f66c 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -1,20 +1,16 @@ -# Invariant to test: the CLI never panics or hits an internal error on any config -# generated from the bundle schema, and a config that deploys cleanly has no drift. +# Invariant: the CLI never panics or hits an internal error on any config generated +# from the bundle schema, and a config that deploys cleanly has no drift. # -# gen_fuzz_config.py walks the schema emitted by the CLI under test and produces a -# random-but-schema-valid config. Most invariant work is shared with the no_drift -# test; the difference is the input is generated, not a curated template. +# gen_fuzz_config.py walks the schema emitted by the CLI under test to produce a +# random schema-valid config; the rest is shared with the no_drift test. # -# Seeds form a window [START, START+COUNT). The window is env-driven so the nightly -# job can sweep a wide, non-overlapping range (see Taskfile.yml test-fuzz) while this -# committed test stays small and deterministic. Everything is routed to LOG.* / *.json -# so output.txt stays empty regardless of the window: a violation fails via exit code, -# not via output diff, which is what lets the same test run under any seed window. +# Seeds form a window [START, START+COUNT), env-driven so the nightly job can sweep a +# wide non-overlapping range while this committed test stays small. All output goes to +# LOG.* / *.json so output.txt stays empty: a violation fails via exit code, not diff, +# which lets the same test run under any seed window. # -# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a freshly deployed random config can -# legitimately differ from the fake server's state, so the local/PR run asserts only -# the cheap no-panic invariant. The nightly job runs this same harness against the -# fake server with a wider seed window and drift on (see Taskfile.yml test-fuzz). +# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a fresh random config can legitimately +# differ from the fake server's state, so the PR run only asserts no-panic. START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" @@ -28,7 +24,7 @@ for ((offset = 0; offset < COUNT; offset++)); do dir="seed-$seed" mkdir -p "$dir" - # Run inside the subshell so a generator crash also prints the repro hint below. + # Subshell so a generator crash also prints the repro hint below. ( cd "$dir" @@ -43,8 +39,7 @@ for ((offset = 0; offset < COUNT; offset++)); do set -e cat LOG.validate LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - # Deploy failed => config was rejected (not a bug). This is the negative of - # the no_drift test's INPUT_CONFIG_OK marker: nothing more to assert. + # Deploy failed => config was rejected, not a bug; nothing more to assert. if [ "$deploy_rc" -ne 0 ]; then exit 0 fi diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 019d2dc6494..caed93c23e5 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -1,5 +1,3 @@ -# Schema fuzzing: generate random configs from the bundle schema and assert -# invariants (see script). Unlike the curated-corpus invariant tests (no_drift, -# migrate), the fuzzer generates its own configs, so drop the inherited -# INPUT_CONFIG matrix. +# Schema fuzzing (see script). Unlike the curated invariant tests, the fuzzer +# generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] From 854f00f643a99fb051d283eaa9472e62419777e3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 29 Jun 2026 13:44:29 +0000 Subject: [PATCH 18/53] acceptance/fuzz: report nightly failures on the PR instead of an issue Mirror the integration-test flow: comment on the PR that introduced the failing commit rather than opening/deduping a tracking issue. --- .github/workflows/push.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e0e103b276c..f30a6a96f05 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -413,8 +413,8 @@ jobs: permissions: id-token: write contents: read - # Failure-reporting step opens/comments a tracking issue. - issues: write + # Failure-reporting step comments on the PR that introduced the failing commit. + pull-requests: write steps: - name: Checkout repository and submodules @@ -434,20 +434,17 @@ jobs: export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) go tool -modfile=tools/task/go.mod task test-fuzz - # Not in test-result, so surface failures as an issue. Reuse one open issue - # (deduped by label) so a recurring failure doesn't spam nightly. + # Not in test-result, so surface failures by commenting on the PR that + # introduced the commit under test. - name: Report failure if: ${{ failure() }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT: ${{ github.sha }} run: | - gh label create fuzz-nightly \ - --description "Nightly schema fuzz invariant failures" \ - --color FBCA04 2>/dev/null || true - body=$(cat <&2 fi # This job groups the result of all the above test jobs. From 5c9ea27fbba6c379d1b2fbacefa8ae6d96fbdf4c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 29 Jun 2026 13:44:31 +0000 Subject: [PATCH 19/53] acceptance/fuzz: reuse the no_drift invariant check instead of duplicating it Extract the no_drift deploy/drift/destroy body into a shared no_drift.sh sourced by both the no_drift test and the fuzzer, so the invariant lives in one place and other invariant tests can be fuzzed the same way. --- acceptance/bundle/invariant/README.md | 3 +- acceptance/bundle/invariant/fuzz/script | 77 +++++++++++++-------- acceptance/bundle/invariant/no_drift.sh | 55 +++++++++++++++ acceptance/bundle/invariant/no_drift/script | 35 +--------- 4 files changed, 106 insertions(+), 64 deletions(-) create mode 100644 acceptance/bundle/invariant/no_drift.sh diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index defcafcf35d..a3b305f4ef1 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -6,7 +6,8 @@ test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. The fuzz/ test instead generates random configs from the live `databricks bundle -schema` (see fuzz/script). Since the schema comes from the CLI under test, an unrelated +schema` (see fuzz/script) and runs each one through the same no_drift.sh check the +no_drift test uses. Since the schema comes from the CLI under test, an unrelated struct change can shift a seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), not flakiness; reproduce with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 84994b9f66c..1af93647ebd 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -2,19 +2,36 @@ # from the bundle schema, and a config that deploys cleanly has no drift. # # gen_fuzz_config.py walks the schema emitted by the CLI under test to produce a -# random schema-valid config; the rest is shared with the no_drift test. +# random schema-valid config; the no-drift / no-panic checks are the shared +# ../no_drift.sh body, the same one the no_drift test runs. Reusing it keeps the +# deploy/drift/destroy assertions in one place and lets other invariant tests be +# fuzzed the same way. # # Seeds form a window [START, START+COUNT), env-driven so the nightly job can sweep a # wide non-overlapping range while this committed test stays small. All output goes to -# LOG.* / *.json so output.txt stays empty: a violation fails via exit code, not diff, -# which lets the same test run under any seed window. +# LOG.* so output.txt stays empty: a violation fails via exit code, not diff, which +# lets the same test run under any seed window. # -# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a fresh random config can legitimately -# differ from the fake server's state, so the PR run only asserts no-panic. +# The CLI is free to reject a generated config; that is not a bug. ../no_drift.sh +# prints INPUT_CONFIG_OK once a config deploys cleanly, so a non-zero result before +# that marker (with no panic) means the config was rejected and is skipped, while a +# panic anywhere or a failure after the marker (drift, destroy) is a real CLI bug. +# +# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a fresh random config can deploy yet +# legitimately differ from the fake server's state, so the committed run only asserts +# no-panic and tells ../no_drift.sh to skip its drift assertion. START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" +if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then + export SKIP_DRIFT_CHECK=1 +fi + +# no_drift.sh deploys via readplanarg, which reads READPLAN; the fuzzer doesn't use +# the saved-plan matrix, so deploy once without it (and satisfy the script's set -u). +export READPLAN="" + # Emit the schema from the CLI under test so the generator always matches it. $CLI bundle schema > schema.json 2>LOG.schema.err cat LOG.schema.err | contains.py '!panic' '!internal error' > /dev/null @@ -24,36 +41,36 @@ for ((offset = 0; offset < COUNT; offset++)); do dir="seed-$seed" mkdir -p "$dir" - # Subshell so a generator crash also prints the repro hint below. + # Subshell so a generator crash or shared-check failure is contained per seed. + set +e ( cd "$dir" - gen_fuzz_config.py --schema ../schema.json --seed "$seed" --unique "$UNIQUE_NAME-$seed" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null + source "$TESTDIR/../no_drift.sh" + ) > "$dir/LOG.check" 2>&1 + rc=$? + set -e + + if [ "$rc" -eq 0 ]; then + continue + fi + + bug="" + + # A panic or internal error is a bug even when the CLI then rejects the config. + if ! cat "$dir"/LOG.validate "$dir"/LOG.deploy 2>/dev/null | contains.py '!panic' '!internal error' > /dev/null; then + bug=1 + fi + + # Failing after INPUT_CONFIG_OK means the config deployed but drifted (or destroy + # failed); failing before it with no panic just means the config was rejected. + if grep -q INPUT_CONFIG_OK "$dir/LOG.check"; then + bug=1 + fi - # The CLI is allowed to reject a generated config, but never to crash. - set +e - $CLI bundle validate &> LOG.validate - $CLI bundle deploy &> LOG.deploy - deploy_rc=$? - set -e - cat LOG.validate LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - - # Deploy failed => config was rejected, not a bug; nothing more to assert. - if [ "$deploy_rc" -ne 0 ]; then - exit 0 - fi - - if [ -n "${FUZZ_CHECK_DRIFT:-}" ]; then - $CLI bundle plan -o json > plan.json 2>LOG.plan.err - cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null - verify_no_drift.py plan.json - fi - - $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - ) || { + if [ -n "$bug" ]; then echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 task test-fuzz" >&2 exit 1 - } + fi done diff --git a/acceptance/bundle/invariant/no_drift.sh b/acceptance/bundle/invariant/no_drift.sh new file mode 100644 index 00000000000..662b27e19f5 --- /dev/null +++ b/acceptance/bundle/invariant/no_drift.sh @@ -0,0 +1,55 @@ +# Shared invariant body: given a databricks.yml in the current directory, deploy it +# and assert there is no drift afterwards, with no panics / internal errors along +# the way. Sourced by no_drift/script (curated configs) and fuzz/script (random +# schema-generated configs) so the deploy/drift/destroy logic lives in one place. + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null + +cleanup() { + # Only destroy what we deployed. A curated config always deploys, but a random + # fuzzed config may be rejected, and destroying nothing just makes extra API + # calls (which fail the local fake server on unstubbed URLs). + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + + # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard + # the lookup against the script's `set -u`. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err +cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null + +trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# Drift is the whole point for the curated no_drift configs, but a random fuzzed +# config can deploy yet legitimately differ from the fake server's state, so the +# fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic invariant is asserted. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # Check both text and JSON plan for no changes + # Note, expect that there maybe more than one resource unchanged + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson + + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan + cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null +fi diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 481f64d9e74..1edd686409a 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -14,36 +14,5 @@ envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml cp databricks.yml LOG.config -cleanup() { - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -# Only compute a plan up front when the READPLAN variant actually consumes it via --plan. -# Without READPLAN, deploy computes its own plan internally, so a separate `bundle plan` -# invocation is pure waste. -if [[ -n "$READPLAN" ]]; then - $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err - cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null -fi - -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK - -# JSON plan asserts every action is "skip" -- a strict superset of the text -# renderer's "Plan: 0 to add, 0 to change, 0 to delete" summary. -$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err -cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null -verify_no_drift.py LOG.planjson +# Deploy and assert no drift. Shared with the fuzz invariant test. +source "$TESTDIR/../no_drift.sh" From 2ea9e3717b084588cda238b6bb06b9cc2ca3c34e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 29 Jun 2026 14:29:49 +0000 Subject: [PATCH 20/53] acceptance/fuzz: skip INPUT_CONFIG_OK marker when deploy is rejected A rejected config never deploys, so emitting the marker made the fuzzer read the re-plan's "needs create" as drift. --- acceptance/bundle/invariant/no_drift.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bundle/invariant/no_drift.sh b/acceptance/bundle/invariant/no_drift.sh index 662b27e19f5..9b3746a5acd 100644 --- a/acceptance/bundle/invariant/no_drift.sh +++ b/acceptance/bundle/invariant/no_drift.sh @@ -33,7 +33,15 @@ $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy +deploy_rc=$? cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise +# the fuzzer reads the re-plan's "needs create" as drift. Curated tests run under +# `bash -e` and already aborted above, so this only fires in the fuzzer subshell. +if [ "$deploy_rc" -ne 0 ]; then + return "$deploy_rc" +fi deployed=1 # Special message to fuzzer that generated config was fine. From b4d1a3107745e3ca36e91b3b6f40329777ac60af Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 30 Jun 2026 08:27:33 +0000 Subject: [PATCH 21/53] acceptance/fuzz: propagate drift failures so the fuzzer detects them The fuzzer runs the shared no_drift.sh body with errexit off and classifies each seed from the captured exit code. The drift block ended with a no-panic check that reset $? to 0, so a config that deployed cleanly but drifted was silently treated as a pass. Accumulate the drift assertions into drift_rc and return it instead. The curated no_drift test (errexit on) is unaffected. Also make verify_no_drift.py fail cleanly on empty/unparseable plan output (when bundle plan itself failed) instead of crashing with a traceback, and tighten the fuzz harness comments. --- acceptance/bin/gen_fuzz_config.py | 19 ++++++--------- acceptance/bin/verify_no_drift.py | 23 ++++++++++-------- acceptance/bundle/invariant/fuzz/script | 32 +++++++++---------------- acceptance/bundle/invariant/no_drift.sh | 15 +++++++----- 4 files changed, 40 insertions(+), 49 deletions(-) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 1c3f53d046d..672f4666419 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -2,13 +2,10 @@ """ Generate a random bundle config from the bundle JSON schema. -Walks the schema (`databricks bundle schema`), resolving $ref and picking concrete -branches of oneOf/anyOf, and emits one random resource as a databricks.yml. Seeded -so a failing run reproduces with the same --seed. - -Feeds the invariant tests (see acceptance/bundle/invariant/). The harness filters out -configs the CLI rejects, so the generator may emit structurally-random-but-sometimes- -invalid configs. +Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf +branches) and emits one random resource as databricks.yml, seeded by --seed. Feeds the +invariant tests; the harness filters out configs the CLI rejects, so output may be +structurally-random but sometimes invalid. """ import argparse @@ -16,13 +13,11 @@ import random import sys -# Cap nesting depth: the schema is recursive (e.g. task -> for_each_task -> task), -# so without a cap the walk would not terminate. +# The schema is recursive (e.g. task -> for_each_task -> task); cap the walk. MAX_DEPTH = 6 -# Matches the ${...} interpolation-string branches the schema wraps every concrete -# field in (see bundle/internal/schema/main.go addInterpolationPatterns). We emit -# concrete values, so these branches are skipped. +# The ${...} interpolation branch the schema wraps every field in (see +# bundle/internal/schema/main.go addInterpolationPatterns); we emit concrete values. INTERPOLATION_MARKER = "\\$\\{" diff --git a/acceptance/bin/verify_no_drift.py b/acceptance/bin/verify_no_drift.py index 9b272c1ce79..19d6ed28b87 100755 --- a/acceptance/bin/verify_no_drift.py +++ b/acceptance/bin/verify_no_drift.py @@ -11,18 +11,21 @@ def check_plan(path): with open(path) as fobj: raw = fobj.read() - changes_detected = 0 - + # Empty or unparseable output means `bundle plan` itself failed; report that + # cleanly instead of crashing with a traceback. + if not raw.strip(): + sys.exit(f"{path}: empty plan output (bundle plan failed)") try: data = json.loads(raw) - for key, value in data["plan"].items(): - action = value.get("action") - if action != "skip": - print(f"Unexpected {action=} for {key}") - changes_detected += 1 - except Exception: - print(raw, flush=True) - raise + except json.JSONDecodeError as e: + sys.exit(f"{path}: invalid plan JSON: {e}\n{raw}") + + changes_detected = 0 + for key, value in data["plan"].items(): + action = value.get("action") + if action != "skip": + print(f"Unexpected {action=} for {key}") + changes_detected += 1 if changes_detected: print(raw, flush=True) diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 1af93647ebd..634200c03b5 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -1,25 +1,16 @@ -# Invariant: the CLI never panics or hits an internal error on any config generated -# from the bundle schema, and a config that deploys cleanly has no drift. +# Invariant: the CLI never panics on a schema-generated config, and a config that +# deploys cleanly has no drift. gen_fuzz_config.py produces a random schema-valid +# config; ../no_drift.sh (shared with the no_drift test) does the deploy/drift/destroy +# checks. Output goes to LOG.* so a violation fails via exit code, not diff, letting +# the same test run under any seed window [START, START+COUNT). # -# gen_fuzz_config.py walks the schema emitted by the CLI under test to produce a -# random schema-valid config; the no-drift / no-panic checks are the shared -# ../no_drift.sh body, the same one the no_drift test runs. Reusing it keeps the -# deploy/drift/destroy assertions in one place and lets other invariant tests be -# fuzzed the same way. +# A rejected config is not a bug: ../no_drift.sh prints INPUT_CONFIG_OK once a config +# deploys, so a non-zero result before that marker (no panic) is just a rejection, +# while a panic anywhere or a failure after it (drift, destroy) is a real bug. # -# Seeds form a window [START, START+COUNT), env-driven so the nightly job can sweep a -# wide non-overlapping range while this committed test stays small. All output goes to -# LOG.* so output.txt stays empty: a violation fails via exit code, not diff, which -# lets the same test run under any seed window. -# -# The CLI is free to reject a generated config; that is not a bug. ../no_drift.sh -# prints INPUT_CONFIG_OK once a config deploys cleanly, so a non-zero result before -# that marker (with no panic) means the config was rejected and is skipped, while a -# panic anywhere or a failure after the marker (drift, destroy) is a real CLI bug. -# -# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a fresh random config can deploy yet +# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet # legitimately differ from the fake server's state, so the committed run only asserts -# no-panic and tells ../no_drift.sh to skip its drift assertion. +# no-panic and skips the drift assertion. START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" @@ -28,8 +19,7 @@ if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then export SKIP_DRIFT_CHECK=1 fi -# no_drift.sh deploys via readplanarg, which reads READPLAN; the fuzzer doesn't use -# the saved-plan matrix, so deploy once without it (and satisfy the script's set -u). +# no_drift.sh reads READPLAN via readplanarg; the fuzzer skips the saved-plan matrix. export READPLAN="" # Emit the schema from the CLI under test so the generator always matches it. diff --git a/acceptance/bundle/invariant/no_drift.sh b/acceptance/bundle/invariant/no_drift.sh index 9b3746a5acd..df0bc319aa1 100644 --- a/acceptance/bundle/invariant/no_drift.sh +++ b/acceptance/bundle/invariant/no_drift.sh @@ -52,12 +52,15 @@ echo INPUT_CONFIG_OK # config can deploy yet legitimately differ from the fake server's state, so the # fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic invariant is asserted. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # Check both text and JSON plan for no changes - # Note, expect that there maybe more than one resource unchanged + # Check both text and JSON plan for no changes (may be >1 unchanged resource). + # The fuzzer runs this with errexit off and reads the return code, so accumulate + # failures into drift_rc instead of letting the trailing no-panic check reset $?. + drift_rc=0 $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null - verify_no_drift.py LOG.planjson + cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + verify_no_drift.py LOG.planjson || drift_rc=1 - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan - cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 + cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + return "$drift_rc" fi From 81fc0bc9593aad3ab364c167aac309fbd1391dd0 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 30 Jun 2026 09:58:44 +0000 Subject: [PATCH 22/53] acceptance/fuzz: make the fuzzer run any invariant, not just no_drift Extract the migrate invariant body into a shared migrate.sh (mirroring no_drift.sh) and have the fuzzer source ../$FUZZ_INVARIANT.sh so it can exercise any invariant. Wire up FUZZ_INVARIANT=[no_drift, migrate] so the schema fuzzer now also stress-tests the Terraform->direct migration on random configs. The fuzzer's panic scan now globs LOG.* rather than naming LOG.validate/LOG.deploy, since different bodies write different logs. --- acceptance/bundle/invariant/README.md | 11 ++- .../bundle/invariant/fuzz/out.test.toml | 1 + acceptance/bundle/invariant/fuzz/script | 22 +++--- acceptance/bundle/invariant/fuzz/test.toml | 4 + acceptance/bundle/invariant/migrate.sh | 73 +++++++++++++++++++ acceptance/bundle/invariant/migrate/script | 39 +--------- 6 files changed, 100 insertions(+), 50 deletions(-) create mode 100644 acceptance/bundle/invariant/migrate.sh diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index a3b305f4ef1..80cc095b9e1 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -6,8 +6,11 @@ test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. The fuzz/ test instead generates random configs from the live `databricks bundle -schema` (see fuzz/script) and runs each one through the same no_drift.sh check the -no_drift test uses. Since the schema comes from the CLI under test, an unrelated -struct change can shift a seed onto a new config. A failure is a real CLI bug (panic, -internal error, or drift), not flakiness; reproduce with +schema` (see fuzz/script) and runs each one through a shared invariant body. The body +is selected by `FUZZ_INVARIANT` (matrixed in fuzz/test.toml) and is the same +`.sh` the matching curated test sources, so the fuzzer can exercise any +invariant: `no_drift.sh` (deploy + no drift) and `migrate.sh` (Terraform deploy + +migrate to direct + no drift) today. Since the schema comes from the CLI under test, +an unrelated struct change can shift a seed onto a new config. A failure is a real CLI +bug (panic, internal error, or drift), not flakiness; reproduce with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index 789aa10c799..aa67f82bc28 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_INVARIANT = ["no_drift", "migrate"] EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 634200c03b5..1d881073abc 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -1,12 +1,13 @@ # Invariant: the CLI never panics on a schema-generated config, and a config that # deploys cleanly has no drift. gen_fuzz_config.py produces a random schema-valid -# config; ../no_drift.sh (shared with the no_drift test) does the deploy/drift/destroy -# checks. Output goes to LOG.* so a violation fails via exit code, not diff, letting -# the same test run under any seed window [START, START+COUNT). +# config; the invariant body ../$FUZZ_INVARIANT.sh (shared with the matching curated +# invariant test, e.g. no_drift or migrate) does the deploy/drift/destroy checks. +# Output goes to LOG.* so a violation fails via exit code, not diff, letting the same +# test run under any seed window [START, START+COUNT). # -# A rejected config is not a bug: ../no_drift.sh prints INPUT_CONFIG_OK once a config -# deploys, so a non-zero result before that marker (no panic) is just a rejection, -# while a panic anywhere or a failure after it (drift, destroy) is a real bug. +# A rejected config is not a bug: every invariant body prints INPUT_CONFIG_OK once a +# config deploys, so a non-zero result before that marker (no panic) is just a +# rejection, while a panic anywhere or a failure after it (drift, destroy) is a real bug. # # Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet # legitimately differ from the fake server's state, so the committed run only asserts @@ -37,7 +38,7 @@ for ((offset = 0; offset < COUNT; offset++)); do cd "$dir" gen_fuzz_config.py --schema ../schema.json --seed "$seed" --unique "$UNIQUE_NAME-$seed" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null - source "$TESTDIR/../no_drift.sh" + source "$TESTDIR/../${FUZZ_INVARIANT:-no_drift}.sh" ) > "$dir/LOG.check" 2>&1 rc=$? set -e @@ -48,8 +49,11 @@ for ((offset = 0; offset < COUNT; offset++)); do bug="" - # A panic or internal error is a bug even when the CLI then rejects the config. - if ! cat "$dir"/LOG.validate "$dir"/LOG.deploy 2>/dev/null | contains.py '!panic' '!internal error' > /dev/null; then + # A panic or internal error anywhere is a bug even when the CLI then rejects the + # config. Invariant bodies write different LOG.* files (no_drift has LOG.validate, + # migrate has LOG.migrate), so scan whatever this run produced rather than naming + # specific files -- a missing name would otherwise fail the pipe under pipefail. + if ! cat "$dir"/LOG.* 2>/dev/null | contains.py '!panic' '!internal error' > /dev/null; then bug=1 fi diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index caed93c23e5..ef0f7b44382 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -1,3 +1,7 @@ # Schema fuzzing (see script). Unlike the curated invariant tests, the fuzzer # generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] + +# Fuzz each invariant body in ../.sh. no_drift runs on the direct engine; +# migrate ignores it and starts from a Terraform deployment (see migrate.sh). +EnvMatrix.FUZZ_INVARIANT = ["no_drift", "migrate"] diff --git a/acceptance/bundle/invariant/migrate.sh b/acceptance/bundle/invariant/migrate.sh new file mode 100644 index 00000000000..00f3948fc48 --- /dev/null +++ b/acceptance/bundle/invariant/migrate.sh @@ -0,0 +1,73 @@ +# Shared invariant body: given a databricks.yml in the current directory, deploy it +# with Terraform, migrate the deployment to the direct engine, and assert there is no +# drift afterwards, with no panics / internal errors along the way. Sourced by +# migrate/script (curated configs) and fuzz/script (random schema-generated configs) +# so the deploy/migrate/drift logic lives in one place. + +# migrate always starts from a Terraform deployment, so drop any engine the caller +# selected (the fuzzer runs the invariant matrix with DATABRICKS_BUNDLE_ENGINE=direct). +unset DATABRICKS_BUNDLE_ENGINE + +cleanup() { + # Only destroy what we deployed. A curated config always deploys, but a random + # fuzzed config may be rejected, and destroying nothing just makes extra API + # calls (which fail the local fake server on unstubbed URLs). + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard + # the lookup against the script's `set -u`. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null + +# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the +# fuzzer reads the failing migrate/drift below as a bug. Curated tests run under +# `bash -e` and already aborted above, so this only fires in the fuzzer subshell. +if [ "$deploy_rc" -ne 0 ]; then + return "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +MIGRATE_ARGS="" +# The terraform provider sorts depends_on entries alphabetically by task_key on Read +# (see terraform-provider-databricks PR #3000). Since depends_on uses TypeList +# (order-sensitive), terraform plan reports positional drift when the bundle config +# specifies depends_on in a different order than the provider's sorted state. +# This is a false positive -- the logical dependencies are identical. +if [[ "${INPUT_CONFIG:-}" == "job_with_depends_on.yml.tmpl" ]]; then + MIGRATE_ARGS="--noplancheck" +fi + +trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate + +cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null + +# Drift is the whole point for the curated migrate configs, but a random fuzzed +# config can migrate yet legitimately differ from the fake server's state, so the +# fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic invariant is asserted. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # The fuzzer runs this with errexit off and reads the return code, so accumulate + # failures into drift_rc instead of letting the trailing no-panic check reset $?. + drift_rc=0 + $CLI bundle plan -o json > plan.json 2>plan.json.err + cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 + verify_no_drift.py plan.json || drift_rc=1 + return "$drift_rc" +fi diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 78f45faa7da..78eb0630e15 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -1,8 +1,6 @@ # Invariant to test: migrate is successful, no drift after deploy # Additional checks: no internal errors / panics in any commands -unset DATABRICKS_BUNDLE_ENGINE - # Copy data files to test directory cp -r "$TESTDIR/../data/." . &> LOG.cp @@ -16,38 +14,5 @@ envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml cp databricks.yml LOG.config -cleanup() { - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null - -echo INPUT_CONFIG_OK - -MIGRATE_ARGS="" -# The terraform provider sorts depends_on entries alphabetically by task_key on Read -# (see terraform-provider-databricks PR #3000). Since depends_on uses TypeList -# (order-sensitive), terraform plan reports positional drift when the bundle config -# specifies depends_on in a different order than the provider's sorted state. -# This is a false positive -- the logical dependencies are identical. -if [[ "$INPUT_CONFIG" == "job_with_depends_on.yml.tmpl" ]]; then - MIGRATE_ARGS="--noplancheck" -fi - -trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate - -cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null - -$CLI bundle plan -o json > plan.json 2>plan.json.err -cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null -verify_no_drift.py plan.json +# Migrate then assert no drift. Shared with the fuzz invariant test. +source "$TESTDIR/../migrate.sh" From f5a940742182525562ece61801eb486dd9958139 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 30 Jun 2026 12:42:11 +0000 Subject: [PATCH 23/53] testserver: round-trip catalog create payload fields CatalogsCreate only echoed a subset of the create request, so a re-read returned null for connection_name, managed_encryption_settings, and custom_max_retention_hours. Because connection_name is recreate_on_changes (immutable), the schema fuzzer's no_drift invariant saw a perpetual recreate; the others showed as update drift. Persist these fields on create so the re-read matches the deployed config. Also clamp the fuzzer's custom_max_retention_hours to UC-valid values (0 or 168-720 hours) so generated catalog configs deploy. --- acceptance/bin/gen_fuzz_config.py | 4 ++++ libs/testserver/catalogs.go | 34 ++++++++++++++++++------------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 672f4666419..1fa03f0fe29 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -104,6 +104,10 @@ def gen_scalar(self, schema, name): if t == "boolean": return self.rng.choice([True, False]) if t == "integer": + # The field is in hours, but UC validates it as a window of 0 or 7-30 + # days; only 0 or 168-720 (hours) are accepted. + if name == "custom_max_retention_hours": + return self.rng.choice([0, self.rng.randint(168, 720)]) return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) if t == "number": return round(self.rng.uniform(0, 1000), 2) diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 351c5a76499..0c86d113c8b 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -25,20 +25,26 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } catalogInfo := catalog.CatalogInfo{ - Name: createRequest.Name, - Comment: createRequest.Comment, - StorageRoot: createRequest.StorageRoot, - ProviderName: createRequest.ProviderName, - ShareName: createRequest.ShareName, - Options: createRequest.Options, - Properties: createRequest.Properties, - FullName: createRequest.Name, - CreatedAt: nowMilli(), - CreatedBy: s.CurrentUser().UserName, - UpdatedBy: s.CurrentUser().UserName, - MetastoreId: nextUUID(), - Owner: s.CurrentUser().UserName, - CatalogType: catalog.CatalogTypeManagedCatalog, + Name: createRequest.Name, + Comment: createRequest.Comment, + // Round-trip the remaining create-request fields so a re-read matches the + // deployed config. Dropping them made connection_name (recreate_on_changes) + // re-plan as a perpetual recreate and managed_encryption_settings as drift. + StorageRoot: createRequest.StorageRoot, + ProviderName: createRequest.ProviderName, + ShareName: createRequest.ShareName, + ConnectionName: createRequest.ConnectionName, + ManagedEncryptionSettings: createRequest.ManagedEncryptionSettings, + CustomMaxRetentionHours: createRequest.CustomMaxRetentionHours, + Options: createRequest.Options, + Properties: createRequest.Properties, + FullName: createRequest.Name, + CreatedAt: nowMilli(), + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + Owner: s.CurrentUser().UserName, + CatalogType: catalog.CatalogTypeManagedCatalog, } catalogInfo.UpdatedAt = catalogInfo.CreatedAt if catalogInfo.Properties == nil && createRequest.Name == catalogNameManagedDefaults { From dfd55e2a4db4f1ffe6ae92944d1b3df77bc575b7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 30 Jun 2026 14:00:50 +0000 Subject: [PATCH 24/53] acceptance/fuzz: add redeploy, canonical, update, destroy_recreate invariants Broaden the fuzz invariant matrix beyond no_drift/migrate with four more schema-driven invariant bodies, each selectable via FUZZ_INVARIANT and following the existing INPUT_CONFIG_OK / SKIP_DRIFT_CHECK contract: - redeploy.sh: deploy twice; the second deploy must be a clean no-op, which exercises the write path twice and catches create handlers that don't round-trip their inputs. - canonical.sh: `bundle validate -o json` must be byte-identical across two runs; guards against nondeterministic serialization. Cloud-independent, so it always runs (not gated behind SKIP_DRIFT_CHECK). - update.sh: edit a comment/description and assert the redeploy is an in-place update (not a recreate) that converges with no drift. Configs without an editable field are skipped before the marker (treated as a rejection). - destroy_recreate.sh: deploy then destroy; a re-plan must want to create everything again, proving destroy left no orphaned state. Add two stdlib-only helpers: edit_fuzz_config.py (flips one comment/description scalar via a line match, no YAML dependency) and verify_plan_action.py (asserts a plan shows the expected action, mirroring bundle/deployplan/action.go). --- acceptance/bin/edit_fuzz_config.py | 57 ++++++++++++ acceptance/bin/verify_plan_action.py | 64 +++++++++++++ acceptance/bundle/invariant/README.md | 15 +++- acceptance/bundle/invariant/canonical.sh | 30 +++++++ .../bundle/invariant/destroy_recreate.sh | 76 ++++++++++++++++ .../bundle/invariant/fuzz/out.test.toml | 9 +- acceptance/bundle/invariant/fuzz/test.toml | 7 +- acceptance/bundle/invariant/redeploy.sh | 77 ++++++++++++++++ acceptance/bundle/invariant/update.sh | 90 +++++++++++++++++++ 9 files changed, 418 insertions(+), 7 deletions(-) create mode 100755 acceptance/bin/edit_fuzz_config.py create mode 100755 acceptance/bin/verify_plan_action.py create mode 100644 acceptance/bundle/invariant/canonical.sh create mode 100644 acceptance/bundle/invariant/destroy_recreate.sh create mode 100644 acceptance/bundle/invariant/redeploy.sh create mode 100644 acceptance/bundle/invariant/update.sh diff --git a/acceptance/bin/edit_fuzz_config.py b/acceptance/bin/edit_fuzz_config.py new file mode 100755 index 00000000000..ef7eefe605d --- /dev/null +++ b/acceptance/bin/edit_fuzz_config.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Edit one updatable field in a generated databricks.yml in place, for the `update` +invariant. It targets a `comment` or `description` scalar -- plain string fields the +update API accepts across resource types -- so a redeploy issues an in-place update +rather than a recreate. + +gen_fuzz_config.py emits every scalar on its own line as `key: `, so a line +match is enough and avoids a YAML dependency. + + edit_fuzz_config.py PATH edit in place; exit 1 if no editable field + edit_fuzz_config.py PATH --detect exit 0 if an editable field exists, else 1 +""" + +import argparse +import re +import sys + +# Allow an optional "- " so a comment/description that is the first key of a list-item +# dict still matches; the captured prefix is preserved verbatim on rewrite. +FIELD_RE = re.compile(r'^(\s*(?:- )?)(comment|description): (".*")\s*$') + +NEW_VALUE = '"fuzz_edited_value"' + + +def find_line(lines): + for i, line in enumerate(lines): + m = FIELD_RE.match(line) + if m: + return i, m + return -1, None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("path") + parser.add_argument("--detect", action="store_true", help="only check, don't edit") + args = parser.parse_args() + + with open(args.path) as f: + lines = f.readlines() + + i, m = find_line(lines) + if m is None: + sys.exit(1) + if args.detect: + return + + prefix, key, _ = m.groups() + lines[i] = f"{prefix}{key}: {NEW_VALUE}\n" + with open(args.path, "w") as f: + f.writelines(lines) + sys.stderr.write(f"edited {key} at line {i + 1}\n") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/verify_plan_action.py b/acceptance/bin/verify_plan_action.py new file mode 100755 index 00000000000..3b706066ba0 --- /dev/null +++ b/acceptance/bin/verify_plan_action.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Check that a `bundle plan -o json` shows an expected action, for invariants beyond +no-drift. + + verify_plan_action.py PATH update every changed resource is an in-place update + (not a recreate) and at least one changed + verify_plan_action.py PATH create every resource is a create (e.g. a re-plan + after destroy must recreate everything) + +Action vocabulary mirrors bundle/deployplan/action.go. +""" + +import json +import sys + +# update_id/resize keep the resource (no recreate), so they count as in-place updates. +ALLOWED = { + "update": {"update", "update_id", "resize"}, + "create": {"create"}, +} +# After a destroy, a "skip" means the resource survived (orphaned state), so skip is +# only tolerated for the update check, where unrelated siblings may be unchanged. +SKIP_OK = {"update": True, "create": False} + + +def main(): + path, expected = sys.argv[1], sys.argv[2] + allowed = ALLOWED[expected] + skip_ok = SKIP_OK[expected] + + with open(path) as fobj: + raw = fobj.read() + + if not raw.strip(): + sys.exit(f"{path}: empty plan output (bundle plan failed)") + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + sys.exit(f"{path}: invalid plan JSON: {e}\n{raw}") + + matched = 0 + bad = 0 + for key, value in data["plan"].items(): + action = value.get("action") + if action == "skip" and skip_ok: + continue + if action in allowed: + matched += 1 + else: + print(f"Unexpected {action=} for {key} (expected {expected})") + bad += 1 + + if matched == 0: + print(f"plan shows no {expected} action; expected at least one") + bad += 1 + + if bad: + print(raw, flush=True) + sys.exit(10) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 80cc095b9e1..92eabd5ef91 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -7,10 +7,17 @@ In order to add a new test, add a config to configs/ and include it in test.toml The fuzz/ test instead generates random configs from the live `databricks bundle schema` (see fuzz/script) and runs each one through a shared invariant body. The body -is selected by `FUZZ_INVARIANT` (matrixed in fuzz/test.toml) and is the same -`.sh` the matching curated test sources, so the fuzzer can exercise any -invariant: `no_drift.sh` (deploy + no drift) and `migrate.sh` (Terraform deploy + -migrate to direct + no drift) today. Since the schema comes from the CLI under test, +is selected by `FUZZ_INVARIANT` (matrixed in fuzz/test.toml) and is a `.sh` +body, so the fuzzer can exercise any invariant: + +- `no_drift.sh` -- deploy, then no drift +- `migrate.sh` -- Terraform deploy, migrate to direct, then no drift +- `redeploy.sh` -- deploy twice; the second deploy must be a no-op +- `canonical.sh` -- `validate -o json` must be byte-identical across two runs +- `update.sh` -- edit a comment/description; the redeploy must update in place (not recreate) +- `destroy_recreate.sh` -- deploy then destroy; a re-plan must recreate everything + +`no_drift.sh` and `migrate.sh` are also sourced by their matching curated tests. Since the schema comes from the CLI under test, an unrelated struct change can shift a seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), not flakiness; reproduce with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. diff --git a/acceptance/bundle/invariant/canonical.sh b/acceptance/bundle/invariant/canonical.sh new file mode 100644 index 00000000000..1cb9920d8fc --- /dev/null +++ b/acceptance/bundle/invariant/canonical.sh @@ -0,0 +1,30 @@ +# Shared invariant body: given a databricks.yml in the current directory, assert that +# `bundle validate -o json` is deterministic -- two runs on the same config must +# produce byte-identical output. Catches nondeterministic map ordering or other +# unstable serialization in config loading/resolution. There is no deploy, so no +# cleanup/destroy and no cloud state. Sourced by fuzz/script (random configs). + +$CLI bundle validate -o json > validate1.json 2>LOG.validate1.err +validate_rc=$? +cat LOG.validate1.err | contains.py '!panic' '!internal error' > /dev/null + +# A rejected config didn't validate; that's not a bug, just an invalid fuzz config, so +# skip the INPUT_CONFIG_OK marker. Curated tests run under `bash -e` and already +# aborted above, so this only fires in the fuzzer subshell. +if [ "$validate_rc" -ne 0 ]; then + return "$validate_rc" +fi + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +$CLI bundle validate -o json > validate2.json 2>LOG.validate2.err +cat LOG.validate2.err | contains.py '!panic' '!internal error' > /dev/null + +# Determinism is cloud-independent and cheap, so unlike drift it always runs (no +# SKIP_DRIFT_CHECK gate): identical input must yield identical output regardless of the +# seed window. A diff here is a real bug, not a fake-server limitation. +diff_rc=0 +diff validate1.json validate2.json > LOG.validate.diff || diff_rc=1 +return "$diff_rc" diff --git a/acceptance/bundle/invariant/destroy_recreate.sh b/acceptance/bundle/invariant/destroy_recreate.sh new file mode 100644 index 00000000000..6e805f33a5c --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate.sh @@ -0,0 +1,76 @@ +# Shared invariant body: given a databricks.yml in the current directory, deploy it, +# destroy it, and assert a re-plan wants to CREATE every resource again -- proving the +# destroy cleared all tracked state with nothing orphaned. A resource that destroy +# forgets to remove from state shows up here as a "skip" (still considered present), +# which is a bug. Sourced by fuzz/script (random configs). + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null + +cleanup() { + # Only destroy what we deployed. The body destroys on the happy path and clears + # `deployed`, so this trap only fires when deploy or destroy failed partway. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy_cleanup + cat LOG.destroy_cleanup | contains.py '!panic' '!internal error' > /dev/null + + # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard + # the lookup against the script's `set -u`. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err +cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null + +trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the +# fuzzer reads the destroy/recreate below as a bug. Curated tests run under `bash -e` +# and already aborted above, so this only fires in the fuzzer subshell. +if [ "$deploy_rc" -ne 0 ]; then + return "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# Destroy unconditionally so any panic lands in LOG.destroy for the harness post-scan; +# whether the destroy was complete (re-plan recreates everything) is gated below. +trace $CLI bundle destroy --auto-approve &> LOG.destroy +destroy_rc=$? +cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + +# On a clean destroy nothing remains, so stop the trap from destroying again (which +# would just make unstubbed API calls against the fake server). +if [ "$destroy_rc" -eq 0 ]; then + deployed="" +fi + +# A random fuzzed config can deploy yet legitimately leave fake-server state that the +# re-plan reads differently, so the fuzzer sets SKIP_DRIFT_CHECK on runs where only the +# no-panic invariant is asserted. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # The fuzzer runs this with errexit off and reads the return code, so accumulate + # failures into recreate_rc instead of letting the trailing no-panic check reset $?. + recreate_rc=0 + [ "$destroy_rc" -eq 0 ] || recreate_rc=1 + + $CLI bundle plan -o json > LOG.recreate_plan.json 2>LOG.recreate_plan.err + cat LOG.recreate_plan.err | contains.py '!panic' '!internal error' > /dev/null || recreate_rc=1 + verify_plan_action.py LOG.recreate_plan.json create || recreate_rc=1 + return "$recreate_rc" +fi diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index aa67f82bc28..611343d30c9 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -2,5 +2,12 @@ Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.FUZZ_INVARIANT = ["no_drift", "migrate"] +EnvMatrix.FUZZ_INVARIANT = [ + "no_drift", + "migrate", + "redeploy", + "canonical", + "update", + "destroy_recreate" +] EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index ef0f7b44382..4ca0c1adee4 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -3,5 +3,8 @@ EnvMatrix.INPUT_CONFIG = [] # Fuzz each invariant body in ../.sh. no_drift runs on the direct engine; -# migrate ignores it and starts from a Terraform deployment (see migrate.sh). -EnvMatrix.FUZZ_INVARIANT = ["no_drift", "migrate"] +# migrate ignores it and starts from a Terraform deployment (see migrate.sh). The +# others deploy on the direct engine and check a different property: redeploy is a +# no-op, canonical is determinism of `validate -o json`, update edits a field and +# expects an in-place update, destroy_recreate expects a re-plan to recreate everything. +EnvMatrix.FUZZ_INVARIANT = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] diff --git a/acceptance/bundle/invariant/redeploy.sh b/acceptance/bundle/invariant/redeploy.sh new file mode 100644 index 00000000000..3fe561f1d32 --- /dev/null +++ b/acceptance/bundle/invariant/redeploy.sh @@ -0,0 +1,77 @@ +# Shared invariant body: given a databricks.yml in the current directory, deploy it, +# then deploy it a SECOND time, and assert the redeploy is a clean no-op (no drift) +# with no panics / internal errors along the way. The distinguishing check vs no_drift +# is the second deploy: a create handler that doesn't round-trip its inputs (or a +# mutator that re-derives a field) surfaces here as a redeploy that wants to change or +# recreate an already-deployed resource. Sourced by fuzz/script (random configs). + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null + +cleanup() { + # Only destroy what we deployed. A curated config always deploys, but a random + # fuzzed config may be rejected, and destroying nothing just makes extra API + # calls (which fail the local fake server on unstubbed URLs). + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + + # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard + # the lookup against the script's `set -u`. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err +cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null + +trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the +# fuzzer reads the redeploy/drift below as a bug. Curated tests run under `bash -e` +# and already aborted above, so this only fires in the fuzzer subshell. +if [ "$deploy_rc" -ne 0 ]; then + return "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# Deploy again on the same config. Run it unconditionally so any panic lands in +# LOG.redeploy for the harness post-scan; whether it converges (success + no drift) is +# part of the drift-class check, gated below. +trace $CLI bundle deploy &> LOG.redeploy +redeploy_rc=$? +cat LOG.redeploy | contains.py '!panic' '!internal error' > /dev/null + +# A random fuzzed config can deploy yet legitimately fail to redeploy or differ from +# the fake server's state, so the fuzzer sets SKIP_DRIFT_CHECK on runs where only the +# no-panic invariant is asserted. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # The fuzzer runs this with errexit off and reads the return code, so accumulate + # failures into drift_rc instead of letting the trailing no-panic check reset $?. + drift_rc=0 + [ "$redeploy_rc" -eq 0 ] || drift_rc=1 + + # Check both text and JSON plan for no changes (may be >1 unchanged resource). + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + verify_no_drift.py LOG.planjson || drift_rc=1 + + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 + cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + return "$drift_rc" +fi diff --git a/acceptance/bundle/invariant/update.sh b/acceptance/bundle/invariant/update.sh new file mode 100644 index 00000000000..531115b5616 --- /dev/null +++ b/acceptance/bundle/invariant/update.sh @@ -0,0 +1,90 @@ +# Shared invariant body: given a databricks.yml in the current directory, deploy it, +# edit one updatable field (a comment/description), and assert the redeploy issues an +# in-place update -- not a recreate -- and leaves no drift. This exercises the update +# (PATCH) path that create-only deploys never touch; a resource whose update path is +# missing or buggy shows up here as a recreate, a spurious unrelated change, or drift. +# Sourced by fuzz/script (random configs). + +# The update invariant only applies to configs with an editable comment/description +# field. A random config without one isn't a bug, so skip it before deploying (no +# INPUT_CONFIG_OK marker, so the fuzzer treats it as a rejection). +if ! edit_fuzz_config.py databricks.yml --detect 2>LOG.detect.err; then + return 0 +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null + +cleanup() { + # Only destroy what we deployed. A curated config always deploys, but a random + # fuzzed config may be rejected, and destroying nothing just makes extra API + # calls (which fail the local fake server on unstubbed URLs). + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + + # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard + # the lookup against the script's `set -u`. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err +cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null + +trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the +# fuzzer reads the update/drift below as a bug. Curated tests run under `bash -e` and +# already aborted above, so this only fires in the fuzzer subshell. +if [ "$deploy_rc" -ne 0 ]; then + return "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# Change the comment/description and re-plan: this plan must show an in-place update. +edit_fuzz_config.py databricks.yml 2>LOG.edit.err +cat LOG.edit.err | contains.py '!Traceback' > /dev/null + +$CLI bundle plan -o json > LOG.update_plan.json 2>LOG.update_plan.err +cat LOG.update_plan.err | contains.py '!panic' '!internal error' > /dev/null + +# Apply the edit. Run it unconditionally so any panic lands in LOG.redeploy for the +# harness post-scan; whether the update is in-place and converges is gated below. +trace $CLI bundle deploy &> LOG.redeploy +redeploy_rc=$? +cat LOG.redeploy | contains.py '!panic' '!internal error' > /dev/null + +# A random fuzzed config can deploy yet legitimately differ from the fake server's +# state on update, so the fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic +# invariant is asserted. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # The fuzzer runs this with errexit off and reads the return code, so accumulate + # failures into update_rc instead of letting the trailing no-panic check reset $?. + update_rc=0 + [ "$redeploy_rc" -eq 0 ] || update_rc=1 + + # The edit must update in place, not recreate. + verify_plan_action.py LOG.update_plan.json update || update_rc=1 + + # And the applied update must converge: a re-plan shows no further changes. + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null || update_rc=1 + verify_no_drift.py LOG.planjson || update_rc=1 + return "$update_rc" +fi From c663329b5ed15168e44531cf952a259ed39f5bf3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 2 Jul 2026 12:54:11 +0000 Subject: [PATCH 25/53] acceptance/fuzz: extract shared invariant prologue and tighten comments Add common.sh with invariant_deploy and _invariant_cleanup so the deploy-based bodies (no_drift, redeploy, update, destroy_recreate) and migrate no longer duplicate the validate/deploy/cleanup prologue. Unify the panic check on '!panic:' across all invariant bodies and fuzz/script so a random generated token containing "panic" can't be a false positive. Also fold in the supporting helpers: check_schema_types.py (fail loud on a schema type the generator can't produce), gen_fuzz_config_check.py plus its selftest (to_yaml contract), util.load_plan shared by the verify_* scripts, and a pass to shorten comments across the harness. --- .github/workflows/push.yml | 4 +- acceptance/bin/check_schema_types.py | 48 ++++++++++++ acceptance/bin/edit_fuzz_config.py | 11 ++- acceptance/bin/gen_fuzz_config.py | 7 ++ acceptance/bin/gen_fuzz_config_check.py | 66 ++++++++++++++++ acceptance/bin/util.py | 13 ++++ acceptance/bin/verify_no_drift.py | 18 ++--- acceptance/bin/verify_plan_action.py | 16 ++-- acceptance/bundle/invariant/canonical.sh | 25 +++--- acceptance/bundle/invariant/common.sh | 49 ++++++++++++ .../bundle/invariant/destroy_recreate.sh | 71 ++++------------- acceptance/bundle/invariant/fuzz/script | 35 ++++----- acceptance/bundle/invariant/migrate.sh | 45 +++-------- acceptance/bundle/invariant/no_drift.sh | 66 +++------------- acceptance/bundle/invariant/redeploy.sh | 75 ++++-------------- acceptance/bundle/invariant/update.sh | 78 ++++--------------- .../selftest/gen_fuzz_config/out.test.toml | 3 + .../selftest/gen_fuzz_config/output.txt | 20 +++++ acceptance/selftest/gen_fuzz_config/script | 1 + 19 files changed, 325 insertions(+), 326 deletions(-) create mode 100755 acceptance/bin/check_schema_types.py create mode 100755 acceptance/bin/gen_fuzz_config_check.py create mode 100644 acceptance/bundle/invariant/common.sh create mode 100644 acceptance/selftest/gen_fuzz_config/out.test.toml create mode 100644 acceptance/selftest/gen_fuzz_config/output.txt create mode 100644 acceptance/selftest/gen_fuzz_config/script diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f30a6a96f05..8e80db2e225 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -427,10 +427,10 @@ jobs: - name: Run tests env: - # start = monotonic GITHUB_RUN_NUMBER * COUNT keeps each nightly window - # non-overlapping, so CI explores new configs every run. FUZZ_SEED_COUNT: "25" run: | + # start = monotonic GITHUB_RUN_NUMBER * COUNT keeps each nightly window + # non-overlapping, so CI explores new configs every run. export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) go tool -modfile=tools/task/go.mod task test-fuzz diff --git a/acceptance/bin/check_schema_types.py b/acceptance/bin/check_schema_types.py new file mode 100755 index 00000000000..6f9e72c0f8d --- /dev/null +++ b/acceptance/bin/check_schema_types.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +Assert every `type` in the bundle schema is one gen_fuzz_config.py can generate, so a new +libs/jsonschema.Type fails loudly here instead of being silently skipped by the fuzz loop. +""" + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import HANDLED_TYPES + + +def collect_types(node, found): + if isinstance(node, dict): + t = node.get("type") + if isinstance(t, str): + found.add(t) + elif isinstance(t, list): + found.update(x for x in t if isinstance(x, str)) + for v in node.values(): + collect_types(v, found) + elif isinstance(node, list): + for v in node: + collect_types(v, found) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--schema", required=True) + args = parser.parse_args() + + with open(args.schema) as f: + schema = json.load(f) + + found = set() + collect_types(schema, found) + + unhandled = sorted(found - HANDLED_TYPES) + if unhandled: + sys.exit(f"check_schema_types: gen_fuzz_config.py cannot generate schema types {unhandled}") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/edit_fuzz_config.py b/acceptance/bin/edit_fuzz_config.py index ef7eefe605d..3237a089e9c 100755 --- a/acceptance/bin/edit_fuzz_config.py +++ b/acceptance/bin/edit_fuzz_config.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 """ -Edit one updatable field in a generated databricks.yml in place, for the `update` -invariant. It targets a `comment` or `description` scalar -- plain string fields the -update API accepts across resource types -- so a redeploy issues an in-place update -rather than a recreate. +Edit a `comment`/`description` scalar in a generated databricks.yml so a redeploy is an +in-place update, not a recreate. Used by the `update` invariant. These fields are safe +to edit across resource types. -gen_fuzz_config.py emits every scalar on its own line as `key: `, so a line -match is enough and avoids a YAML dependency. +gen_fuzz_config.py emits one scalar per line as `key: `, so a regex match suffices +(no YAML dependency). edit_fuzz_config.py PATH edit in place; exit 1 if no editable field edit_fuzz_config.py PATH --detect exit 0 if an editable field exists, else 1 diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 1fa03f0fe29..390a96723f5 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -20,6 +20,10 @@ # bundle/internal/schema/main.go addInterpolationPatterns); we emit concrete values. INTERPOLATION_MARKER = "\\$\\{" +# Types the generator can produce; keep in sync with libs/jsonschema.Type. +SCALAR_TYPES = {"boolean", "integer", "number", "string"} +HANDLED_TYPES = SCALAR_TYPES | {"object", "array"} + class Generator: def __init__(self, schema, rng, unique): @@ -111,6 +115,9 @@ def gen_scalar(self, schema, name): return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) if t == "number": return round(self.rng.uniform(0, 1000), 2) + # Fail loud on an unknown type; a missing type is "any" and falls through to string. + if t is not None and t not in SCALAR_TYPES: + sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") # string (default) if name in ("name", "display_name"): return f"fuzz-{name}-{self.unique}" diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py new file mode 100755 index 00000000000..a54bf43b18c --- /dev/null +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Contract check for gen_fuzz_config.to_yaml: every scalar is on its own line as +`key: `. edit_fuzz_config.py relies on this to edit a field by regex, not a YAML +parser. Prints each case's YAML (diffed by the harness) and exits non-zero on a violation. +""" + +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from edit_fuzz_config import FIELD_RE +from gen_fuzz_config import to_yaml + +# Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, empty containers. +CASES = [ + {"comment": "value: with a colon", "description": 'quote " and : colon'}, + {"resources": {"jobs": {"j": {"name": "n", "tags": {"team": "jobs"}}}}}, + {"tasks": [{"description": "d", "timeout_seconds": 3600}, {"comment": "c"}]}, + {"nums": [0, 1, 2], "flag": True, "ratio": 1.5, "empty_map": {}, "empty_list": []}, +] + +HEADER = re.compile(r"[\w.\-]+:$") # non-empty container: `key:` +SCALAR = re.compile(r"[\w.\-]+: (.+)$") # `key: ` + + +def check_line(line): + rest = line.lstrip(" ") + rest = rest.removeprefix("- ") + if HEADER.fullmatch(rest): + return # container header; value is on following lines + m = SCALAR.fullmatch(rest) + if m: + json.loads(m.group(1)) # value must be single-line JSON + return + json.loads(rest) # bare list scalar: `- ` + + +def main(): + failed = False + for case in CASES: + text = to_yaml(case) + sys.stdout.write(text) + for line in text.splitlines(): + if not line.strip(): + continue + try: + check_line(line) + except ValueError: + sys.stderr.write(f"contract violation: not `key: `: {line!r}\n") + failed = True + + # edit_fuzz_config's FIELD_RE must still match when the value contains a colon (CASES[0]). + if not any(FIELD_RE.match(line) for line in to_yaml(CASES[0]).splitlines()): + sys.stderr.write("FIELD_RE did not match a comment/description line\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/util.py b/acceptance/bin/util.py index 3ee8d65bc90..da100803191 100644 --- a/acceptance/bin/util.py +++ b/acceptance/bin/util.py @@ -31,3 +31,16 @@ def run(cmd): if result.returncode != 0: raise RunError(f"{cmd} failed with code {result.returncode}") return result + + +def load_plan(path): + # Empty or invalid output means `bundle plan` failed; exit cleanly (no traceback) + # so the fuzzer treats it as a rejected config, not a bug. Returns (data, raw). + with open(path) as fobj: + raw = fobj.read() + if not raw.strip(): + sys.exit(f"{path}: empty plan output (bundle plan failed)") + try: + return json.loads(raw), raw + except json.JSONDecodeError as e: + sys.exit(f"{path}: invalid plan JSON: {e}\n{raw}") diff --git a/acceptance/bin/verify_no_drift.py b/acceptance/bin/verify_no_drift.py index 19d6ed28b87..3afb3e3dd9a 100755 --- a/acceptance/bin/verify_no_drift.py +++ b/acceptance/bin/verify_no_drift.py @@ -3,22 +3,16 @@ Check that all actions in plan are "skip". """ -import json +import os import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from util import load_plan + def check_plan(path): - with open(path) as fobj: - raw = fobj.read() - - # Empty or unparseable output means `bundle plan` itself failed; report that - # cleanly instead of crashing with a traceback. - if not raw.strip(): - sys.exit(f"{path}: empty plan output (bundle plan failed)") - try: - data = json.loads(raw) - except json.JSONDecodeError as e: - sys.exit(f"{path}: invalid plan JSON: {e}\n{raw}") + data, raw = load_plan(path) changes_detected = 0 for key, value in data["plan"].items(): diff --git a/acceptance/bin/verify_plan_action.py b/acceptance/bin/verify_plan_action.py index 3b706066ba0..d08c0113875 100755 --- a/acceptance/bin/verify_plan_action.py +++ b/acceptance/bin/verify_plan_action.py @@ -11,9 +11,13 @@ Action vocabulary mirrors bundle/deployplan/action.go. """ -import json +import os import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from util import load_plan + # update_id/resize keep the resource (no recreate), so they count as in-place updates. ALLOWED = { "update": {"update", "update_id", "resize"}, @@ -29,15 +33,7 @@ def main(): allowed = ALLOWED[expected] skip_ok = SKIP_OK[expected] - with open(path) as fobj: - raw = fobj.read() - - if not raw.strip(): - sys.exit(f"{path}: empty plan output (bundle plan failed)") - try: - data = json.loads(raw) - except json.JSONDecodeError as e: - sys.exit(f"{path}: invalid plan JSON: {e}\n{raw}") + data, raw = load_plan(path) matched = 0 bad = 0 diff --git a/acceptance/bundle/invariant/canonical.sh b/acceptance/bundle/invariant/canonical.sh index 1cb9920d8fc..e533c952635 100644 --- a/acceptance/bundle/invariant/canonical.sh +++ b/acceptance/bundle/invariant/canonical.sh @@ -1,30 +1,25 @@ -# Shared invariant body: given a databricks.yml in the current directory, assert that -# `bundle validate -o json` is deterministic -- two runs on the same config must -# produce byte-identical output. Catches nondeterministic map ordering or other -# unstable serialization in config loading/resolution. There is no deploy, so no -# cleanup/destroy and no cloud state. Sourced by fuzz/script (random configs). +# Shared invariant body: assert `bundle validate -o json` is deterministic -- two runs +# must be byte-identical. Catches unstable map ordering / serialization in config +# loading. No deploy, so no cleanup or cloud state. Sourced by fuzz/script. $CLI bundle validate -o json > validate1.json 2>LOG.validate1.err validate_rc=$? -cat LOG.validate1.err | contains.py '!panic' '!internal error' > /dev/null +cat LOG.validate1.err | contains.py '!panic:' '!internal error' > /dev/null -# A rejected config didn't validate; that's not a bug, just an invalid fuzz config, so -# skip the INPUT_CONFIG_OK marker. Curated tests run under `bash -e` and already -# aborted above, so this only fires in the fuzzer subshell. +# A config that fails to validate is an invalid fuzz config, not a bug, so skip the +# marker (curated tests already aborted above under `bash -e`). if [ "$validate_rc" -ne 0 ]; then return "$validate_rc" fi -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. +# Marks a good config for the fuzzer: any failure after this is a detected bug. echo INPUT_CONFIG_OK $CLI bundle validate -o json > validate2.json 2>LOG.validate2.err -cat LOG.validate2.err | contains.py '!panic' '!internal error' > /dev/null +cat LOG.validate2.err | contains.py '!panic:' '!internal error' > /dev/null -# Determinism is cloud-independent and cheap, so unlike drift it always runs (no -# SKIP_DRIFT_CHECK gate): identical input must yield identical output regardless of the -# seed window. A diff here is a real bug, not a fake-server limitation. +# Determinism is cloud-independent and cheap, so it always runs (no SKIP_DRIFT_CHECK +# gate): identical input must yield identical output. A diff is a real bug. diff_rc=0 diff validate1.json validate2.json > LOG.validate.diff || diff_rc=1 return "$diff_rc" diff --git a/acceptance/bundle/invariant/common.sh b/acceptance/bundle/invariant/common.sh new file mode 100644 index 00000000000..73132f31b4d --- /dev/null +++ b/acceptance/bundle/invariant/common.sh @@ -0,0 +1,49 @@ +# Shared prologue for the deploy-based invariant bodies (no_drift, redeploy, update, +# destroy_recreate). migrate reuses only the cleanup trap; canonical uses neither. + +_invariant_cleanup() { + # Destroy only what we deployed: a rejected fuzz config deployed nothing, and + # destroying nothing hits unstubbed URLs on the local fake server. + if [ -z "${deployed:-}" ]; then + return + fi + + # destroy_recreate destroys to LOG.destroy itself, so it points cleanup elsewhere. + trace $CLI bundle destroy --auto-approve &> "${CLEANUP_LOG:-LOG.destroy}" + cat "${CLEANUP_LOG:-LOG.destroy}" | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard + # the lookup against the script's `set -u`. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +# Validate and deploy databricks.yml. On success sets `deployed=1` and prints +# INPUT_CONFIG_OK; a rejected config leaves `deployed` unset with the code in +# `deploy_rc`. Call on a bare line (not in if/||) so `set -e` still aborts curated tests. +invariant_deploy() { + # We redirect output rather than record it because some configs that are being tested may produce warnings + trace $CLI bundle validate &> LOG.validate + cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + + trap _invariant_cleanup EXIT + + $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err + cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null + + trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy + deploy_rc=$? + cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null + + # A rejected config skips the marker below, so the fuzzer counts it as a rejection, + # not a bug (curated tests already aborted above under `bash -e`). + if [ "$deploy_rc" -ne 0 ]; then + return "$deploy_rc" + fi + deployed=1 + + # Marks a good config for the fuzzer: any failure after this is a detected bug. + echo INPUT_CONFIG_OK +} diff --git a/acceptance/bundle/invariant/destroy_recreate.sh b/acceptance/bundle/invariant/destroy_recreate.sh index 6e805f33a5c..3736291e65c 100644 --- a/acceptance/bundle/invariant/destroy_recreate.sh +++ b/acceptance/bundle/invariant/destroy_recreate.sh @@ -1,76 +1,37 @@ -# Shared invariant body: given a databricks.yml in the current directory, deploy it, -# destroy it, and assert a re-plan wants to CREATE every resource again -- proving the -# destroy cleared all tracked state with nothing orphaned. A resource that destroy -# forgets to remove from state shows up here as a "skip" (still considered present), -# which is a bug. Sourced by fuzz/script (random configs). +# Shared invariant body: deploy databricks.yml, destroy it, and assert a re-plan wants +# to CREATE everything again -- proving destroy cleared all tracked state. A resource +# destroy forgets shows up as "skip" (still present), a bug. Sourced by fuzz/script. -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate +source "$TESTDIR/../common.sh" -cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null +# This body destroys to LOG.destroy itself, so the cleanup trap must log elsewhere. +CLEANUP_LOG=LOG.destroy_cleanup -cleanup() { - # Only destroy what we deployed. The body destroys on the happy path and clears - # `deployed`, so this trap only fires when deploy or destroy failed partway. - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy_cleanup - cat LOG.destroy_cleanup | contains.py '!panic' '!internal error' > /dev/null - - # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard - # the lookup against the script's `set -u`. - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null - -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - -# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the -# fuzzer reads the destroy/recreate below as a bug. Curated tests run under `bash -e` -# and already aborted above, so this only fires in the fuzzer subshell. -if [ "$deploy_rc" -ne 0 ]; then +invariant_deploy +if [ -z "${deployed:-}" ]; then return "$deploy_rc" fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK -# Destroy unconditionally so any panic lands in LOG.destroy for the harness post-scan; -# whether the destroy was complete (re-plan recreates everything) is gated below. +# Destroy unconditionally so any panic lands in LOG.destroy for the post-scan; +# completeness (re-plan recreates everything) is gated below. trace $CLI bundle destroy --auto-approve &> LOG.destroy destroy_rc=$? -cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null +cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null -# On a clean destroy nothing remains, so stop the trap from destroying again (which -# would just make unstubbed API calls against the fake server). +# Clean destroy leaves nothing, so stop the trap from destroying again (unstubbed calls). if [ "$destroy_rc" -eq 0 ]; then deployed="" fi -# A random fuzzed config can deploy yet legitimately leave fake-server state that the -# re-plan reads differently, so the fuzzer sets SKIP_DRIFT_CHECK on runs where only the -# no-panic invariant is asserted. +# A fuzzed config can deploy yet legitimately leave state the re-plan reads differently, +# so the fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # The fuzzer runs this with errexit off and reads the return code, so accumulate - # failures into recreate_rc instead of letting the trailing no-panic check reset $?. + # errexit is off under the fuzzer; accumulate into recreate_rc so the trailing check can't reset $?. recreate_rc=0 [ "$destroy_rc" -eq 0 ] || recreate_rc=1 $CLI bundle plan -o json > LOG.recreate_plan.json 2>LOG.recreate_plan.err - cat LOG.recreate_plan.err | contains.py '!panic' '!internal error' > /dev/null || recreate_rc=1 + cat LOG.recreate_plan.err | contains.py '!panic:' '!internal error' > /dev/null || recreate_rc=1 verify_plan_action.py LOG.recreate_plan.json create || recreate_rc=1 return "$recreate_rc" fi diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 1d881073abc..579b8456d9f 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -1,17 +1,14 @@ -# Invariant: the CLI never panics on a schema-generated config, and a config that -# deploys cleanly has no drift. gen_fuzz_config.py produces a random schema-valid -# config; the invariant body ../$FUZZ_INVARIANT.sh (shared with the matching curated -# invariant test, e.g. no_drift or migrate) does the deploy/drift/destroy checks. -# Output goes to LOG.* so a violation fails via exit code, not diff, letting the same -# test run under any seed window [START, START+COUNT). +# Invariant: the CLI never panics on a schema-generated config, and a clean deploy has +# no drift. gen_fuzz_config.py produces a random config; the body ../$FUZZ_INVARIANT.sh +# (shared with the curated invariant tests) runs the deploy/drift/destroy checks. Output +# goes to LOG.* so a violation fails by exit code, letting the same test run under any +# seed window [START, START+COUNT). # -# A rejected config is not a bug: every invariant body prints INPUT_CONFIG_OK once a -# config deploys, so a non-zero result before that marker (no panic) is just a -# rejection, while a panic anywhere or a failure after it (drift, destroy) is a real bug. +# Rejection vs bug: bodies print INPUT_CONFIG_OK once a config deploys, so a non-zero +# result before the marker is a rejection; a panic anywhere, or a failure after it, is a bug. # -# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet -# legitimately differ from the fake server's state, so the committed run only asserts -# no-panic and skips the drift assertion. +# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet legitimately +# differ from the fake server, so the committed run asserts only no-panic. START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" @@ -25,7 +22,10 @@ export READPLAN="" # Emit the schema from the CLI under test so the generator always matches it. $CLI bundle schema > schema.json 2>LOG.schema.err -cat LOG.schema.err | contains.py '!panic' '!internal error' > /dev/null +cat LOG.schema.err | contains.py '!panic:' '!internal error' > /dev/null + +# Fail loud on a schema type the generator can't produce (the loop below would hide it). +check_schema_types.py --schema schema.json for ((offset = 0; offset < COUNT; offset++)); do seed=$((START + offset)) @@ -49,11 +49,10 @@ for ((offset = 0; offset < COUNT; offset++)); do bug="" - # A panic or internal error anywhere is a bug even when the CLI then rejects the - # config. Invariant bodies write different LOG.* files (no_drift has LOG.validate, - # migrate has LOG.migrate), so scan whatever this run produced rather than naming - # specific files -- a missing name would otherwise fail the pipe under pipefail. - if ! cat "$dir"/LOG.* 2>/dev/null | contains.py '!panic' '!internal error' > /dev/null; then + # A panic anywhere is a bug even if the CLI then rejects the config. Bodies write + # different LOG.* names, so scan them all rather than naming one (a missing name + # would fail the pipe under pipefail). + if ! cat "$dir"/LOG.* 2>/dev/null | contains.py '!panic:' '!internal error' > /dev/null; then bug=1 fi diff --git a/acceptance/bundle/invariant/migrate.sh b/acceptance/bundle/invariant/migrate.sh index 00f3948fc48..2a31562c8de 100644 --- a/acceptance/bundle/invariant/migrate.sh +++ b/acceptance/bundle/invariant/migrate.sh @@ -1,48 +1,27 @@ -# Shared invariant body: given a databricks.yml in the current directory, deploy it -# with Terraform, migrate the deployment to the direct engine, and assert there is no -# drift afterwards, with no panics / internal errors along the way. Sourced by -# migrate/script (curated configs) and fuzz/script (random schema-generated configs) -# so the deploy/migrate/drift logic lives in one place. +# Shared invariant body: deploy databricks.yml with Terraform, migrate to the direct +# engine, and assert no drift, no panics. Sourced by migrate/script (curated configs) +# and fuzz/script (random configs). # migrate always starts from a Terraform deployment, so drop any engine the caller # selected (the fuzzer runs the invariant matrix with DATABRICKS_BUNDLE_ENGINE=direct). unset DATABRICKS_BUNDLE_ENGINE -cleanup() { - # Only destroy what we deployed. A curated config always deploys, but a random - # fuzzed config may be rejected, and destroying nothing just makes extra API - # calls (which fail the local fake server on unstubbed URLs). - if [ -z "${deployed:-}" ]; then - return - fi +source "$TESTDIR/../common.sh" - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard - # the lookup against the script's `set -u`. - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT +trap _invariant_cleanup EXIT trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy deploy_rc=$? cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null -# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the -# fuzzer reads the failing migrate/drift below as a bug. Curated tests run under -# `bash -e` and already aborted above, so this only fires in the fuzzer subshell. +# A rejected config skips the marker below, so the fuzzer counts it as a rejection, not +# a bug (curated tests already aborted above under `bash -e`). if [ "$deploy_rc" -ne 0 ]; then return "$deploy_rc" fi deployed=1 -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. +# Marks a good config for the fuzzer: any failure after this is a detected bug. echo INPUT_CONFIG_OK MIGRATE_ARGS="" @@ -59,12 +38,10 @@ trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null -# Drift is the whole point for the curated migrate configs, but a random fuzzed -# config can migrate yet legitimately differ from the fake server's state, so the -# fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic invariant is asserted. +# A fuzzed config can migrate yet legitimately differ from the fake server, so the +# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # The fuzzer runs this with errexit off and reads the return code, so accumulate - # failures into drift_rc instead of letting the trailing no-panic check reset $?. + # errexit is off under the fuzzer; accumulate into drift_rc so the trailing check can't reset $?. drift_rc=0 $CLI bundle plan -o json > plan.json 2>plan.json.err cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 diff --git a/acceptance/bundle/invariant/no_drift.sh b/acceptance/bundle/invariant/no_drift.sh index df0bc319aa1..1498d488462 100644 --- a/acceptance/bundle/invariant/no_drift.sh +++ b/acceptance/bundle/invariant/no_drift.sh @@ -1,66 +1,24 @@ -# Shared invariant body: given a databricks.yml in the current directory, deploy it -# and assert there is no drift afterwards, with no panics / internal errors along -# the way. Sourced by no_drift/script (curated configs) and fuzz/script (random -# schema-generated configs) so the deploy/drift/destroy logic lives in one place. +# Shared invariant body: deploy databricks.yml and assert no drift, no panics. Sourced +# by no_drift/script (curated configs) and fuzz/script (random configs). -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate +source "$TESTDIR/../common.sh" -cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null - -cleanup() { - # Only destroy what we deployed. A curated config always deploys, but a random - # fuzzed config may be rejected, and destroying nothing just makes extra API - # calls (which fail the local fake server on unstubbed URLs). - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - - # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard - # the lookup against the script's `set -u`. - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null - -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - -# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise -# the fuzzer reads the re-plan's "needs create" as drift. Curated tests run under -# `bash -e` and already aborted above, so this only fires in the fuzzer subshell. -if [ "$deploy_rc" -ne 0 ]; then +invariant_deploy +if [ -z "${deployed:-}" ]; then return "$deploy_rc" fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK -# Drift is the whole point for the curated no_drift configs, but a random fuzzed -# config can deploy yet legitimately differ from the fake server's state, so the -# fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic invariant is asserted. +# A fuzzed config can deploy yet legitimately differ from the fake server, so the +# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # Check both text and JSON plan for no changes (may be >1 unchanged resource). - # The fuzzer runs this with errexit off and reads the return code, so accumulate - # failures into drift_rc instead of letting the trailing no-panic check reset $?. + # Check both text and JSON plan for no changes. errexit is off under the fuzzer, so + # accumulate into drift_rc; the trailing no-panic check must not reset $?. drift_rc=0 $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 verify_no_drift.py LOG.planjson || drift_rc=1 - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 - cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 + cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 return "$drift_rc" fi diff --git a/acceptance/bundle/invariant/redeploy.sh b/acceptance/bundle/invariant/redeploy.sh index 3fe561f1d32..37754974f91 100644 --- a/acceptance/bundle/invariant/redeploy.sh +++ b/acceptance/bundle/invariant/redeploy.sh @@ -1,77 +1,34 @@ -# Shared invariant body: given a databricks.yml in the current directory, deploy it, -# then deploy it a SECOND time, and assert the redeploy is a clean no-op (no drift) -# with no panics / internal errors along the way. The distinguishing check vs no_drift -# is the second deploy: a create handler that doesn't round-trip its inputs (or a -# mutator that re-derives a field) surfaces here as a redeploy that wants to change or -# recreate an already-deployed resource. Sourced by fuzz/script (random configs). +# Shared invariant body: deploy databricks.yml, then deploy again and assert the second +# deploy is a clean no-op. Catches create handlers that don't round-trip their inputs +# (or mutators that re-derive a field), which surface as a redeploy wanting to change or +# recreate. Sourced by fuzz/script (random configs). -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate +source "$TESTDIR/../common.sh" -cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null - -cleanup() { - # Only destroy what we deployed. A curated config always deploys, but a random - # fuzzed config may be rejected, and destroying nothing just makes extra API - # calls (which fail the local fake server on unstubbed URLs). - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - - # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard - # the lookup against the script's `set -u`. - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null - -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - -# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the -# fuzzer reads the redeploy/drift below as a bug. Curated tests run under `bash -e` -# and already aborted above, so this only fires in the fuzzer subshell. -if [ "$deploy_rc" -ne 0 ]; then +invariant_deploy +if [ -z "${deployed:-}" ]; then return "$deploy_rc" fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK -# Deploy again on the same config. Run it unconditionally so any panic lands in -# LOG.redeploy for the harness post-scan; whether it converges (success + no drift) is -# part of the drift-class check, gated below. +# Deploy again, unconditionally, so any panic lands in LOG.redeploy for the post-scan; +# convergence (success + no drift) is gated below. trace $CLI bundle deploy &> LOG.redeploy redeploy_rc=$? -cat LOG.redeploy | contains.py '!panic' '!internal error' > /dev/null +cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null -# A random fuzzed config can deploy yet legitimately fail to redeploy or differ from -# the fake server's state, so the fuzzer sets SKIP_DRIFT_CHECK on runs where only the -# no-panic invariant is asserted. +# A fuzzed config can deploy yet legitimately fail to redeploy or differ, so the fuzzer +# sets SKIP_DRIFT_CHECK to assert only no-panic. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # The fuzzer runs this with errexit off and reads the return code, so accumulate - # failures into drift_rc instead of letting the trailing no-panic check reset $?. + # errexit is off under the fuzzer; accumulate into drift_rc so the trailing check can't reset $?. drift_rc=0 [ "$redeploy_rc" -eq 0 ] || drift_rc=1 # Check both text and JSON plan for no changes (may be >1 unchanged resource). $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 verify_no_drift.py LOG.planjson || drift_rc=1 - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 - cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null || drift_rc=1 + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 + cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 return "$drift_rc" fi diff --git a/acceptance/bundle/invariant/update.sh b/acceptance/bundle/invariant/update.sh index 531115b5616..69e74c4c844 100644 --- a/acceptance/bundle/invariant/update.sh +++ b/acceptance/bundle/invariant/update.sh @@ -1,81 +1,37 @@ -# Shared invariant body: given a databricks.yml in the current directory, deploy it, -# edit one updatable field (a comment/description), and assert the redeploy issues an -# in-place update -- not a recreate -- and leaves no drift. This exercises the update -# (PATCH) path that create-only deploys never touch; a resource whose update path is -# missing or buggy shows up here as a recreate, a spurious unrelated change, or drift. -# Sourced by fuzz/script (random configs). +# Shared invariant body: deploy databricks.yml, edit a comment/description, and assert +# the redeploy is an in-place update, not a recreate, with no drift. Exercises the +# update (PATCH) path create-only deploys never touch. Sourced by fuzz/script. -# The update invariant only applies to configs with an editable comment/description -# field. A random config without one isn't a bug, so skip it before deploying (no -# INPUT_CONFIG_OK marker, so the fuzzer treats it as a rejection). +source "$TESTDIR/../common.sh" + +# Only configs with an editable comment/description apply here; skip others before +# deploying (no marker, so the fuzzer treats it as a rejection, not a bug). if ! edit_fuzz_config.py databricks.yml --detect 2>LOG.detect.err; then return 0 fi -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate - -cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null - -cleanup() { - # Only destroy what we deployed. A curated config always deploys, but a random - # fuzzed config may be rejected, and destroying nothing just makes extra API - # calls (which fail the local fake server on unstubbed URLs). - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - - # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard - # the lookup against the script's `set -u`. - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null - -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - -# A rejected config didn't deploy, so skip the INPUT_CONFIG_OK marker; otherwise the -# fuzzer reads the update/drift below as a bug. Curated tests run under `bash -e` and -# already aborted above, so this only fires in the fuzzer subshell. -if [ "$deploy_rc" -ne 0 ]; then +invariant_deploy +if [ -z "${deployed:-}" ]; then return "$deploy_rc" fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK # Change the comment/description and re-plan: this plan must show an in-place update. edit_fuzz_config.py databricks.yml 2>LOG.edit.err cat LOG.edit.err | contains.py '!Traceback' > /dev/null $CLI bundle plan -o json > LOG.update_plan.json 2>LOG.update_plan.err -cat LOG.update_plan.err | contains.py '!panic' '!internal error' > /dev/null +cat LOG.update_plan.err | contains.py '!panic:' '!internal error' > /dev/null -# Apply the edit. Run it unconditionally so any panic lands in LOG.redeploy for the -# harness post-scan; whether the update is in-place and converges is gated below. +# Apply the edit, unconditionally, so any panic lands in LOG.redeploy for the post-scan; +# in-place update and convergence are gated below. trace $CLI bundle deploy &> LOG.redeploy redeploy_rc=$? -cat LOG.redeploy | contains.py '!panic' '!internal error' > /dev/null +cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null -# A random fuzzed config can deploy yet legitimately differ from the fake server's -# state on update, so the fuzzer sets SKIP_DRIFT_CHECK on runs where only the no-panic -# invariant is asserted. +# A fuzzed config can deploy yet legitimately differ on update, so the fuzzer sets +# SKIP_DRIFT_CHECK to assert only no-panic. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # The fuzzer runs this with errexit off and reads the return code, so accumulate - # failures into update_rc instead of letting the trailing no-panic check reset $?. + # errexit is off under the fuzzer; accumulate into update_rc so the trailing check can't reset $?. update_rc=0 [ "$redeploy_rc" -eq 0 ] || update_rc=1 @@ -84,7 +40,7 @@ if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then # And the applied update must converge: a re-plan shows no further changes. $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null || update_rc=1 + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null || update_rc=1 verify_no_drift.py LOG.planjson || update_rc=1 return "$update_rc" fi diff --git a/acceptance/selftest/gen_fuzz_config/out.test.toml b/acceptance/selftest/gen_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/gen_fuzz_config/output.txt b/acceptance/selftest/gen_fuzz_config/output.txt new file mode 100644 index 00000000000..75c5a658de4 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/output.txt @@ -0,0 +1,20 @@ +comment: "value: with a colon" +description: "quote \" and : colon" +resources: + jobs: + j: + name: "n" + tags: + team: "jobs" +tasks: + - description: "d" + timeout_seconds: 3600 + - comment: "c" +nums: + - 0 + - 1 + - 2 +flag: true +ratio: 1.5 +empty_map: {} +empty_list: [] diff --git a/acceptance/selftest/gen_fuzz_config/script b/acceptance/selftest/gen_fuzz_config/script new file mode 100644 index 00000000000..2737c67674d --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/script @@ -0,0 +1 @@ +gen_fuzz_config_check.py From dab9d30a1425f43614fbc11031b9187fdc078f69 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 2 Jul 2026 13:06:44 +0000 Subject: [PATCH 26/53] testserver: move catalog round-trip comment above the added fields The comment describes connection_name, managed_encryption_settings and custom_max_retention_hours, which are the fields the fix added. It sat above StorageRoot/ProviderName/ShareName, which were already round-tripped. Move it directly above the three added fields so it annotates them. --- libs/testserver/catalogs.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 0c86d113c8b..26a5740a3b5 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -25,14 +25,14 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } catalogInfo := catalog.CatalogInfo{ - Name: createRequest.Name, - Comment: createRequest.Comment, - // Round-trip the remaining create-request fields so a re-read matches the - // deployed config. Dropping them made connection_name (recreate_on_changes) - // re-plan as a perpetual recreate and managed_encryption_settings as drift. - StorageRoot: createRequest.StorageRoot, - ProviderName: createRequest.ProviderName, - ShareName: createRequest.ShareName, + Name: createRequest.Name, + Comment: createRequest.Comment, + StorageRoot: createRequest.StorageRoot, + ProviderName: createRequest.ProviderName, + ShareName: createRequest.ShareName, + // Round-trip these so a re-read matches the deployed config: connection_name is + // recreate_on_changes (immutable), so dropping it re-planned as a perpetual + // recreate, and managed_encryption_settings showed as drift. ConnectionName: createRequest.ConnectionName, ManagedEncryptionSettings: createRequest.ManagedEncryptionSettings, CustomMaxRetentionHours: createRequest.CustomMaxRetentionHours, From 8f9b3fa5aeb832a3b4dcc6270bf15d7ddc2b35d3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 6 Jul 2026 13:46:53 +0000 Subject: [PATCH 27/53] Restructure invariant fuzz tests into per-target scripts Replace the shared `.sh` invariant bodies (sourced via FUZZ_INVARIANT and common.sh) with self-contained invariant test directories that double as fuzz targets, selected by FUZZ_TARGET. Each target runs over the curated INPUT_CONFIG matrix and, when FUZZ_SEED is set, against a schema-generated random config. - Inline the deploy/drift logic into no_drift/script and migrate/script - Re-add redeploy, canonical, update, and destroy_recreate as invariant dirs - Drop common.sh and the standalone *.sh bodies - Point fuzz/script at ../$FUZZ_TARGET/script and refresh the README --- acceptance/bundle/invariant/README.md | 28 +++--- acceptance/bundle/invariant/canonical.sh | 25 ------ .../bundle/invariant/canonical/out.test.toml | 54 ++++++++++++ .../bundle/invariant/canonical/output.txt | 1 + acceptance/bundle/invariant/canonical/script | 43 ++++++++++ acceptance/bundle/invariant/common.sh | 49 ----------- .../bundle/invariant/destroy_recreate.sh | 37 -------- .../invariant/destroy_recreate/out.test.toml | 54 ++++++++++++ .../invariant/destroy_recreate/output.txt | 1 + .../bundle/invariant/destroy_recreate/script | 85 +++++++++++++++++++ .../bundle/invariant/fuzz/out.test.toml | 2 +- acceptance/bundle/invariant/fuzz/script | 28 +++--- acceptance/bundle/invariant/fuzz/test.toml | 9 +- acceptance/bundle/invariant/migrate.sh | 50 ----------- acceptance/bundle/invariant/migrate/script | 75 +++++++++++++--- acceptance/bundle/invariant/no_drift.sh | 24 ------ acceptance/bundle/invariant/no_drift/script | 74 +++++++++++++--- acceptance/bundle/invariant/redeploy.sh | 34 -------- .../bundle/invariant/redeploy/out.test.toml | 54 ++++++++++++ .../bundle/invariant/redeploy/output.txt | 1 + acceptance/bundle/invariant/redeploy/script | 74 ++++++++++++++++ acceptance/bundle/invariant/update.sh | 46 ---------- .../bundle/invariant/update/out.test.toml | 54 ++++++++++++ acceptance/bundle/invariant/update/output.txt | 1 + acceptance/bundle/invariant/update/script | 84 ++++++++++++++++++ 25 files changed, 664 insertions(+), 323 deletions(-) delete mode 100644 acceptance/bundle/invariant/canonical.sh create mode 100644 acceptance/bundle/invariant/canonical/out.test.toml create mode 100644 acceptance/bundle/invariant/canonical/output.txt create mode 100644 acceptance/bundle/invariant/canonical/script delete mode 100644 acceptance/bundle/invariant/common.sh delete mode 100644 acceptance/bundle/invariant/destroy_recreate.sh create mode 100644 acceptance/bundle/invariant/destroy_recreate/out.test.toml create mode 100644 acceptance/bundle/invariant/destroy_recreate/output.txt create mode 100644 acceptance/bundle/invariant/destroy_recreate/script delete mode 100644 acceptance/bundle/invariant/migrate.sh delete mode 100644 acceptance/bundle/invariant/no_drift.sh delete mode 100644 acceptance/bundle/invariant/redeploy.sh create mode 100644 acceptance/bundle/invariant/redeploy/out.test.toml create mode 100644 acceptance/bundle/invariant/redeploy/output.txt create mode 100644 acceptance/bundle/invariant/redeploy/script delete mode 100644 acceptance/bundle/invariant/update.sh create mode 100644 acceptance/bundle/invariant/update/out.test.toml create mode 100644 acceptance/bundle/invariant/update/output.txt create mode 100644 acceptance/bundle/invariant/update/script diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 92eabd5ef91..38ea67a1b5a 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -5,19 +5,19 @@ test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. -The fuzz/ test instead generates random configs from the live `databricks bundle -schema` (see fuzz/script) and runs each one through a shared invariant body. The body -is selected by `FUZZ_INVARIANT` (matrixed in fuzz/test.toml) and is a `.sh` -body, so the fuzzer can exercise any invariant: +The fuzz/ test generates random configs from the live `databricks bundle schema` +(see fuzz/script) and runs each one through a real invariant test script. The target is +selected by `FUZZ_TARGET` (matrixed in fuzz/test.toml); each target is also a curated +invariant test that runs over the `INPUT_CONFIG` matrix: -- `no_drift.sh` -- deploy, then no drift -- `migrate.sh` -- Terraform deploy, migrate to direct, then no drift -- `redeploy.sh` -- deploy twice; the second deploy must be a no-op -- `canonical.sh` -- `validate -o json` must be byte-identical across two runs -- `update.sh` -- edit a comment/description; the redeploy must update in place (not recreate) -- `destroy_recreate.sh` -- deploy then destroy; a re-plan must recreate everything +- `no_drift` -- deploy, then no drift +- `migrate` -- Terraform deploy, migrate to direct, then no drift +- `redeploy` -- deploy twice; the second deploy must be a no-op +- `canonical` -- `validate -o json` must be byte-identical across two runs +- `update` -- edit a comment/description; the redeploy must update in place (not recreate) +- `destroy_recreate` -- deploy then destroy; a re-plan must recreate everything -`no_drift.sh` and `migrate.sh` are also sourced by their matching curated tests. Since the schema comes from the CLI under test, -an unrelated struct change can shift a seed onto a new config. A failure is a real CLI -bug (panic, internal error, or drift), not flakiness; reproduce with -`FUZZ_SEED_START= FUZZ_SEED_COUNT=1 task test-fuzz`. +Since the schema comes from the CLI under test, an unrelated struct change can shift a +seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), +not flakiness; reproduce with +`FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_TARGET=no_drift task test-fuzz`. diff --git a/acceptance/bundle/invariant/canonical.sh b/acceptance/bundle/invariant/canonical.sh deleted file mode 100644 index e533c952635..00000000000 --- a/acceptance/bundle/invariant/canonical.sh +++ /dev/null @@ -1,25 +0,0 @@ -# Shared invariant body: assert `bundle validate -o json` is deterministic -- two runs -# must be byte-identical. Catches unstable map ordering / serialization in config -# loading. No deploy, so no cleanup or cloud state. Sourced by fuzz/script. - -$CLI bundle validate -o json > validate1.json 2>LOG.validate1.err -validate_rc=$? -cat LOG.validate1.err | contains.py '!panic:' '!internal error' > /dev/null - -# A config that fails to validate is an invalid fuzz config, not a bug, so skip the -# marker (curated tests already aborted above under `bash -e`). -if [ "$validate_rc" -ne 0 ]; then - return "$validate_rc" -fi - -# Marks a good config for the fuzzer: any failure after this is a detected bug. -echo INPUT_CONFIG_OK - -$CLI bundle validate -o json > validate2.json 2>LOG.validate2.err -cat LOG.validate2.err | contains.py '!panic:' '!internal error' > /dev/null - -# Determinism is cloud-independent and cheap, so it always runs (no SKIP_DRIFT_CHECK -# gate): identical input must yield identical output. A diff is a real bug. -diff_rc=0 -diff validate1.json validate2.json > LOG.validate.diff || diff_rc=1 -return "$diff_rc" diff --git a/acceptance/bundle/invariant/canonical/out.test.toml b/acceptance/bundle/invariant/canonical/out.test.toml new file mode 100644 index 00000000000..4c1c45e02e2 --- /dev/null +++ b/acceptance/bundle/invariant/canonical/out.test.toml @@ -0,0 +1,54 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/canonical/output.txt b/acceptance/bundle/invariant/canonical/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/canonical/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/canonical/script b/acceptance/bundle/invariant/canonical/script new file mode 100644 index 00000000000..d31f11622c2 --- /dev/null +++ b/acceptance/bundle/invariant/canonical/script @@ -0,0 +1,43 @@ +# Invariant to test: `bundle validate -o json` is deterministic -- two runs must be +# byte-identical. Catches unstable map ordering / serialization in config loading. +# No deploy, so no cleanup or cloud state. + +if [ -n "${FUZZ_SEED:-}" ]; then + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +$CLI bundle validate -o json > validate1.json 2>LOG.validate1.err +validate_rc=$? +cat LOG.validate1.err | contains.py '!panic:' '!internal error' > /dev/null + +# A config that fails to validate is an invalid fuzz config, not a bug, so stop before +# the marker (curated tests already aborted above under `bash -e`). +if [ "$validate_rc" -ne 0 ]; then + exit "$validate_rc" +fi + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +$CLI bundle validate -o json > validate2.json 2>LOG.validate2.err +cat LOG.validate2.err | contains.py '!panic:' '!internal error' > /dev/null + +# Determinism is cloud-independent and cheap, so it always runs (no SKIP_DRIFT_CHECK +# gate): identical input must yield identical output. A diff is a real bug. +diff validate1.json validate2.json > LOG.validate.diff diff --git a/acceptance/bundle/invariant/common.sh b/acceptance/bundle/invariant/common.sh deleted file mode 100644 index 73132f31b4d..00000000000 --- a/acceptance/bundle/invariant/common.sh +++ /dev/null @@ -1,49 +0,0 @@ -# Shared prologue for the deploy-based invariant bodies (no_drift, redeploy, update, -# destroy_recreate). migrate reuses only the cleanup trap; canonical uses neither. - -_invariant_cleanup() { - # Destroy only what we deployed: a rejected fuzz config deployed nothing, and - # destroying nothing hits unstubbed URLs on the local fake server. - if [ -z "${deployed:-}" ]; then - return - fi - - # destroy_recreate destroys to LOG.destroy itself, so it points cleanup elsewhere. - trace $CLI bundle destroy --auto-approve &> "${CLEANUP_LOG:-LOG.destroy}" - cat "${CLEANUP_LOG:-LOG.destroy}" | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present. The fuzzer has no named INPUT_CONFIG, so guard - # the lookup against the script's `set -u`. - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -# Validate and deploy databricks.yml. On success sets `deployed=1` and prints -# INPUT_CONFIG_OK; a rejected config leaves `deployed` unset with the code in -# `deploy_rc`. Call on a bare line (not in if/||) so `set -e` still aborts curated tests. -invariant_deploy() { - # We redirect output rather than record it because some configs that are being tested may produce warnings - trace $CLI bundle validate &> LOG.validate - cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null - - trap _invariant_cleanup EXIT - - $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err - cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null - - trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy - deploy_rc=$? - cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null - - # A rejected config skips the marker below, so the fuzzer counts it as a rejection, - # not a bug (curated tests already aborted above under `bash -e`). - if [ "$deploy_rc" -ne 0 ]; then - return "$deploy_rc" - fi - deployed=1 - - # Marks a good config for the fuzzer: any failure after this is a detected bug. - echo INPUT_CONFIG_OK -} diff --git a/acceptance/bundle/invariant/destroy_recreate.sh b/acceptance/bundle/invariant/destroy_recreate.sh deleted file mode 100644 index 3736291e65c..00000000000 --- a/acceptance/bundle/invariant/destroy_recreate.sh +++ /dev/null @@ -1,37 +0,0 @@ -# Shared invariant body: deploy databricks.yml, destroy it, and assert a re-plan wants -# to CREATE everything again -- proving destroy cleared all tracked state. A resource -# destroy forgets shows up as "skip" (still present), a bug. Sourced by fuzz/script. - -source "$TESTDIR/../common.sh" - -# This body destroys to LOG.destroy itself, so the cleanup trap must log elsewhere. -CLEANUP_LOG=LOG.destroy_cleanup - -invariant_deploy -if [ -z "${deployed:-}" ]; then - return "$deploy_rc" -fi - -# Destroy unconditionally so any panic lands in LOG.destroy for the post-scan; -# completeness (re-plan recreates everything) is gated below. -trace $CLI bundle destroy --auto-approve &> LOG.destroy -destroy_rc=$? -cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - -# Clean destroy leaves nothing, so stop the trap from destroying again (unstubbed calls). -if [ "$destroy_rc" -eq 0 ]; then - deployed="" -fi - -# A fuzzed config can deploy yet legitimately leave state the re-plan reads differently, -# so the fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # errexit is off under the fuzzer; accumulate into recreate_rc so the trailing check can't reset $?. - recreate_rc=0 - [ "$destroy_rc" -eq 0 ] || recreate_rc=1 - - $CLI bundle plan -o json > LOG.recreate_plan.json 2>LOG.recreate_plan.err - cat LOG.recreate_plan.err | contains.py '!panic:' '!internal error' > /dev/null || recreate_rc=1 - verify_plan_action.py LOG.recreate_plan.json create || recreate_rc=1 - return "$recreate_rc" -fi diff --git a/acceptance/bundle/invariant/destroy_recreate/out.test.toml b/acceptance/bundle/invariant/destroy_recreate/out.test.toml new file mode 100644 index 00000000000..4c1c45e02e2 --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate/out.test.toml @@ -0,0 +1,54 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/destroy_recreate/output.txt b/acceptance/bundle/invariant/destroy_recreate/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/destroy_recreate/script b/acceptance/bundle/invariant/destroy_recreate/script new file mode 100644 index 00000000000..2f3f7ed288d --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate/script @@ -0,0 +1,85 @@ +# Invariant to test: after deploy then destroy, a re-plan wants to CREATE everything +# again -- proving destroy cleared all tracked state. A resource destroy forgets shows +# up as "skip" (still present), a bug. +# Additional checks: no internal errors / panics in validate/plan/deploy/destroy + +if [ -n "${FUZZ_SEED:-}" ]; then + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + # This test destroys to LOG.destroy itself, so the trap logs elsewhere to keep + # the body's destroy output (and any panic) intact for the post-run scan. + trace $CLI bundle destroy --auto-approve &> LOG.destroy_cleanup + cat LOG.destroy_cleanup | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# Destroy unconditionally so any panic lands in LOG.destroy for the post-scan; +# completeness (re-plan recreates everything) is gated below. +trace $CLI bundle destroy --auto-approve &> LOG.destroy +destroy_rc=$? +cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + +# A clean destroy leaves nothing, so stop the cleanup trap from destroying again +# (which would hit unstubbed URLs on the fake server). +if [ "$destroy_rc" -eq 0 ]; then + deployed="" +fi + +# A fuzzed config can deploy yet legitimately leave state a re-plan reads differently, +# so the fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check +# that a re-plan recreates everything. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + if [ "$destroy_rc" -ne 0 ]; then + exit "$destroy_rc" + fi + + $CLI bundle plan -o json > LOG.recreate_plan.json 2>LOG.recreate_plan.err + cat LOG.recreate_plan.err | contains.py '!panic:' '!internal error' > /dev/null + verify_plan_action.py LOG.recreate_plan.json create +fi diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index 611343d30c9..b60e7ec13dc 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -2,7 +2,7 @@ Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.FUZZ_INVARIANT = [ +EnvMatrix.FUZZ_TARGET = [ "no_drift", "migrate", "redeploy", diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 579b8456d9f..e8377aea705 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -1,11 +1,7 @@ -# Invariant: the CLI never panics on a schema-generated config, and a clean deploy has -# no drift. gen_fuzz_config.py produces a random config; the body ../$FUZZ_INVARIANT.sh -# (shared with the curated invariant tests) runs the deploy/drift/destroy checks. Output -# goes to LOG.* so a violation fails by exit code, letting the same test run under any -# seed window [START, START+COUNT). -# -# Rejection vs bug: bodies print INPUT_CONFIG_OK once a config deploys, so a non-zero -# result before the marker is a rejection; a panic anywhere, or a failure after it, is a bug. +# Invariant fuzzing: generate a random config per seed and run the real invariant test +# script (no_drift/script or migrate/script). Rejection vs bug: those scripts print +# INPUT_CONFIG_OK once a config deploys, so a non-zero result before the marker is a +# rejection; a panic anywhere, or a failure after it, is a bug. # # Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet legitimately # differ from the fake server, so the committed run asserts only no-panic. @@ -17,7 +13,7 @@ if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then export SKIP_DRIFT_CHECK=1 fi -# no_drift.sh reads READPLAN via readplanarg; the fuzzer skips the saved-plan matrix. +# no_drift/script reads READPLAN via readplanarg; the fuzzer skips the saved-plan matrix. export READPLAN="" # Emit the schema from the CLI under test so the generator always matches it. @@ -32,13 +28,13 @@ for ((offset = 0; offset < COUNT; offset++)); do dir="seed-$seed" mkdir -p "$dir" - # Subshell so a generator crash or shared-check failure is contained per seed. + # Subshell so a generator crash or invariant failure is contained per seed. set +e ( cd "$dir" - gen_fuzz_config.py --schema ../schema.json --seed "$seed" --unique "$UNIQUE_NAME-$seed" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - source "$TESTDIR/../${FUZZ_INVARIANT:-no_drift}.sh" + export FUZZ_SEED="$seed" + export FUZZ_SCHEMA="../schema.json" + source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" ) > "$dir/LOG.check" 2>&1 rc=$? set -e @@ -49,9 +45,7 @@ for ((offset = 0; offset < COUNT; offset++)); do bug="" - # A panic anywhere is a bug even if the CLI then rejects the config. Bodies write - # different LOG.* names, so scan them all rather than naming one (a missing name - # would fail the pipe under pipefail). + # A panic anywhere is a bug even if the CLI then rejects the config. if ! cat "$dir"/LOG.* 2>/dev/null | contains.py '!panic:' '!internal error' > /dev/null; then bug=1 fi @@ -63,7 +57,7 @@ for ((offset = 0; offset < COUNT; offset++)); do fi if [ -n "$bug" ]; then - echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 task test-fuzz" >&2 + echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} task test-fuzz" >&2 exit 1 fi done diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 4ca0c1adee4..872ce9abfba 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -2,9 +2,6 @@ # generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] -# Fuzz each invariant body in ../.sh. no_drift runs on the direct engine; -# migrate ignores it and starts from a Terraform deployment (see migrate.sh). The -# others deploy on the direct engine and check a different property: redeploy is a -# no-op, canonical is determinism of `validate -o json`, update edits a field and -# expects an in-place update, destroy_recreate expects a re-plan to recreate everything. -EnvMatrix.FUZZ_INVARIANT = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] +# Run the real invariant test script for each target. migrate ignores +# DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] diff --git a/acceptance/bundle/invariant/migrate.sh b/acceptance/bundle/invariant/migrate.sh deleted file mode 100644 index 2a31562c8de..00000000000 --- a/acceptance/bundle/invariant/migrate.sh +++ /dev/null @@ -1,50 +0,0 @@ -# Shared invariant body: deploy databricks.yml with Terraform, migrate to the direct -# engine, and assert no drift, no panics. Sourced by migrate/script (curated configs) -# and fuzz/script (random configs). - -# migrate always starts from a Terraform deployment, so drop any engine the caller -# selected (the fuzzer runs the invariant matrix with DATABRICKS_BUNDLE_ENGINE=direct). -unset DATABRICKS_BUNDLE_ENGINE - -source "$TESTDIR/../common.sh" - -trap _invariant_cleanup EXIT - -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null - -# A rejected config skips the marker below, so the fuzzer counts it as a rejection, not -# a bug (curated tests already aborted above under `bash -e`). -if [ "$deploy_rc" -ne 0 ]; then - return "$deploy_rc" -fi -deployed=1 - -# Marks a good config for the fuzzer: any failure after this is a detected bug. -echo INPUT_CONFIG_OK - -MIGRATE_ARGS="" -# The terraform provider sorts depends_on entries alphabetically by task_key on Read -# (see terraform-provider-databricks PR #3000). Since depends_on uses TypeList -# (order-sensitive), terraform plan reports positional drift when the bundle config -# specifies depends_on in a different order than the provider's sorted state. -# This is a false positive -- the logical dependencies are identical. -if [[ "${INPUT_CONFIG:-}" == "job_with_depends_on.yml.tmpl" ]]; then - MIGRATE_ARGS="--noplancheck" -fi - -trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate - -cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null - -# A fuzzed config can migrate yet legitimately differ from the fake server, so the -# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # errexit is off under the fuzzer; accumulate into drift_rc so the trailing check can't reset $?. - drift_rc=0 - $CLI bundle plan -o json > plan.json 2>plan.json.err - cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 - verify_no_drift.py plan.json || drift_rc=1 - return "$drift_rc" -fi diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 78eb0630e15..cc484fb79da 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -1,18 +1,73 @@ # Invariant to test: migrate is successful, no drift after deploy # Additional checks: no internal errors / panics in any commands -# Copy data files to test directory -cp -r "$TESTDIR/../data/." . &> LOG.cp +unset DATABRICKS_BUNDLE_ENGINE -# Run init script if present -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init +if [ -n "${FUZZ_SEED:-}" ]; then + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config fi -envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null -cp databricks.yml LOG.config + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 -# Migrate then assert no drift. Shared with the fuzz invariant test. -source "$TESTDIR/../migrate.sh" +echo INPUT_CONFIG_OK + +MIGRATE_ARGS="" +# The terraform provider sorts depends_on entries alphabetically by task_key on Read +# (see terraform-provider-databricks PR #3000). Since depends_on uses TypeList +# (order-sensitive), terraform plan reports positional drift when the bundle config +# specifies depends_on in a different order than the provider's sorted state. +# This is a false positive -- the logical dependencies are identical. +if [[ "${INPUT_CONFIG:-}" == "job_with_depends_on.yml.tmpl" ]]; then + MIGRATE_ARGS="--noplancheck" +fi + +trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate + +cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null + +# A fuzzed config can migrate yet legitimately differ from the fake server, so the +# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + $CLI bundle plan -o json > plan.json 2>plan.json.err + cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py plan.json +fi diff --git a/acceptance/bundle/invariant/no_drift.sh b/acceptance/bundle/invariant/no_drift.sh deleted file mode 100644 index 1498d488462..00000000000 --- a/acceptance/bundle/invariant/no_drift.sh +++ /dev/null @@ -1,24 +0,0 @@ -# Shared invariant body: deploy databricks.yml and assert no drift, no panics. Sourced -# by no_drift/script (curated configs) and fuzz/script (random configs). - -source "$TESTDIR/../common.sh" - -invariant_deploy -if [ -z "${deployed:-}" ]; then - return "$deploy_rc" -fi - -# A fuzzed config can deploy yet legitimately differ from the fake server, so the -# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # Check both text and JSON plan for no changes. errexit is off under the fuzzer, so - # accumulate into drift_rc; the trailing no-panic check must not reset $?. - drift_rc=0 - $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 - verify_no_drift.py LOG.planjson || drift_rc=1 - - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 - cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 - return "$drift_rc" -fi diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 1edd686409a..a12fb972fe0 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -1,18 +1,72 @@ # Invariant to test: no drift after deploy # Additional checks: no internal errors / panics in validate/plan/deploy -# Copy data files to test directory -cp -r "$TESTDIR/../data/." . &> LOG.cp +if [ -n "${FUZZ_SEED:-}" ]; then + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp -# Run init script if present -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err +cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null + +trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" fi +deployed=1 -envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK -cp databricks.yml LOG.config +# A fuzzed config can deploy yet legitimately differ from the fake server, so the +# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # Check both text and JSON plan for no changes + # Note, expect that there maybe more than one resource unchanged + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson -# Deploy and assert no drift. Shared with the fuzz invariant test. -source "$TESTDIR/../no_drift.sh" + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan + cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null +fi diff --git a/acceptance/bundle/invariant/redeploy.sh b/acceptance/bundle/invariant/redeploy.sh deleted file mode 100644 index 37754974f91..00000000000 --- a/acceptance/bundle/invariant/redeploy.sh +++ /dev/null @@ -1,34 +0,0 @@ -# Shared invariant body: deploy databricks.yml, then deploy again and assert the second -# deploy is a clean no-op. Catches create handlers that don't round-trip their inputs -# (or mutators that re-derive a field), which surface as a redeploy wanting to change or -# recreate. Sourced by fuzz/script (random configs). - -source "$TESTDIR/../common.sh" - -invariant_deploy -if [ -z "${deployed:-}" ]; then - return "$deploy_rc" -fi - -# Deploy again, unconditionally, so any panic lands in LOG.redeploy for the post-scan; -# convergence (success + no drift) is gated below. -trace $CLI bundle deploy &> LOG.redeploy -redeploy_rc=$? -cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null - -# A fuzzed config can deploy yet legitimately fail to redeploy or differ, so the fuzzer -# sets SKIP_DRIFT_CHECK to assert only no-panic. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # errexit is off under the fuzzer; accumulate into drift_rc so the trailing check can't reset $?. - drift_rc=0 - [ "$redeploy_rc" -eq 0 ] || drift_rc=1 - - # Check both text and JSON plan for no changes (may be >1 unchanged resource). - $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 - verify_no_drift.py LOG.planjson || drift_rc=1 - - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan || drift_rc=1 - cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null || drift_rc=1 - return "$drift_rc" -fi diff --git a/acceptance/bundle/invariant/redeploy/out.test.toml b/acceptance/bundle/invariant/redeploy/out.test.toml new file mode 100644 index 00000000000..4c1c45e02e2 --- /dev/null +++ b/acceptance/bundle/invariant/redeploy/out.test.toml @@ -0,0 +1,54 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/redeploy/output.txt b/acceptance/bundle/invariant/redeploy/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/redeploy/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/redeploy/script b/acceptance/bundle/invariant/redeploy/script new file mode 100644 index 00000000000..42d9cebcd5a --- /dev/null +++ b/acceptance/bundle/invariant/redeploy/script @@ -0,0 +1,74 @@ +# Invariant to test: a second deploy of an unchanged config is a clean no-op +# Additional checks: no internal errors / panics in validate/plan/deploy +# +# Catches create handlers that don't round-trip their inputs (or mutators that +# re-derive a field), which surface as a redeploy wanting to change or recreate. + +if [ -n "${FUZZ_SEED:-}" ]; then + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# A fuzzed config can deploy yet legitimately fail to redeploy or differ, so the fuzzer +# sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check the no-op. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + trace $CLI bundle deploy &> LOG.redeploy + cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null + + # Check both text and JSON plan for no changes + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson + + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan + cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null +fi diff --git a/acceptance/bundle/invariant/update.sh b/acceptance/bundle/invariant/update.sh deleted file mode 100644 index 69e74c4c844..00000000000 --- a/acceptance/bundle/invariant/update.sh +++ /dev/null @@ -1,46 +0,0 @@ -# Shared invariant body: deploy databricks.yml, edit a comment/description, and assert -# the redeploy is an in-place update, not a recreate, with no drift. Exercises the -# update (PATCH) path create-only deploys never touch. Sourced by fuzz/script. - -source "$TESTDIR/../common.sh" - -# Only configs with an editable comment/description apply here; skip others before -# deploying (no marker, so the fuzzer treats it as a rejection, not a bug). -if ! edit_fuzz_config.py databricks.yml --detect 2>LOG.detect.err; then - return 0 -fi - -invariant_deploy -if [ -z "${deployed:-}" ]; then - return "$deploy_rc" -fi - -# Change the comment/description and re-plan: this plan must show an in-place update. -edit_fuzz_config.py databricks.yml 2>LOG.edit.err -cat LOG.edit.err | contains.py '!Traceback' > /dev/null - -$CLI bundle plan -o json > LOG.update_plan.json 2>LOG.update_plan.err -cat LOG.update_plan.err | contains.py '!panic:' '!internal error' > /dev/null - -# Apply the edit, unconditionally, so any panic lands in LOG.redeploy for the post-scan; -# in-place update and convergence are gated below. -trace $CLI bundle deploy &> LOG.redeploy -redeploy_rc=$? -cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null - -# A fuzzed config can deploy yet legitimately differ on update, so the fuzzer sets -# SKIP_DRIFT_CHECK to assert only no-panic. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # errexit is off under the fuzzer; accumulate into update_rc so the trailing check can't reset $?. - update_rc=0 - [ "$redeploy_rc" -eq 0 ] || update_rc=1 - - # The edit must update in place, not recreate. - verify_plan_action.py LOG.update_plan.json update || update_rc=1 - - # And the applied update must converge: a re-plan shows no further changes. - $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null || update_rc=1 - verify_no_drift.py LOG.planjson || update_rc=1 - return "$update_rc" -fi diff --git a/acceptance/bundle/invariant/update/out.test.toml b/acceptance/bundle/invariant/update/out.test.toml new file mode 100644 index 00000000000..4c1c45e02e2 --- /dev/null +++ b/acceptance/bundle/invariant/update/out.test.toml @@ -0,0 +1,54 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/update/output.txt b/acceptance/bundle/invariant/update/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/update/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/update/script b/acceptance/bundle/invariant/update/script new file mode 100644 index 00000000000..e1b2c28acf8 --- /dev/null +++ b/acceptance/bundle/invariant/update/script @@ -0,0 +1,84 @@ +# Invariant to test: editing a comment/description redeploys as an in-place update, not +# a recreate, and converges. Exercises the update (PATCH) path create-only deploys never +# touch. +# Additional checks: no internal errors / panics in validate/plan/deploy + +if [ -n "${FUZZ_SEED:-}" ]; then + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# A fuzzed config can deploy yet legitimately differ on update, so the fuzzer sets +# SKIP_DRIFT_CHECK to assert only no-panic; curated configs check the update path. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # Only configs with an editable comment/description exercise the update path; + # others just verify the deploy above (edit_fuzz_config.py --detect exits 1). + if edit_fuzz_config.py databricks.yml --detect 2>LOG.detect.err; then + # Change the comment/description; the re-plan must show an in-place update. + edit_fuzz_config.py databricks.yml 2>LOG.edit.err + cat LOG.edit.err | contains.py '!Traceback' > /dev/null + + $CLI bundle plan -o json > LOG.update_plan.json 2>LOG.update_plan.err + cat LOG.update_plan.err | contains.py '!panic:' '!internal error' > /dev/null + + trace $CLI bundle deploy &> LOG.redeploy + cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null + + # The edit must update in place, not recreate. + verify_plan_action.py LOG.update_plan.json update + + # And the applied update must converge: a re-plan shows no further changes. + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson + fi +fi From 49813ed72f08d992a8994baedec0e9659ae8d7fe Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 7 Jul 2026 09:05:23 +0000 Subject: [PATCH 28/53] invariant: regenerate out.test.toml for volume_path_job_ref The origin/main merge added the volume_path_job_ref.yml.tmpl fuzz template to the INPUT_CONFIG matrix, but the canonical/destroy_recreate/redeploy/update out.test.toml goldens were not regenerated, failing the "changed or new files" CI guard. --- acceptance/bundle/invariant/canonical/out.test.toml | 1 + acceptance/bundle/invariant/destroy_recreate/out.test.toml | 1 + acceptance/bundle/invariant/redeploy/out.test.toml | 1 + acceptance/bundle/invariant/update/out.test.toml | 1 + 4 files changed, 4 insertions(+) diff --git a/acceptance/bundle/invariant/canonical/out.test.toml b/acceptance/bundle/invariant/canonical/out.test.toml index 4c1c45e02e2..35535594224 100644 --- a/acceptance/bundle/invariant/canonical/out.test.toml +++ b/acceptance/bundle/invariant/canonical/out.test.toml @@ -50,5 +50,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/destroy_recreate/out.test.toml b/acceptance/bundle/invariant/destroy_recreate/out.test.toml index 4c1c45e02e2..35535594224 100644 --- a/acceptance/bundle/invariant/destroy_recreate/out.test.toml +++ b/acceptance/bundle/invariant/destroy_recreate/out.test.toml @@ -50,5 +50,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/redeploy/out.test.toml b/acceptance/bundle/invariant/redeploy/out.test.toml index 4c1c45e02e2..35535594224 100644 --- a/acceptance/bundle/invariant/redeploy/out.test.toml +++ b/acceptance/bundle/invariant/redeploy/out.test.toml @@ -50,5 +50,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/update/out.test.toml b/acceptance/bundle/invariant/update/out.test.toml index 4c1c45e02e2..35535594224 100644 --- a/acceptance/bundle/invariant/update/out.test.toml +++ b/acceptance/bundle/invariant/update/out.test.toml @@ -50,5 +50,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] From ba5309a70c55a562bc2b76d3ca1264d4a820b11d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 7 Jul 2026 09:05:40 +0000 Subject: [PATCH 29/53] fuzz: pin catalog/schema references and emit valid grants The generator produced random catalog_name/schema_name values and random or empty grants. The fake test server accepts them, but real UC rejects them (CATALOG_DOES_NOT_EXIST, invalid principal/privilege), so such configs deploy locally yet drift or fail on cloud, masking real invariant coverage. Pin catalog_name to "main" and schema_name to "default" (the seeded objects used by the curated invariant configs), and emit one known-good grant per grant-bearing securable type ("account users" plus a privilege valid for that type). This removes the spurious grants drift and the reference-rejection class. --- acceptance/bin/gen_fuzz_config.py | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 390a96723f5..177a242ded1 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -24,12 +24,38 @@ SCALAR_TYPES = {"boolean", "integer", "number", "string"} HANDLED_TYPES = SCALAR_TYPES | {"object", "array"} +# Cross-resource references must resolve to objects that exist on every +# workspace (the fake test server and real UC alike). "main"/"default" are the +# standard seeded catalog/schema; these mirror acceptance/bundle/invariant/configs. +# Without pinning, the generator emits random names that the fake server accepts +# but real UC rejects (e.g. CATALOG_DOES_NOT_EXIST), so the config is dropped at +# deploy and never exercises the invariant. +DEFAULT_CATALOG = "main" +DEFAULT_SCHEMA = "default" + +# "account users" is a group present on every workspace, plus one privilege UC +# accepts for each grant-bearing securable type (from the curated configs). Real +# UC rejects an unknown principal or a privilege that doesn't apply to the +# securable, so a random grant would deploy on the fake server yet fail on cloud. +DEFAULT_PRINCIPAL = "account users" +GRANT_PRIVILEGE = { + "catalogs": "USE_CATALOG", + "schemas": "USE_SCHEMA", + "volumes": "READ_VOLUME", + "registered_models": "EXECUTE", + "external_locations": "READ_FILES", + "vector_search_indexes": "SELECT", +} + class Generator: def __init__(self, schema, rng, unique): self.root = schema self.rng = rng self.unique = unique + # Set to the top-level resource type before generating its element, so + # grants can pick a privilege valid for that securable. + self.rtype = None def resolve(self, schema): # Follow $ref chains, e.g. "#/$defs/github.com/.../resources.Job", nested @@ -54,6 +80,9 @@ def gen(self, schema, depth, name=""): if not isinstance(schema, dict) or not schema: return self.gen_scalar({"type": "string"}, name) + if name == "grants": + return self.gen_grants() + if "const" in schema: return schema["const"] if schema.get("enum"): @@ -103,6 +132,14 @@ def gen_array(self, schema, depth, name): return [] return [self.gen(items, depth + 1, name) for _ in range(self.rng.randint(1, 3))] + def gen_grants(self): + # One known-good grant for the current securable. Skip grants for a type + # we have no valid privilege for, rather than emit one real UC rejects. + privilege = GRANT_PRIVILEGE.get(self.rtype) + if privilege is None: + return [] + return [{"principal": DEFAULT_PRINCIPAL, "privileges": [privilege]}] + def gen_scalar(self, schema, name): t = schema.get("type") if t == "boolean": @@ -119,6 +156,11 @@ def gen_scalar(self, schema, name): if t is not None and t not in SCALAR_TYPES: sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") # string (default) + # Pin cross-resource references to seeded defaults (see constants above). + if name == "catalog_name": + return DEFAULT_CATALOG + if name == "schema_name": + return DEFAULT_SCHEMA if name in ("name", "display_name"): return f"fuzz-{name}-{self.unique}" return self.token() @@ -151,6 +193,7 @@ def gen_config(schema, seed, unique, allowed): element = obj["additionalProperties"] key = f"fuzz_{rtype}_{seed}" + gen.rtype = rtype instance = gen.gen(element, 0) return { "bundle": {"name": f"fuzz-{unique}"}, From 15b4ea72eb46f109de763348c242fd7a94ca4b90 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 7 Jul 2026 19:56:08 +0000 Subject: [PATCH 30/53] fuzz: generate multiple resources with cross-references Extend the invariant fuzzer to emit more than one resource per config (--resource-count, matrixed as FUZZ_RESOURCE_COUNT) and link two of them with a ${resources.*} reference so the interpolation and deploy-ordering paths are exercised. The reference targets an input identity field (name/display_name) so it resolves for every resource type and converges without drift. Also improve generated-config validity: skip output-only/computed fields (x-databricks-field-behaviors OUTPUT_ONLY, readOnly, and a name list) to avoid false drift after migrate, emit valid permissions per resource type, force prevent_destroy=false so destroy_recreate can run, and add required fields the schema omits for registered_models. --- acceptance/bin/gen_fuzz_config.py | 195 ++++++++++++++++-- acceptance/bin/gen_fuzz_config_check.py | 64 +++++- acceptance/bundle/invariant/README.md | 6 +- acceptance/bundle/invariant/canonical/script | 2 +- .../bundle/invariant/destroy_recreate/script | 2 +- .../bundle/invariant/fuzz/out.test.toml | 1 + acceptance/bundle/invariant/fuzz/test.toml | 1 + acceptance/bundle/invariant/migrate/script | 2 +- acceptance/bundle/invariant/no_drift/script | 2 +- acceptance/bundle/invariant/redeploy/script | 2 +- acceptance/bundle/invariant/update/script | 2 +- 11 files changed, 256 insertions(+), 23 deletions(-) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 177a242ded1..825a7e1bff0 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -3,9 +3,11 @@ Generate a random bundle config from the bundle JSON schema. Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf -branches) and emits one random resource as databricks.yml, seeded by --seed. Feeds the -invariant tests; the harness filters out configs the CLI rejects, so output may be -structurally-random but sometimes invalid. +branches) and emits one or more random resources as databricks.yml, seeded by --seed. +With --resource-count > 1 it also links two resources with a ${resources.*} reference so +the interpolation and deploy-ordering machinery is exercised. Feeds the invariant tests; +the harness filters out configs the CLI rejects, so output may be structurally-random but +sometimes invalid. """ import argparse @@ -47,6 +49,51 @@ "vector_search_indexes": "SELECT", } +# Permissions blocks cannot be variable references; each entry needs a concrete +# principal and a level valid for the resource type (see invariant configs). +DEFAULT_PERMISSION_GROUP = "users" +PERMISSION_LEVEL = { + "alerts": "CAN_MANAGE", + "apps": "CAN_USE", + "clusters": "CAN_ATTACH_TO", + "dashboards": "CAN_READ", + "database_instances": "CAN_USE", + "experiments": "CAN_READ", + "genie_spaces": "CAN_READ", + "jobs": "CAN_VIEW", + "model_serving_endpoints": "CAN_VIEW", + "models": "CAN_READ", + "pipelines": "CAN_VIEW", + "postgres_projects": "CAN_USE", + "secret_scopes": "READ", + "sql_warehouses": "CAN_VIEW", + "vector_search_endpoints": "CAN_USE", +} + +# Fields the bundle schema still lists but the user never sets (backend output / +# computed). Emitting them causes false drift after terraform→direct migrate. +# Keep in sync with bundle/direct/dresources/resources.yml output_only and +# backend_defaults where the field is not user-writable. +SKIP_PROPERTY_NAMES = frozenset( + { + "browse_only", + "created_at", + "created_by", + "creator_name", + "full_name", + "metastore_id", + "owner", + "storage_location", + "updated_at", + "updated_by", + } +) + +# Resource types whose schema omits required[] but need these fields to deploy. +RESOURCE_REQUIRED_FIELDS = { + "registered_models": frozenset({"catalog_name", "name", "schema_name"}), +} + class Generator: def __init__(self, schema, rng, unique): @@ -75,6 +122,25 @@ def choose_branch(self, branches): concrete = [b for b in branches if not self.is_interpolation(b)] return self.rng.choice(concrete or branches) + def field_behaviors(self, schema): + if not isinstance(schema, dict): + return [] + resolved = self.resolve(schema) + behaviors = list(schema.get("x-databricks-field-behaviors", [])) + if resolved is not schema: + behaviors.extend(resolved.get("x-databricks-field-behaviors", [])) + return behaviors + + def should_skip_property(self, prop_name, prop_schema): + if prop_name in SKIP_PROPERTY_NAMES: + return True + resolved = self.resolve(prop_schema) + if "OUTPUT_ONLY" in self.field_behaviors(prop_schema): + return True + if resolved.get("readOnly"): + return True + return False + def gen(self, schema, depth, name=""): schema = self.resolve(schema) if not isinstance(schema, dict) or not schema: @@ -82,6 +148,8 @@ def gen(self, schema, depth, name=""): if name == "grants": return self.gen_grants() + if name == "permissions": + return self.gen_permissions() if "const" in schema: return schema["const"] @@ -105,9 +173,13 @@ def is_map(self, schema): def gen_object(self, schema, depth): props = schema.get("properties", {}) required = set(schema.get("required", [])) + if depth == 0 and self.rtype: + required |= RESOURCE_REQUIRED_FIELDS.get(self.rtype, set()) result = {} for prop_name, prop_schema in props.items(): + if self.should_skip_property(prop_name, prop_schema): + continue # Always emit required fields; emit optional ones less often as we go # deeper to keep configs from exploding. keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) @@ -140,9 +212,20 @@ def gen_grants(self): return [] return [{"principal": DEFAULT_PRINCIPAL, "privileges": [privilege]}] + def gen_permissions(self): + # One known-good permission for the current resource. Skip types we have + # no valid level for, rather than emit a random principal or ${...} ref. + level = PERMISSION_LEVEL.get(self.rtype) + if level is None: + return [] + return [{"level": level, "group_name": DEFAULT_PERMISSION_GROUP}] + def gen_scalar(self, schema, name): t = schema.get("type") if t == "boolean": + # destroy_recreate invariant requires destroy to succeed. + if name == "prevent_destroy": + return False return self.rng.choice([True, False]) if t == "integer": # The field is in hours, but UC validates it as a window of 0 or 7-30 @@ -176,15 +259,8 @@ def resource_types(schema, gen): return obj["properties"] -def gen_config(schema, seed, unique, allowed): - rng = random.Random(seed) - gen = Generator(schema, rng, unique) - - types = resource_types(schema, gen) - candidates = [t for t in types if not allowed or t in allowed] - if not candidates: - sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") - rtype = rng.choice(sorted(candidates)) +def gen_resource(schema, gen, types, candidates, seed, unique, index, resource_count): + rtype = gen.rng.choice(sorted(candidates)) # Each resource type is a map ref; the element schema is the object branch's # additionalProperties. @@ -192,12 +268,95 @@ def gen_config(schema, seed, unique, allowed): obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") element = obj["additionalProperties"] - key = f"fuzz_{rtype}_{seed}" + if resource_count == 1: + key = f"fuzz_{rtype}_{seed}" + gen.unique = unique + else: + key = f"fuzz_{rtype}_{seed}_{index}" + gen.unique = f"{unique}-{index}" gen.rtype = rtype instance = gen.gen(element, 0) + return rtype, key, instance, gen.resolve(element) + + +def object_properties(gen, schema): + # The resource element is oneOf[object, ${...} string]; return the object + # branch's properties, matching the branch gen() picks to build the instance. + schema = gen.resolve(schema) + if "properties" in schema: + return schema["properties"] + for key in ("oneOf", "anyOf"): + for branch in schema.get(key, []): + resolved = gen.resolve(branch) + if "properties" in resolved: + return resolved["properties"] + return {} + + +def cross_ref_field(gen, element): + # A free-text scalar safe to overwrite with a reference; both names cover most + # resource types (jobs use "description", UC resources use "comment"). + props = object_properties(gen, element) + for field in ("description", "comment"): + if field in props: + return field + return None + + +def target_ref_field(instance): + # Reference the target's identity field: a string, so the type stays compatible + # with the description/comment field it lands in, and an input (not an output like + # ".id") so it resolves for every resource type and converges without drift. + # Output-field references are covered by the curated cross-ref configs. + for field in ("name", "display_name"): + if isinstance(instance.get(field), str): + return field + return None + + +def inject_cross_ref(gen, records): + # Link two resources so deploy has to order them and resolve the reference. + if len(records) < 2: + return + sources = [r for r in records if r["ref_field"]] + gen.rng.shuffle(sources) + for source in sources: + targets = [t for t in records if t["key"] != source["key"] and target_ref_field(t["instance"])] + if not targets: + continue + target = gen.rng.choice(targets) + field = target_ref_field(target["instance"]) + source["instance"][source["ref_field"]] = f"${{resources.{target['rtype']}.{target['key']}.{field}}}" + return + + +def gen_config(schema, seed, unique, allowed, resource_count=1): + if resource_count < 1: + sys.exit(f"gen_fuzz_config: --resource-count must be >= 1, got {resource_count}") + + gen = Generator(schema, random.Random(seed), unique) + + types = resource_types(schema, gen) + candidates = [t for t in types if not allowed or t in allowed] + if not candidates: + sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") + + records = [] + for index in range(resource_count): + rtype, key, instance, element = gen_resource( + schema, gen, types, candidates, seed, unique, index, resource_count + ) + records.append({"rtype": rtype, "key": key, "instance": instance, "ref_field": cross_ref_field(gen, element)}) + + inject_cross_ref(gen, records) + + resources = {} + for record in records: + resources.setdefault(record["rtype"], {})[record["key"]] = record["instance"] + return { "bundle": {"name": f"fuzz-{unique}"}, - "resources": {rtype: {key: instance}}, + "resources": resources, } @@ -240,13 +399,19 @@ def main(): default="", help="Comma-separated allow-list of resource types (default: all)", ) + parser.add_argument( + "--resource-count", + type=int, + default=1, + help="Number of resources to emit (default: 1)", + ) args = parser.parse_args() with open(args.schema) as f: schema = json.load(f) allowed = {r.strip() for r in args.resources.split(",") if r.strip()} - config = gen_config(schema, args.seed, args.unique, allowed) + config = gen_config(schema, args.seed, args.unique, allowed, args.resource_count) sys.stdout.write(to_yaml(config)) diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py index a54bf43b18c..f707144148a 100755 --- a/acceptance/bin/gen_fuzz_config_check.py +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from edit_fuzz_config import FIELD_RE -from gen_fuzz_config import to_yaml +from gen_fuzz_config import SKIP_PROPERTY_NAMES, gen_config, to_yaml # Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, empty containers. CASES = [ @@ -58,6 +58,68 @@ def main(): sys.stderr.write("FIELD_RE did not match a comment/description line\n") failed = True + # Multi-resource configs merge types under resources., and one resource + # references another's name so the interpolation/ordering path is exercised. + def resource_type(field): + return { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": {"name": {"type": "string"}, field: {"type": "string"}}, + "required": ["name"], + }, + } + ] + } + + multi = gen_config( + { + "$defs": {}, + "properties": { + "resources": { + "oneOf": [ + { + "type": "object", + "properties": { + "jobs": resource_type("description"), + "volumes": resource_type("comment"), + }, + } + ] + } + }, + }, + seed=42, + unique="check", + allowed=set(), + resource_count=2, + ) + if len(multi["resources"]) < 1 or sum(len(v) for v in multi["resources"].values()) != 2: + sys.stderr.write("gen_config did not emit two resources\n") + failed = True + + values = [v for insts in multi["resources"].values() for inst in insts.values() for v in inst.values()] + if not any(isinstance(v, str) and v.startswith("${resources.") for v in values): + sys.stderr.write("gen_config did not inject a cross-resource reference\n") + failed = True + + schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../bundle/schema/jsonschema.json") + with open(schema_path) as f: + schema = json.load(f) + seed24 = gen_config(schema, seed=24, unique="check", allowed={"registered_models"}, resource_count=1) + rm = seed24["resources"]["registered_models"]["fuzz_registered_models_24"] + if SKIP_PROPERTY_NAMES & set(rm): + sys.stderr.write( + f"seed 24 registered_models emitted output-only fields: {sorted(SKIP_PROPERTY_NAMES & set(rm))}\n" + ) + failed = True + for field in ("name", "catalog_name", "schema_name"): + if field not in rm: + sys.stderr.write(f"seed 24 registered_models missing {field}\n") + failed = True + if failed: sys.exit(1) diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 38ea67a1b5a..d1443d16c9e 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -8,7 +8,10 @@ In order to add a new test, add a config to configs/ and include it in test.toml The fuzz/ test generates random configs from the live `databricks bundle schema` (see fuzz/script) and runs each one through a real invariant test script. The target is selected by `FUZZ_TARGET` (matrixed in fuzz/test.toml); each target is also a curated -invariant test that runs over the `INPUT_CONFIG` matrix: +invariant test that runs over the `INPUT_CONFIG` matrix. `FUZZ_RESOURCE_COUNT` (also +matrixed in fuzz/test.toml) controls how many resources each generated config contains; +with more than one, the generator links two of them with a `${resources.*}` reference so +the interpolation and deploy-ordering paths are exercised. - `no_drift` -- deploy, then no drift - `migrate` -- Terraform deploy, migrate to direct, then no drift @@ -21,3 +24,4 @@ Since the schema comes from the CLI under test, an unrelated struct change can s seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), not flakiness; reproduce with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_TARGET=no_drift task test-fuzz`. +For a multi-resource repro, add `FUZZ_RESOURCE_COUNT=2`. diff --git a/acceptance/bundle/invariant/canonical/script b/acceptance/bundle/invariant/canonical/script index d31f11622c2..0e57fd79778 100644 --- a/acceptance/bundle/invariant/canonical/script +++ b/acceptance/bundle/invariant/canonical/script @@ -3,7 +3,7 @@ # No deploy, so no cleanup or cloud state. if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/destroy_recreate/script b/acceptance/bundle/invariant/destroy_recreate/script index 2f3f7ed288d..dc3a8381331 100644 --- a/acceptance/bundle/invariant/destroy_recreate/script +++ b/acceptance/bundle/invariant/destroy_recreate/script @@ -4,7 +4,7 @@ # Additional checks: no internal errors / panics in validate/plan/deploy/destroy if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index b60e7ec13dc..2256ca7b2af 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2"] EnvMatrix.FUZZ_TARGET = [ "no_drift", "migrate", diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 872ce9abfba..c0c75c7e77b 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -5,3 +5,4 @@ EnvMatrix.INPUT_CONFIG = [] # Run the real invariant test script for each target. migrate ignores # DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] +EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2"] diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index cc484fb79da..8085533b880 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -4,7 +4,7 @@ unset DATABRICKS_BUNDLE_ENGINE if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index a12fb972fe0..31a1b8f8b3d 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -2,7 +2,7 @@ # Additional checks: no internal errors / panics in validate/plan/deploy if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/redeploy/script b/acceptance/bundle/invariant/redeploy/script index 42d9cebcd5a..ade3308b0d8 100644 --- a/acceptance/bundle/invariant/redeploy/script +++ b/acceptance/bundle/invariant/redeploy/script @@ -5,7 +5,7 @@ # re-derive a field), which surface as a redeploy wanting to change or recreate. if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/update/script b/acceptance/bundle/invariant/update/script index e1b2c28acf8..9395f76c099 100644 --- a/acceptance/bundle/invariant/update/script +++ b/acceptance/bundle/invariant/update/script @@ -4,7 +4,7 @@ # Additional checks: no internal errors / panics in validate/plan/deploy if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" > databricks.yml 2>LOG.gen.err + gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else From 565b0d635732b3abd4c98f9bc54deb457b492c6d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 7 Jul 2026 20:07:17 +0000 Subject: [PATCH 31/53] fuzz: keep first resource stable across --resource-count Gate the resource key/name suffix on the resource index, not the total count, so a given seed produces the same first resource whether --resource-count is 1 or greater. Later resources stay index-suffixed to remain unique. --- acceptance/bin/gen_fuzz_config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 825a7e1bff0..e17fb99bb57 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -259,7 +259,7 @@ def resource_types(schema, gen): return obj["properties"] -def gen_resource(schema, gen, types, candidates, seed, unique, index, resource_count): +def gen_resource(schema, gen, types, candidates, seed, unique, index): rtype = gen.rng.choice(sorted(candidates)) # Each resource type is a map ref; the element schema is the object branch's @@ -268,7 +268,10 @@ def gen_resource(schema, gen, types, candidates, seed, unique, index, resource_c obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") element = obj["additionalProperties"] - if resource_count == 1: + # The first resource keeps the bare key/name so a seed produces the same first + # resource regardless of --resource-count; later resources are index-suffixed to + # stay unique within the config. + if index == 0: key = f"fuzz_{rtype}_{seed}" gen.unique = unique else: @@ -343,9 +346,7 @@ def gen_config(schema, seed, unique, allowed, resource_count=1): records = [] for index in range(resource_count): - rtype, key, instance, element = gen_resource( - schema, gen, types, candidates, seed, unique, index, resource_count - ) + rtype, key, instance, element = gen_resource(schema, gen, types, candidates, seed, unique, index) records.append({"rtype": rtype, "key": key, "instance": instance, "ref_field": cross_ref_field(gen, element)}) inject_cross_ref(gen, records) From 5b7837ff64af5fc71c2cee4067e9454568f87f5c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 8 Jul 2026 07:54:27 +0000 Subject: [PATCH 32/53] fuzz: link every resource to an earlier one, not just one pair inject_cross_ref stopped after a single ${resources.*} edge, so a config with N resources still exercised only one reference. Link each resource to an earlier one instead, keeping the graph acyclic so deploy can order it, and add resource-count 3 to the matrix so the multi-link path runs in CI. --- acceptance/bin/gen_fuzz_config.py | 19 +++++++++++-------- acceptance/bundle/invariant/README.md | 5 +++-- acceptance/bundle/invariant/fuzz/test.toml | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index e17fb99bb57..fe7a14ed642 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -4,8 +4,9 @@ Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf branches) and emits one or more random resources as databricks.yml, seeded by --seed. -With --resource-count > 1 it also links two resources with a ${resources.*} reference so -the interpolation and deploy-ordering machinery is exercised. Feeds the invariant tests; +With --resource-count > 1 it also links resources with ${resources.*} references (each +resource referencing an earlier one) so the interpolation and deploy-ordering machinery is +exercised. Feeds the invariant tests; the harness filters out configs the CLI rejects, so output may be structurally-random but sometimes invalid. """ @@ -318,19 +319,21 @@ def target_ref_field(instance): def inject_cross_ref(gen, records): - # Link two resources so deploy has to order them and resolve the reference. + # Link resources so deploy has to order them and resolve the references. A + # record may only reference an earlier one, so the reference graph stays + # acyclic: deploy must topologically order resources, and a cycle can't be + # ordered (the config would be rejected instead of exercising the invariant). if len(records) < 2: return - sources = [r for r in records if r["ref_field"]] - gen.rng.shuffle(sources) - for source in sources: - targets = [t for t in records if t["key"] != source["key"] and target_ref_field(t["instance"])] + for i, source in enumerate(records): + if not source["ref_field"]: + continue + targets = [t for t in records[:i] if target_ref_field(t["instance"])] if not targets: continue target = gen.rng.choice(targets) field = target_ref_field(target["instance"]) source["instance"][source["ref_field"]] = f"${{resources.{target['rtype']}.{target['key']}.{field}}}" - return def gen_config(schema, seed, unique, allowed, resource_count=1): diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index d1443d16c9e..59ea2b13858 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -10,8 +10,9 @@ The fuzz/ test generates random configs from the live `databricks bundle schema` selected by `FUZZ_TARGET` (matrixed in fuzz/test.toml); each target is also a curated invariant test that runs over the `INPUT_CONFIG` matrix. `FUZZ_RESOURCE_COUNT` (also matrixed in fuzz/test.toml) controls how many resources each generated config contains; -with more than one, the generator links two of them with a `${resources.*}` reference so -the interpolation and deploy-ordering paths are exercised. +with more than one, the generator links them with `${resources.*}` references (each +resource referencing an earlier one, so the graph stays acyclic) so the interpolation and +deploy-ordering paths are exercised. - `no_drift` -- deploy, then no drift - `migrate` -- Terraform deploy, migrate to direct, then no drift diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index c0c75c7e77b..4692311bd27 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -5,4 +5,4 @@ EnvMatrix.INPUT_CONFIG = [] # Run the real invariant test script for each target. migrate ignores # DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] -EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2"] +EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] From 252e16c66e7571860d415d166dcaf685bcfa651e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 8 Jul 2026 12:56:08 +0000 Subject: [PATCH 33/53] invariant: regenerate fuzz out.test.toml for FUZZ_RESOURCE_COUNT=3 test.toml already lists resource count 3; regenerate the recorded matrix to match so the acceptance framework does not flag a mismatch. --- acceptance/bundle/invariant/fuzz/out.test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index 2256ca7b2af..4200e3cf0bd 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -2,7 +2,7 @@ Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2"] +EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] EnvMatrix.FUZZ_TARGET = [ "no_drift", "migrate", From fa5ef1cc07552398f265de397d0a67a6e649bcee Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 10 Jul 2026 08:22:10 +0000 Subject: [PATCH 34/53] fuzz: stage data fixtures and pin typed fields to cut rejections Point file_path/source_code_path fields at the staged data/. fixtures and pin typed-string fields so generated configs deploy instead of getting rejected. Also ignore the local .fuzztmp/ driver scratch directory. --- .gitignore | 3 + acceptance/bin/gen_fuzz_config.py | 78 ++++++++++++++++++++++++- acceptance/bundle/invariant/fuzz/script | 6 +- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 543b638a537..63a15332d02 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,9 @@ dist/ # Per-module golangci-lint TMPDIR (configured in Taskfile.yml) /.tmp/ +# Local fuzz driver scratch (see .fuzztmp/run_fuzz.sh) +.fuzztmp/ + # Go workspace file go.work go.work.sum diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index fe7a14ed642..b19682b6e92 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -13,7 +13,9 @@ import argparse import json +import os import random +import re import sys # The schema is recursive (e.g. task -> for_each_task -> task); cap the walk. @@ -81,6 +83,9 @@ "created_at", "created_by", "creator_name", + # An etag is a read value the backend assigns; the CLI rejects one set in + # bundle config (e.g. "genie space ... has an etag set. Etags must not be set"). + "etag", "full_name", "metastore_id", "owner", @@ -90,11 +95,51 @@ } ) -# Resource types whose schema omits required[] but need these fields to deploy. +# Resource types whose schema omits required[] (or whose required[] can't be honored +# in bundle YAML) but which need these fields to deploy. See RESOURCE_FIELD_ALLOWLIST +# and the *_BY_RESOURCE tables below for the values these fields take. RESOURCE_REQUIRED_FIELDS = { "registered_models": frozenset({"catalog_name", "name", "schema_name"}), + "dashboards": frozenset({"display_name", "file_path", "warehouse_id"}), + "alerts": frozenset({"display_name", "file_path", "warehouse_id"}), + "apps": frozenset({"name", "source_code_path"}), + "genie_spaces": frozenset({"serialized_space", "title", "warehouse_id"}), } +# Fields to drop for a specific resource type because they conflict with the field +# set we do emit. Dashboards and Genie spaces take their body from file_path XOR an +# inline serialized_* field, so emitting both is rejected ("both ... are set"). +RESOURCE_SKIP_FIELDS = { + "dashboards": frozenset({"serialized_dashboard"}), + "genie_spaces": frozenset({"file_path"}), + "apps": frozenset({"git_repository", "git_source"}), +} + +# Resource types where only a fixed field set is allowed in bundle YAML. Alerts read +# their spec from the .dbalert.json referenced by file_path; the CLI rejects any other +# field (see bundle/config/mutator/load_dbalert_files.go allowedInYAML). +RESOURCE_FIELD_ALLOWLIST = { + "alerts": frozenset({"display_name", "file_path", "lifecycle", "permissions", "warehouse_id"}), +} + +# file_path points at a serialized-body fixture copied into every seed dir from +# acceptance/bundle/invariant/data (see fuzz/script). The extension selects the parser. +FILE_PATH_BY_RESOURCE = { + "dashboards": "./dashboard.lvdash.json", + "alerts": "./alert.dbalert.json", +} + +# A local directory holding app source, also copied in from data/. +APP_SOURCE_CODE_PATH = "./app" + +# An absolute workspace path is treated as already-remote, skipping the local-notebook +# existence/extension check a bare token would fail. +NOTEBOOK_PATH = "/Shared/notebook" + +# Fields declared as string in the schema but parsed as google.protobuf.Duration at +# config load (e.g. suspend_timeout_duration, ttl); a bare token fails to parse. +DURATION_VALUE = "3600s" + class Generator: def __init__(self, schema, rng, unique): @@ -135,6 +180,8 @@ def field_behaviors(self, schema): def should_skip_property(self, prop_name, prop_schema): if prop_name in SKIP_PROPERTY_NAMES: return True + if prop_name in RESOURCE_SKIP_FIELDS.get(self.rtype, ()): + return True resolved = self.resolve(prop_schema) if "OUTPUT_ONLY" in self.field_behaviors(prop_schema): return True @@ -143,6 +190,11 @@ def should_skip_property(self, prop_name, prop_schema): return False def gen(self, schema, depth, name=""): + # A Genie space body is a free-form interface{}; the backend rejects unknown + # keys, so emit the minimal accepted body instead of a random object. + if name == "serialized_space": + return {"version": 1} + schema = self.resolve(schema) if not isinstance(schema, dict) or not schema: return self.gen_scalar({"type": "string"}, name) @@ -174,11 +226,17 @@ def is_map(self, schema): def gen_object(self, schema, depth): props = schema.get("properties", {}) required = set(schema.get("required", [])) + allowlist = None if depth == 0 and self.rtype: required |= RESOURCE_REQUIRED_FIELDS.get(self.rtype, set()) + allowlist = RESOURCE_FIELD_ALLOWLIST.get(self.rtype) result = {} for prop_name, prop_schema in props.items(): + # A restricted resource (e.g. alerts) rejects any field outside its + # allow-list, even a schema-required one supplied via the file instead. + if allowlist is not None and prop_name not in allowlist: + continue if self.should_skip_property(prop_name, prop_schema): continue # Always emit required fields; emit optional ones less often as we go @@ -240,11 +298,27 @@ def gen_scalar(self, schema, name): if t is not None and t not in SCALAR_TYPES: sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") # string (default) - # Pin cross-resource references to seeded defaults (see constants above). + # Pin cross-resource references and typed-string fields to values the backend + # accepts; a random token fails format/existence validation and drops the config. if name == "catalog_name": return DEFAULT_CATALOG if name == "schema_name": return DEFAULT_SCHEMA + if name == "warehouse_id": + return os.environ.get("TEST_DEFAULT_WAREHOUSE_ID", "") + if name == "notebook_path": + return NOTEBOOK_PATH + if name == "source_code_path": + return APP_SOURCE_CODE_PATH + if name == "file_path": + return FILE_PATH_BY_RESOURCE.get(self.rtype, self.token()) + if name.endswith("_duration") or name == "ttl": + return DURATION_VALUE + if name == "name" and self.rtype == "vector_search_indexes": + # UC requires the full three-level catalog.schema.table name, and each + # part accepts only alphanumerics and underscores. + table = re.sub(r"[^0-9a-zA-Z_]", "_", f"fuzz_index_{self.unique}") + return f"{DEFAULT_CATALOG}.{DEFAULT_SCHEMA}.{table}" if name in ("name", "display_name"): return f"fuzz-{name}-{self.unique}" return self.token() diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index e8377aea705..d950bb89f89 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -32,6 +32,10 @@ for ((offset = 0; offset < COUNT; offset++)); do set +e ( cd "$dir" + # The generator points file_path/source_code_path fields at these fixtures + # (see gen_fuzz_config.py *_BY_RESOURCE), so make them resolvable from the + # bundle root, matching how the curated invariant scripts stage data/. + cp -r "$TESTDIR/../data/." . export FUZZ_SEED="$seed" export FUZZ_SCHEMA="../schema.json" source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" @@ -57,7 +61,7 @@ for ((offset = 0; offset < COUNT; offset++)); do fi if [ -n "$bug" ]; then - echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} task test-fuzz" >&2 + echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 exit 1 fi done From 2c1ebcd815dd837a7c4f939f8b7ae8c56ebe6489 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 10 Jul 2026 08:22:24 +0000 Subject: [PATCH 35/53] fuzz: add mutate mode that perturbs curated invariant configs Add a mutate engine (mutate_fuzz_config.py) that applies seeded delete/set/dangerous-value mutations to a curated base config, and a fuzz_gen_config.py dispatcher selected by FUZZ_MODE (generate|mutate). Wire the dispatcher into the six invariant target scripts, matrix FUZZ_MODE in fuzz/test.toml (excluding redundant mutate runs at resource-count 2/3), and add a selftest covering the YAML loader round-trip and mutation determinism. --- acceptance/bin/fuzz_gen_config.py | 73 +++++++ acceptance/bin/mutate_fuzz_config.py | 203 ++++++++++++++++++ acceptance/bin/mutate_fuzz_config_check.py | 66 ++++++ acceptance/bundle/invariant/canonical/script | 2 +- .../bundle/invariant/destroy_recreate/script | 2 +- .../bundle/invariant/fuzz/out.test.toml | 1 + acceptance/bundle/invariant/fuzz/test.toml | 9 + acceptance/bundle/invariant/migrate/script | 2 +- acceptance/bundle/invariant/no_drift/script | 2 +- acceptance/bundle/invariant/redeploy/script | 2 +- acceptance/bundle/invariant/update/script | 2 +- .../selftest/mutate_fuzz_config/out.test.toml | 3 + .../selftest/mutate_fuzz_config/output.txt | 36 ++++ acceptance/selftest/mutate_fuzz_config/script | 1 + 14 files changed, 398 insertions(+), 6 deletions(-) create mode 100755 acceptance/bin/fuzz_gen_config.py create mode 100755 acceptance/bin/mutate_fuzz_config.py create mode 100755 acceptance/bin/mutate_fuzz_config_check.py create mode 100644 acceptance/selftest/mutate_fuzz_config/out.test.toml create mode 100644 acceptance/selftest/mutate_fuzz_config/output.txt create mode 100644 acceptance/selftest/mutate_fuzz_config/script diff --git a/acceptance/bin/fuzz_gen_config.py b/acceptance/bin/fuzz_gen_config.py new file mode 100755 index 00000000000..a228bb0c82a --- /dev/null +++ b/acceptance/bin/fuzz_gen_config.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from +FUZZ_MODE so the invariant target scripts don't each duplicate the branch: + + generate (default) - build a config from scratch by walking `bundle schema` + (gen_fuzz_config.py). + mutate - start from a curated invariant config and perturb it + (mutate_fuzz_config.py). + +Reads its inputs from the environment the invariant scripts already export: FUZZ_SEED, +FUZZ_SCHEMA, UNIQUE_NAME, FUZZ_RESOURCES, FUZZ_RESOURCE_COUNT, TESTDIR. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables +from gen_fuzz_config import gen_config, to_yaml +from mutate_fuzz_config import load_yaml, mutate + +# Curated single-resource configs that deploy standalone against the fake server (only +# $UNIQUE_NAME, no init script). All are also in the invariant INPUT_CONFIG matrix, so +# they stay deploy-verified. The seed selects one; mutate_fuzz_config perturbs it. +MUTATE_BASES = [ + "catalog", + "external_location", + "job", + "model", + "model_serving_endpoint", + "pipeline", + "registered_model", + "schema", + "secret_scope", + "sql_warehouse", + "volume", +] + + +def generate(seed): + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + allowed = {r.strip() for r in os.environ.get("FUZZ_RESOURCES", "").split(",") if r.strip()} + unique = f"{os.environ['UNIQUE_NAME']}-{seed}" + count = int(os.environ.get("FUZZ_RESOURCE_COUNT", "1")) + return to_yaml(gen_config(schema, seed, unique, allowed, count)) + + +def mutate_base(seed): + name = MUTATE_BASES[seed % len(MUTATE_BASES)] + path = os.path.join(os.environ["TESTDIR"], "..", "configs", name + ".yml.tmpl") + with open(path) as f: + rendered = substitute_variables(f.read()) + config = load_yaml(rendered) + return to_yaml(mutate(config, seed)) + + +def main(): + seed = int(os.environ["FUZZ_SEED"]) + mode = os.environ.get("FUZZ_MODE", "generate") + if mode == "generate": + sys.stdout.write(generate(seed)) + elif mode == "mutate": + sys.stdout.write(mutate_base(seed)) + else: + sys.exit(f"fuzz_gen_config: unknown FUZZ_MODE {mode!r}") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py new file mode 100755 index 00000000000..652ac525a79 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +Mutate a known-good bundle config by deleting and perturbing random fields. + +Complements gen_fuzz_config.py (generate-from-scratch via schema walk): instead of +building a config from the schema, this starts from a curated invariant config that +already deploys and applies a few seeded mutations (delete a field, replace a scalar +with a fuzz token, a boundary/dangerous value, or an empty container). It exercises the +CLI's handling of perturbed-but-realistic input, and reaches a much higher deploy rate +than the schema walk, since the base already resolves. + +Reads the base databricks.yml (already envsubst-rendered) from stdin, writes the mutated +config to stdout. --seed makes the mutation reproducible. + +The invariant harness only asserts no-panic on fuzzed configs (SKIP_DRIFT_CHECK), so a +mutation that makes the config invalid is fine: the CLI must reject it cleanly, not crash. +""" + +import argparse +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import to_yaml + +# Values chosen to probe near-range-end and dangerous-character handling: empty and +# whitespace-only strings, an over-long string, embedded newlines/tabs, non-ASCII, quotes, +# a dangling ${...} reference, a path-traversal string, and integer boundaries. +DANGEROUS = [ + "", + " ", + "a" * 300, + "line1\nline2", + "tab\there", + "\U0001f680-unicode-\u00e9", + 'quote"and\'apostrophe', + "${resources.jobs.does_not_exist.id}", + "../../etc/passwd", + 2**31, + -(2**31), + 2**63 - 1, + -1, +] + + +def tokenize(text): + # (indent, content) per non-blank, non-comment line. Only full-line comments are + # stripped; the curated bases don't use trailing "#" in values. + out = [] + for raw in text.splitlines(): + stripped = raw.lstrip(" ") + if not stripped or stripped.startswith("#"): + continue + out.append((len(raw) - len(stripped), stripped.rstrip())) + return out + + +def scalar(text): + if text in ("", "null", "~"): + return None + if text == "true": + return True + if text == "false": + return False + try: + return int(text) + except ValueError: + pass + try: + return float(text) + except ValueError: + pass + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +def parse_block(tokens, i, indent): + if i >= len(tokens): + return {}, i + first = tokens[i][1] + if first.startswith("- ") or first == "-": + return parse_seq(tokens, i, indent) + if ": " in first or first.endswith(":"): + return parse_map(tokens, i, indent) + # Bare scalar: the whole block is a single value (e.g. a list scalar item). + return scalar(first), i + 1 + + +def parse_map(tokens, i, indent): + result = {} + while i < len(tokens) and tokens[i][0] == indent: + content = tokens[i][1] + if content.startswith("- "): + break + if ": " in content: + key, _, rest = content.partition(": ") + result[key.strip()] = scalar(rest) + i += 1 + elif content.endswith(":"): + key = content[:-1].strip() + i += 1 + if i < len(tokens) and tokens[i][0] > indent: + value, i = parse_block(tokens, i, tokens[i][0]) + else: + value = None + result[key] = value + else: + break + return result, i + + +def parse_seq(tokens, i, indent): + result = [] + while i < len(tokens) and tokens[i][0] == indent and (tokens[i][1].startswith("- ") or tokens[i][1] == "-"): + after = tokens[i][1][2:] if tokens[i][1].startswith("- ") else "" + child_indent = indent + 2 + # The item is its own block: the inline remainder (re-indented to child_indent) + # plus any deeper continuation lines that belong to it. + item = [] + if after: + item.append((child_indent, after)) + i += 1 + while i < len(tokens) and tokens[i][0] >= child_indent: + item.append(tokens[i]) + i += 1 + result.append(parse_block(item, 0, child_indent)[0] if item else None) + return result, i + + +def load_yaml(text): + tokens = tokenize(text) + value, _ = parse_block(tokens, 0, 0) + return value + + +def collect(node, out): + # (container, key) for every child, so a mutation can delete or replace it in place. + if isinstance(node, dict): + for k, v in node.items(): + out.append((node, k)) + collect(v, out) + elif isinstance(node, list): + for idx, v in enumerate(node): + out.append((node, idx)) + collect(v, out) + + +def token(rng): + return "fuzz_" + "".join(rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def mutate_once(rng, roots): + refs = [] + for root in roots: + collect(root, refs) + if not refs: + return + container, key = rng.choice(refs) + op = rng.choice(["delete", "scalar", "dangerous", "empty"]) + if op == "delete": + del container[key] + elif op == "scalar": + container[key] = token(rng) + elif op == "dangerous": + container[key] = rng.choice(DANGEROUS) + else: + container[key] = rng.choice([{}, [], None]) + + +def mutate(config, seed): + rng = random.Random(seed) + + # Mutate only inside resource instances: keep bundle/name and the + # resources.. skeleton so there is always something to deploy, while + # every field of the instance (including required ones) is fair game. + roots = [] + for instances in config.get("resources", {}).values(): + if isinstance(instances, dict): + roots.extend(v for v in instances.values() if isinstance(v, (dict, list))) + + for _ in range(rng.randint(1, 3)): + mutate_once(rng, roots) + + return config + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--seed", type=int, required=True, help="RNG seed (for reproducibility)") + args = parser.parse_args() + + config = load_yaml(sys.stdin.read()) + if not isinstance(config, dict): + sys.exit("mutate_fuzz_config: base config did not parse to a mapping") + + sys.stdout.write(to_yaml(mutate(config, args.seed))) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py new file mode 100755 index 00000000000..6df04885dc6 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Contract check for mutate_fuzz_config's minimal YAML loader and mutation engine +(the harness diffs stdout; a non-zero exit marks a violation on stderr): + +- The loader round-trips every curated base config: load -> to_yaml -> load is a + fixed point, so a base template the loader can't represent is caught here rather + than as a confusing fuzz failure. +- Mutation is deterministic for a fixed seed (reproducible repros). + +It also prints a few mutated configs so an accidental change to the algorithm shows up +as an output diff. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables +from fuzz_gen_config import MUTATE_BASES +from mutate_fuzz_config import load_yaml, mutate, to_yaml + +CONFIGS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "invariant", "configs") + + +def render(name): + with open(os.path.join(CONFIGS, name + ".yml.tmpl")) as f: + return substitute_variables(f.read()) + + +def main(): + # Fixed so the printed configs are stable regardless of the harness's unique name. + os.environ["UNIQUE_NAME"] = "check" + failed = False + + for name in MUTATE_BASES: + text = render(name) + parsed = load_yaml(text) + if not isinstance(parsed, dict) or "resources" not in parsed: + sys.stderr.write(f"{name}: base did not parse to a config with resources\n") + failed = True + continue + # load -> emit -> load must be a fixed point. + if load_yaml(to_yaml(parsed)) != parsed: + sys.stderr.write(f"{name}: loader is not a round-trip fixed point\n") + failed = True + + # Mutation must be reproducible for a fixed seed. + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("volume")), seed)) + b = to_yaml(mutate(load_yaml(render("volume")), seed)) + if a != b: + sys.stderr.write(f"seed {seed}: mutation is not deterministic\n") + failed = True + + for seed in range(3): + sys.stdout.write(f"=== volume seed={seed} ===\n") + sys.stdout.write(to_yaml(mutate(load_yaml(render("volume")), seed))) + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/invariant/canonical/script b/acceptance/bundle/invariant/canonical/script index 0e57fd79778..fe8acc5eb42 100644 --- a/acceptance/bundle/invariant/canonical/script +++ b/acceptance/bundle/invariant/canonical/script @@ -3,7 +3,7 @@ # No deploy, so no cleanup or cloud state. if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/destroy_recreate/script b/acceptance/bundle/invariant/destroy_recreate/script index dc3a8381331..7e615904f5f 100644 --- a/acceptance/bundle/invariant/destroy_recreate/script +++ b/acceptance/bundle/invariant/destroy_recreate/script @@ -4,7 +4,7 @@ # Additional checks: no internal errors / panics in validate/plan/deploy/destroy if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index 4200e3cf0bd..d8d85f175d1 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] EnvMatrix.FUZZ_TARGET = [ "no_drift", diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 4692311bd27..a27ecdec440 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -6,3 +6,12 @@ EnvMatrix.INPUT_CONFIG = [] # DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] + +# generate = build a config from the schema (gen_fuzz_config.py); mutate = perturb a +# curated invariant config (mutate_fuzz_config.py). Dispatched by fuzz_gen_config.py. +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] + +# mutate starts from a single-resource base and ignores FUZZ_RESOURCE_COUNT, so only run +# it once rather than duplicating the same output across the count matrix. +EnvMatrixExclude.mutate_count2 = ["FUZZ_MODE=mutate", "FUZZ_RESOURCE_COUNT=2"] +EnvMatrixExclude.mutate_count3 = ["FUZZ_MODE=mutate", "FUZZ_RESOURCE_COUNT=3"] diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 8085533b880..a4357474be3 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -4,7 +4,7 @@ unset DATABRICKS_BUNDLE_ENGINE if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 31a1b8f8b3d..b5d2a569d2d 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -2,7 +2,7 @@ # Additional checks: no internal errors / panics in validate/plan/deploy if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/redeploy/script b/acceptance/bundle/invariant/redeploy/script index ade3308b0d8..18382b44b4a 100644 --- a/acceptance/bundle/invariant/redeploy/script +++ b/acceptance/bundle/invariant/redeploy/script @@ -5,7 +5,7 @@ # re-derive a field), which surface as a redeploy wanting to change or recreate. if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/bundle/invariant/update/script b/acceptance/bundle/invariant/update/script index 9395f76c099..7a91c38cef9 100644 --- a/acceptance/bundle/invariant/update/script +++ b/acceptance/bundle/invariant/update/script @@ -4,7 +4,7 @@ # Additional checks: no internal errors / panics in validate/plan/deploy if [ -n "${FUZZ_SEED:-}" ]; then - gen_fuzz_config.py --schema "${FUZZ_SCHEMA}" --seed "$FUZZ_SEED" --unique "$UNIQUE_NAME-$FUZZ_SEED" --resources "${FUZZ_RESOURCES:-}" --resource-count "${FUZZ_RESOURCE_COUNT:-1}" > databricks.yml 2>LOG.gen.err + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err cat LOG.gen.err | contains.py '!Traceback' > /dev/null cp databricks.yml LOG.config else diff --git a/acceptance/selftest/mutate_fuzz_config/out.test.toml b/acceptance/selftest/mutate_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/mutate_fuzz_config/output.txt b/acceptance/selftest/mutate_fuzz_config/output.txt new file mode 100644 index 00000000000..4b7289cfa51 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/output.txt @@ -0,0 +1,36 @@ +=== volume seed=0 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: "main" + schema_name: [] + grants: + - principal: "account users" +=== volume seed=1 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: " " + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" +=== volume seed=2 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" diff --git a/acceptance/selftest/mutate_fuzz_config/script b/acceptance/selftest/mutate_fuzz_config/script new file mode 100644 index 00000000000..2d57c739261 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/script @@ -0,0 +1 @@ +mutate_fuzz_config_check.py From b2ddf69e82519b7ed1a9ac94c3f593c4c2d2958f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 13 Jul 2026 09:18:46 +0000 Subject: [PATCH 36/53] fuzz: probe dangerous and near-range-end values in generate mode The schema-walk generator pinned every scalar to a known-good value, so dangerous / near-range-end input was only exercised by mutate mode. Inject DANGEROUS_STRINGS/DANGEROUS_INTS into free-form scalars (~15% of the time) so generate mode probes empty/whitespace/over-long/control-char strings and int32/int64 boundaries while pinned fields keep values that still deploy. Centralize the two lists in gen_fuzz_config.py so mutate_fuzz_config.py reuses them unchanged. --- acceptance/bin/gen_fuzz_config.py | 40 +++++++++++++++++++++++-- acceptance/bin/gen_fuzz_config_check.py | 10 ++++++- acceptance/bin/mutate_fuzz_config.py | 25 ++++------------ acceptance/bundle/invariant/README.md | 4 ++- 4 files changed, 54 insertions(+), 25 deletions(-) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index b19682b6e92..147930fbb5e 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -6,9 +6,10 @@ branches) and emits one or more random resources as databricks.yml, seeded by --seed. With --resource-count > 1 it also links resources with ${resources.*} references (each resource referencing an earlier one) so the interpolation and deploy-ordering machinery is -exercised. Feeds the invariant tests; -the harness filters out configs the CLI rejects, so output may be structurally-random but -sometimes invalid. +exercised. Free-form scalars are occasionally replaced with dangerous / near-range-end +values (DANGEROUS_STRINGS, DANGEROUS_INTS) to probe the CLI's input handling. Feeds the +invariant tests; the harness filters out configs the CLI rejects, so output may be +structurally-random but sometimes invalid. """ import argparse @@ -140,6 +141,32 @@ # config load (e.g. suspend_timeout_duration, ttl); a bare token fails to parse. DURATION_VALUE = "3600s" +# Dangerous / near-range-end probes injected into free-form scalars: empty and +# whitespace-only strings, an over-long string, embedded newlines/tabs, non-ASCII, quotes, +# a dangling ${...} reference, a path-traversal string, and int32/int64 boundaries. The CLI +# must reject or round-trip these without panicking; mutate_fuzz_config.py reuses both lists. +DANGEROUS_STRINGS = [ + "", + " ", + "a" * 300, + "line1\nline2", + "tab\there", + "\U0001f680-unicode-\u00e9", + "quote\"and'apostrophe", + "${resources.jobs.does_not_exist.id}", + "../../etc/passwd", +] +DANGEROUS_INTS = [ + 2**31, + -(2**31), + 2**63 - 1, + -1, +] + +# Only inject a dangerous value some of the time: a fuzzed field mostly keeps a plausible +# value so the config still deploys and exercises the invariant, not just the reject path. +DANGEROUS_PROB = 0.15 + class Generator: def __init__(self, schema, rng, unique): @@ -291,6 +318,8 @@ def gen_scalar(self, schema, name): # days; only 0 or 168-720 (hours) are accepted. if name == "custom_max_retention_hours": return self.rng.choice([0, self.rng.randint(168, 720)]) + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_INTS) return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) if t == "number": return round(self.rng.uniform(0, 1000), 2) @@ -321,6 +350,11 @@ def gen_scalar(self, schema, name): return f"{DEFAULT_CATALOG}.{DEFAULT_SCHEMA}.{table}" if name in ("name", "display_name"): return f"fuzz-{name}-{self.unique}" + # A free-form string with no pinned meaning (e.g. description, comment, tag value): + # probe dangerous / near-range-end input here, where a rejected or normalized value + # doesn't just fail the field-format check a pinned field above guards against. + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_STRINGS) return self.token() def token(self): diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py index f707144148a..8649f4c90b5 100755 --- a/acceptance/bin/gen_fuzz_config_check.py +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from edit_fuzz_config import FIELD_RE -from gen_fuzz_config import SKIP_PROPERTY_NAMES, gen_config, to_yaml +from gen_fuzz_config import DANGEROUS_STRINGS, SKIP_PROPERTY_NAMES, gen_config, to_yaml # Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, empty containers. CASES = [ @@ -58,6 +58,14 @@ def main(): sys.stderr.write("FIELD_RE did not match a comment/description line\n") failed = True + # Generate mode now emits DANGEROUS_STRINGS into free-form fields; each must still + # serialize to a single `key: ` line so edit_fuzz_config can rewrite it in place. + for i, val in enumerate(DANGEROUS_STRINGS): + line = to_yaml({"description": val}).rstrip("\n") + if "\n" in line or not FIELD_RE.match(line): + sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line comment/description contract: {line!r}\n") + failed = True + # Multi-resource configs merge types under resources., and one resource # references another's name so the interpolation/ordering path is exercised. def resource_type(field): diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py index 652ac525a79..cd9c9336801 100755 --- a/acceptance/bin/mutate_fuzz_config.py +++ b/acceptance/bin/mutate_fuzz_config.py @@ -23,26 +23,11 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from gen_fuzz_config import to_yaml - -# Values chosen to probe near-range-end and dangerous-character handling: empty and -# whitespace-only strings, an over-long string, embedded newlines/tabs, non-ASCII, quotes, -# a dangling ${...} reference, a path-traversal string, and integer boundaries. -DANGEROUS = [ - "", - " ", - "a" * 300, - "line1\nline2", - "tab\there", - "\U0001f680-unicode-\u00e9", - 'quote"and\'apostrophe', - "${resources.jobs.does_not_exist.id}", - "../../etc/passwd", - 2**31, - -(2**31), - 2**63 - 1, - -1, -] +from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, to_yaml + +# Same near-range-end and dangerous-character probes the schema-walk generator injects into +# free-form scalars; here we drop them onto any field (see mutate_once). +DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS def tokenize(text): diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 59ea2b13858..3f653324498 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -12,7 +12,9 @@ invariant test that runs over the `INPUT_CONFIG` matrix. `FUZZ_RESOURCE_COUNT` ( matrixed in fuzz/test.toml) controls how many resources each generated config contains; with more than one, the generator links them with `${resources.*}` references (each resource referencing an earlier one, so the graph stays acyclic) so the interpolation and -deploy-ordering paths are exercised. +deploy-ordering paths are exercised. Free-form scalars are occasionally replaced with +dangerous / near-range-end values (empty, whitespace, over-long, control characters, +int32/int64 boundaries) to probe the CLI's input handling. - `no_drift` -- deploy, then no drift - `migrate` -- Terraform deploy, migrate to direct, then no drift From c3338877319035c6dfff074ddc881048d1d789b1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 13 Jul 2026 13:58:52 +0000 Subject: [PATCH 37/53] acc/fuzz: skip immutable comment/description in update invariant The update invariant edits a comment/description and asserts an in-place update, but some resources classify that field as recreate_on_changes (e.g. model_serving_endpoints.description). A fuzz-generated config that set such a field made the correct recreate look like an invariant bug. edit_fuzz_config.py now reads recreate_on_changes from resources.yml, tracks the enclosing resource type while scanning, and skips immutable comment/description fields (picking a mutable one or reporting none). Adds a contract self-check (edit_fuzz_config_check.py) and a selftest. --- acceptance/bin/edit_fuzz_config.py | 69 +++++++++++++-- acceptance/bin/edit_fuzz_config_check.py | 84 +++++++++++++++++++ .../selftest/edit_fuzz_config/out.test.toml | 3 + .../selftest/edit_fuzz_config/output.txt | 4 + acceptance/selftest/edit_fuzz_config/script | 1 + 5 files changed, 155 insertions(+), 6 deletions(-) create mode 100755 acceptance/bin/edit_fuzz_config_check.py create mode 100644 acceptance/selftest/edit_fuzz_config/out.test.toml create mode 100644 acceptance/selftest/edit_fuzz_config/output.txt create mode 100644 acceptance/selftest/edit_fuzz_config/script diff --git a/acceptance/bin/edit_fuzz_config.py b/acceptance/bin/edit_fuzz_config.py index 3237a089e9c..e2449c4e60f 100755 --- a/acceptance/bin/edit_fuzz_config.py +++ b/acceptance/bin/edit_fuzz_config.py @@ -1,11 +1,16 @@ #!/usr/bin/env python3 """ Edit a `comment`/`description` scalar in a generated databricks.yml so a redeploy is an -in-place update, not a recreate. Used by the `update` invariant. These fields are safe -to edit across resource types. +in-place update, not a recreate. Used by the `update` invariant. + +Most resources take a `comment`/`description` edit as an in-place update, but some +classify it as immutable (recreate on change) in the direct engine's resource spec +(e.g. model_serving_endpoints.description). Editing such a field replans as a recreate, +which the update invariant would wrongly flag as a bug, so we skip it and pick a mutable +field elsewhere (or report none). gen_fuzz_config.py emits one scalar per line as `key: `, so a regex match suffices -(no YAML dependency). +(no YAML dependency for the edit itself); the immutable set is read from resources.yml. edit_fuzz_config.py PATH edit in place; exit 1 if no editable field edit_fuzz_config.py PATH --detect exit 0 if an editable field exists, else 1 @@ -14,18 +19,70 @@ import argparse import re import sys +from pathlib import Path # Allow an optional "- " so a comment/description that is the first key of a list-item # dict still matches; the captured prefix is preserved verbatim on rewrite. FIELD_RE = re.compile(r'^(\s*(?:- )?)(comment|description): (".*")\s*$') +# A resource type header directly under `resources:` (two-space indent, as emitted by +# gen_fuzz_config.py and the curated templates). +TYPE_RE = re.compile(r"^ ([\w-]+):\s*$") + NEW_VALUE = '"fuzz_edited_value"' +# resources.yml is the source of truth for field mutability; acceptance/bin sits two +# levels below the repo root, and the real dir (not a copy) is on PATH, so __file__ +# resolves here. +RESOURCES_YML = Path(__file__).resolve().parents[2] / "bundle" / "direct" / "dresources" / "resources.yml" + + +def immutable_fields(): + """Map resource type -> set of fields that recreate on change (immutable). + + resources.yml has a fixed two-space layout (`resources:` -> ` :` -> + ` recreate_on_changes:` -> ` - field: `), so a small line parser + avoids a YAML dependency the harness's Python does not have. + """ + result = {} + current_type = None + in_recreate = False + for raw in RESOURCES_YML.read_text().splitlines(): + if not raw.strip() or raw.lstrip().startswith("#"): + continue + indent = len(raw) - len(raw.lstrip()) + stripped = raw.strip() + if indent == 2 and stripped.endswith(":"): + current_type = stripped[:-1] + in_recreate = False + elif indent == 4 and stripped.endswith(":"): + in_recreate = stripped == "recreate_on_changes:" + elif in_recreate and current_type: + m = re.match(r"-\s*field:\s*(\S+)", stripped) + if m: + result.setdefault(current_type, set()).add(m.group(1)) + return result + -def find_line(lines): +def find_line(lines, immutable): + current_type = None + in_resources = False for i, line in enumerate(lines): + stripped = line.rstrip("\n") + # Track the enclosing resource type so an immutable comment/description is skipped. + if stripped == "resources:": + in_resources = True + current_type = None + continue + if in_resources: + m_type = TYPE_RE.match(stripped) + if m_type: + current_type = m_type.group(1) + elif stripped and not stripped[0].isspace(): + in_resources = False + current_type = None m = FIELD_RE.match(line) - if m: + if m and m.group(2) not in immutable.get(current_type, ()): return i, m return -1, None @@ -39,7 +96,7 @@ def main(): with open(args.path) as f: lines = f.readlines() - i, m = find_line(lines) + i, m = find_line(lines, immutable_fields()) if m is None: sys.exit(1) if args.detect: diff --git a/acceptance/bin/edit_fuzz_config_check.py b/acceptance/bin/edit_fuzz_config_check.py new file mode 100755 index 00000000000..b8160dbe627 --- /dev/null +++ b/acceptance/bin/edit_fuzz_config_check.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Contract check for edit_fuzz_config's field selection (the harness diffs stdout; a +non-zero exit marks a violation on stderr): + +- A comment/description that recreates on change for its resource type (per + resources.yml, e.g. model_serving_endpoints.description) is never chosen, so the + update invariant does not assert an in-place update the backend cannot perform. +- A mutable comment/description is still chosen, even when an immutable one appears + first. +- The immutable map actually loads and reflects resources.yml. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from edit_fuzz_config import find_line, immutable_fields + +IMMUTABLE_ONLY = """\ +resources: + model_serving_endpoints: + foo: + name: "test-endpoint" + description: "old" +""" + +IMMUTABLE_THEN_MUTABLE = """\ +resources: + model_serving_endpoints: + foo: + description: "immutable" + jobs: + bar: + description: "mutable" +""" + +MUTABLE_ONLY = """\ +resources: + jobs: + bar: + description: "mutable" +""" + + +def choose(text, immutable): + i, m = find_line(text.splitlines(keepends=True), immutable) + return None if m is None else i + + +def main(): + immutable = immutable_fields() + failed = False + + # Guards the loader and the classification the update invariant relies on. + serving_immutable = "description" in immutable.get("model_serving_endpoints", set()) + if not serving_immutable: + sys.stderr.write("expected model_serving_endpoints.description to be immutable in resources.yml\n") + failed = True + + if choose(IMMUTABLE_ONLY, immutable) is not None: + sys.stderr.write("picked an immutable description\n") + failed = True + + if choose(IMMUTABLE_THEN_MUTABLE, immutable) != 6: + sys.stderr.write("expected to skip the immutable description and pick the mutable one\n") + failed = True + + if choose(MUTABLE_ONLY, immutable) != 3: + sys.stderr.write("expected to pick the mutable description\n") + failed = True + + print(f"model_serving_endpoints.description immutable: {serving_immutable}") + print(f"IMMUTABLE_ONLY: {choose(IMMUTABLE_ONLY, immutable)}") + print(f"IMMUTABLE_THEN_MUTABLE: {choose(IMMUTABLE_THEN_MUTABLE, immutable)}") + print(f"MUTABLE_ONLY: {choose(MUTABLE_ONLY, immutable)}") + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/selftest/edit_fuzz_config/out.test.toml b/acceptance/selftest/edit_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/edit_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/edit_fuzz_config/output.txt b/acceptance/selftest/edit_fuzz_config/output.txt new file mode 100644 index 00000000000..4037d0995cf --- /dev/null +++ b/acceptance/selftest/edit_fuzz_config/output.txt @@ -0,0 +1,4 @@ +model_serving_endpoints.description immutable: True +IMMUTABLE_ONLY: None +IMMUTABLE_THEN_MUTABLE: 6 +MUTABLE_ONLY: 3 diff --git a/acceptance/selftest/edit_fuzz_config/script b/acceptance/selftest/edit_fuzz_config/script new file mode 100644 index 00000000000..78dfbef7825 --- /dev/null +++ b/acceptance/selftest/edit_fuzz_config/script @@ -0,0 +1 @@ +edit_fuzz_config_check.py From e4bac490ee016611de9e92bca5e2c0c6e330021a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 13 Jul 2026 14:09:22 +0000 Subject: [PATCH 38/53] acc/fuzz: treat unmodeled testserver routes as rejections, not failures The schema fuzzer generates resource types the testserver may not model. Hitting an unregistered route made the testserver call t.Errorf, failing the whole fuzz run for a coverage gap rather than a CLI bug. Add a testserver IgnoreUnhandledRequests option (exposed as a test.toml field, plumbed through startLocalServer) that returns 501 and logs the gap instead of failing. The invariant fuzz script already treats a non-zero result before INPUT_CONFIG_OK as a rejection, so such configs are now skipped cleanly. Curated tests leave the flag false so genuine missing-handler bugs stay loud. Covered by a testserver unit test. --- acceptance/bundle/invariant/fuzz/test.toml | 5 +++ acceptance/internal/config.go | 6 ++++ acceptance/internal/prepare_server.go | 4 ++- libs/testserver/server.go | 15 +++++++- libs/testserver/server_test.go | 42 ++++++++++++++++++++++ 5 files changed, 70 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index a27ecdec440..ffa4ede2e36 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -2,6 +2,11 @@ # generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] +# The fuzzer can emit resource types the testserver does not model. A missing +# handler is a coverage gap, not a CLI bug, so return 501 (the config is then +# rejected) instead of failing the whole run. +IgnoreUnhandledRequests = true + # Run the real invariant test script for each target. migrate ignores # DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index f981e790e0b..a815b76aba5 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -83,6 +83,12 @@ type TestConfig struct { // out.requests.txt RecordRequests *bool + // Return 501 for requests with no registered handler instead of failing the + // test. Used by the schema fuzzer, which generates resource types the + // testserver may not model: a missing handler is a coverage gap that should + // reject the config, not a CLI bug that fails the run. + IgnoreUnhandledRequests *bool + // List of request headers to include when recording requests. IncludeRequestHeaders []string diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 3b1be7a79b3..5c1c6591631 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -151,7 +151,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o // Default case. Start a dedicated local server for the test with the server stubs configured // as overrides. - host := startLocalServer(t, config.Server, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir) + host := startLocalServer(t, config.Server, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir, isTruePtr(config.IgnoreUnhandledRequests)) cfg := &sdkconfig.Config{ Host: host, Token: token, @@ -204,8 +204,10 @@ func startLocalServer(t *testing.T, logRequests bool, includeHeaders []string, outputDir string, + ignoreUnhandledRequests bool, ) string { s := testserver.New(t) + s.IgnoreUnhandledRequests = ignoreUnhandledRequests // Record API requests in out.requests.txt if RecordRequests is true // in test.toml diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5ae3141bfd2..72be4612417 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -73,6 +73,13 @@ type Server struct { RequestCallback func(request *Request) ResponseCallback func(request *Request, response *EncodedResponse) + + // IgnoreUnhandledRequests turns a request with no registered handler into a + // plain 501 instead of a test failure. The schema fuzzer generates resource + // types the testserver may not model; a missing handler there is a coverage + // gap, so the caller sees the 501 and rejects the config rather than failing + // the whole run. Curated tests leave this false so real gaps stay loud. + IgnoreUnhandledRequests bool } type Request struct { @@ -278,7 +285,12 @@ func New(t testutil.TestingT) *Server { body = fmt.Sprintf("[%d bytes] %s", len(bodyBytes), bodyBytes) } - t.Errorf(`No handler for URL: %s + if s.IgnoreUnhandledRequests { + // Coverage gap, not a CLI bug: log for visibility but let the 501 + // below flow back so the caller can reject the config. + t.Logf("No handler for URL (ignored): %s", r.URL) + } else { + t.Errorf(`No handler for URL: %s Body: %s For acceptance tests, add this to test.toml: @@ -287,6 +299,7 @@ Pattern = %q Response.Body = '' # Response.StatusCode = `, r.URL, body, pattern) + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotImplemented) diff --git a/libs/testserver/server_test.go b/libs/testserver/server_test.go index 6c6dfe8c160..141a873c9bd 100644 --- a/libs/testserver/server_test.go +++ b/libs/testserver/server_test.go @@ -3,12 +3,54 @@ package testserver_test import ( "net/http" "net/http/httptest" + "sync" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// recordingT wraps a real TestingT but records Errorf calls instead of failing, +// so a test can assert whether the server would have failed the run. +type recordingT struct { + testutil.TestingT + mu sync.Mutex + errCount int +} + +func (r *recordingT) Errorf(format string, args ...any) { + r.mu.Lock() + defer r.mu.Unlock() + r.errCount++ +} + +func (r *recordingT) errors() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.errCount +} + +func TestIgnoreUnhandledRequests(t *testing.T) { + for _, ignore := range []bool{false, true} { + rt := &recordingT{TestingT: t} + s := testserver.New(rt) + s.IgnoreUnhandledRequests = ignore + + resp, err := http.Get(s.URL + "/api/2.0/no-such-endpoint") + require.NoError(t, err) + assert.Equal(t, http.StatusNotImplemented, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + + if ignore { + assert.Zero(t, rt.errors(), "unhandled request must not fail the test when ignored") + } else { + assert.Positive(t, rt.errors(), "unhandled request must fail the test by default") + } + } +} + func TestIsLocalhostProbe(t *testing.T) { tests := []struct { name string From c043cd159c690eed9f03e844896cfe8eb98ee138 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 13 Jul 2026 14:50:13 +0000 Subject: [PATCH 39/53] acc/fuzz: bound seed execution by time to separate hangs from slow runs Fuzz variants had two time failure modes that both read as bugs: a single seed could hang indefinitely, and a slow-but-progressing variant (mutate + drift at ~30s/seed) could exceed the per-test timeout across a large seed count. Both surfaced only as an opaque "test timed out". Add a per-seed cap (FUZZ_SEED_TIMEOUT, default 180s) enforced via GNU timeout: a seed past the cap is SIGQUIT'd (Go dumps goroutines, so a real deadlock is diagnosable) then SIGKILL'd, and reported as a hang with a reproduce hint, distinct from a drift bug. Where timeout is unavailable (macOS/Windows) seeds run uncapped as before, and FUZZ_SEED_TIMEOUT=0 disables the cap for live inspection. Add an optional overall budget (FUZZ_TIME_BUDGET): the loop stops launching new seeds past it and exits cleanly, so a slow variant tests as many seeds as fit instead of being force-killed. The nightly task sets it under a raised per-variant Timeout; the committed run keeps its defaults and empty output. --- Taskfile.yml | 5 ++ acceptance/bundle/invariant/fuzz/script | 73 ++++++++++++++++++---- acceptance/bundle/invariant/fuzz/test.toml | 5 ++ 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index d5e96842b1e..d4b77d9e803 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -740,6 +740,11 @@ tasks: # narrow it via FUZZ_SEED_START/COUNT. export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" + # Slow variants (mutate + drift at ~30s/seed) can't finish 200 seeds inside the + # per-variant Timeout, so bound each variant's wall clock (with headroom under + # that Timeout for the last seed): it tests as many seeds as fit and passes, + # instead of being force-killed and read as a bug. + export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" {{.GO_TOOL}} gotestsum \ --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ --no-summary=skipped \ diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index d950bb89f89..4d9eff1a5c5 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -9,6 +9,20 @@ START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" +# Per-seed wall-clock cap. Normal seeds finish in seconds, so one that blows past this +# generous budget is stuck, not slow: SIGQUIT it first (Go dumps goroutines, so a real +# deadlock is diagnosable from the seed's LOG.*) then SIGKILL, and report it as a hang +# instead of letting it silently consume the whole per-test timeout. Enforced with GNU +# `timeout`; where that is absent (e.g. macOS/Windows) seeds run uncapped as before. +SEED_TIMEOUT="${FUZZ_SEED_TIMEOUT:-180}" + +# Optional overall wall-clock budget in seconds. A slow-but-progressing variant (mutate +# with drift-checking runs ~30s/seed) can exceed the per-test timeout across a large +# FUZZ_SEED_COUNT; when set, stop launching new seeds past this many seconds and exit +# cleanly having tested as many as fit, rather than being force-killed and read as a +# failure. Unset (the committed run) means no cap. +BUDGET="${FUZZ_TIME_BUDGET:-}" + if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then export SKIP_DRIFT_CHECK=1 fi @@ -23,23 +37,52 @@ cat LOG.schema.err | contains.py '!panic:' '!internal error' > /dev/null # Fail loud on a schema type the generator can't produce (the loop below would hide it). check_schema_types.py --schema schema.json +# One seed's worth of work, factored out so it can run either directly or under `timeout`. +seed_body() { + cd "$1" + # The generator points file_path/source_code_path fields at these fixtures + # (see gen_fuzz_config.py *_BY_RESOURCE), so make them resolvable from the + # bundle root, matching how the curated invariant scripts stage data/. + cp -r "$TESTDIR/../data/." . + export FUZZ_SEED="$2" + export FUZZ_SCHEMA="../schema.json" + source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" +} + +# `timeout` spawns a fresh bash that does not inherit shell functions, so export the +# acceptance helpers (trace, title, envsubst, readplanarg, ...) and seed_body for it. +export -f $(compgen -A function) + +# Run one seed, capped at SEED_TIMEOUT when `timeout` is available. A subshell (or the +# fresh bash under timeout) keeps a generator crash or invariant failure contained. +run_seed() { + local dir="$1" seed="$2" + # SEED_TIMEOUT=0 (or no `timeout` binary) runs the seed uncapped, matching the + # reproduce hint that disables the watchdog so a hang can be inspected live. + if [ "$SEED_TIMEOUT" != "0" ] && command -v timeout > /dev/null 2>&1; then + timeout --signal=QUIT --kill-after=10s "$SEED_TIMEOUT" \ + bash -euo pipefail -c 'seed_body "$@"' _ "$dir" "$seed" > "$dir/LOG.check" 2>&1 + else + ( seed_body "$dir" "$seed" ) > "$dir/LOG.check" 2>&1 + fi +} + for ((offset = 0; offset < COUNT; offset++)); do + # Stop before the per-test timeout kills us mid-seed; the variant then passes, + # having tested as many seeds as the budget allowed. This is a clean stop, not a + # failure, so keep it out of the compared stdout/stderr (the committed run asserts + # empty output); the nightly can read it from the preserved work dir. + if [ -n "$BUDGET" ] && [ "$SECONDS" -ge "$BUDGET" ]; then + echo "fuzz: stopping after $offset/$COUNT seeds; hit FUZZ_TIME_BUDGET=${BUDGET}s" > LOG.budget + break + fi + seed=$((START + offset)) dir="seed-$seed" mkdir -p "$dir" - # Subshell so a generator crash or invariant failure is contained per seed. set +e - ( - cd "$dir" - # The generator points file_path/source_code_path fields at these fixtures - # (see gen_fuzz_config.py *_BY_RESOURCE), so make them resolvable from the - # bundle root, matching how the curated invariant scripts stage data/. - cp -r "$TESTDIR/../data/." . - export FUZZ_SEED="$seed" - export FUZZ_SCHEMA="../schema.json" - source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" - ) > "$dir/LOG.check" 2>&1 + run_seed "$dir" "$seed" rc=$? set -e @@ -47,6 +90,14 @@ for ((offset = 0; offset < COUNT; offset++)); do continue fi + # GNU timeout exits 124 when it had to signal the seed (137 if the SIGKILL backstop + # fired): the seed hung past SEED_TIMEOUT. Report it as a hang, distinct from a drift + # bug, with any goroutine dump preserved in the seed's LOG.* for triage. + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 + exit 1 + fi + bug="" # A panic anywhere is a bug even if the CLI then rejects the config. diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index ffa4ede2e36..0985bd7a2fd 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -2,6 +2,11 @@ # generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] +# Raise the inherited 10m cap so the nightly FUZZ_TIME_BUDGET (script) has room to +# stop cleanly, plus the tail of a final in-flight seed, before this fires. The +# committed run (5 seeds, no drift) finishes in seconds regardless. +Timeout = '20m' + # The fuzzer can emit resource types the testserver does not model. A missing # handler is a coverage gap, not a CLI bug, so return 501 (the config is then # rejected) instead of failing the whole run. From 92e85b11f4b0ab25abbd8fad09382082e245ee2b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 13 Jul 2026 15:02:59 +0000 Subject: [PATCH 40/53] acc/fuzz: record per-seed classification and a per-variant tally Triaging a fuzz run meant hand-parsing every seed's LOG.* across preserved temp dirs to sort deployed / rejected / testserver-gap / hang / invariant bug. Reuse the classification the loop already computes and append one machine-readable line per seed to LOG.summary, plus a per-variant totals block on a clean run. A 501 "No stub found" is tracked as a coverage gap distinct from a genuine config rejection. LOG.summary is a file, not stdout, so the committed run's empty-output assertion is unaffected. --- acceptance/bundle/invariant/fuzz/script | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 4d9eff1a5c5..47c6b715819 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -67,6 +67,13 @@ run_seed() { fi } +# Append one machine-readable line per seed to LOG.summary so a run can be tallied +# (deployed / rejected / gap / hang / bug) without grepping every seed's logs. It is a +# file, not stdout, so the committed run's empty-output assertion is unaffected. +record() { + echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate} count=${FUZZ_RESOURCE_COUNT:-1}" >> LOG.summary +} + for ((offset = 0; offset < COUNT; offset++)); do # Stop before the per-test timeout kills us mid-seed; the variant then passes, # having tested as many seeds as the budget allowed. This is a clean stop, not a @@ -87,6 +94,7 @@ for ((offset = 0; offset < COUNT; offset++)); do set -e if [ "$rc" -eq 0 ]; then + record deployed "$seed" continue fi @@ -94,6 +102,7 @@ for ((offset = 0; offset < COUNT; offset++)); do # fired): the seed hung past SEED_TIMEOUT. Report it as a hang, distinct from a drift # bug, with any goroutine dump preserved in the seed's LOG.* for triage. if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + record hang "$seed" echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 exit 1 fi @@ -112,7 +121,29 @@ for ((offset = 0; offset < COUNT; offset++)); do fi if [ -n "$bug" ]; then + record bug "$seed" echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 exit 1 fi + + # Not a bug. A 501 "No stub found" means the config hit a route the testserver does + # not model (a coverage gap, see IgnoreUnhandledRequests); anything else failing + # before INPUT_CONFIG_OK is a genuine config rejection. + if grep -qs "No stub found for pattern" "$dir"/LOG.*; then + record gap "$seed" + else + record rejected "$seed" + fi done + +# Per-variant tally so the nightly can sum outcomes across variants at a glance rather +# than re-deriving them from every seed's logs. Only reached on a clean run; a bug or +# hang exits above, and the per-seed lines already recorded remain for triage. +if [ -f LOG.summary ]; then + # Snapshot the counts before appending the header, else awk would also count it. + totals=$(awk '{print $1}' LOG.summary | sort | uniq -c) + { + echo "--- totals ---" + echo "$totals" + } >> LOG.summary +fi From 927cff40d708e8989cd04eede2e398e78a600b65 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 13 Jul 2026 15:08:57 +0000 Subject: [PATCH 41/53] acc/fuzz: tighten comments added in this PR Trim the multi-line explanations added across the fuzz harness changes to short, why-focused notes; no behavior change. --- Taskfile.yml | 6 +-- acceptance/bin/edit_fuzz_config.py | 18 ++++---- acceptance/bundle/invariant/fuzz/script | 49 ++++++++-------------- acceptance/bundle/invariant/fuzz/test.toml | 10 ++--- acceptance/internal/config.go | 7 ++-- libs/testserver/server.go | 11 ++--- 6 files changed, 37 insertions(+), 64 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index d4b77d9e803..e50d62a1b09 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -740,10 +740,8 @@ tasks: # narrow it via FUZZ_SEED_START/COUNT. export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" - # Slow variants (mutate + drift at ~30s/seed) can't finish 200 seeds inside the - # per-variant Timeout, so bound each variant's wall clock (with headroom under - # that Timeout for the last seed): it tests as many seeds as fit and passes, - # instead of being force-killed and read as a bug. + # Slow variants can't finish 200 seeds inside the per-variant Timeout, so bound + # each variant's wall clock: it tests as many seeds as fit and passes, not killed. export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" {{.GO_TOOL}} gotestsum \ --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ diff --git a/acceptance/bin/edit_fuzz_config.py b/acceptance/bin/edit_fuzz_config.py index e2449c4e60f..a71cf82771f 100755 --- a/acceptance/bin/edit_fuzz_config.py +++ b/acceptance/bin/edit_fuzz_config.py @@ -3,11 +3,9 @@ Edit a `comment`/`description` scalar in a generated databricks.yml so a redeploy is an in-place update, not a recreate. Used by the `update` invariant. -Most resources take a `comment`/`description` edit as an in-place update, but some -classify it as immutable (recreate on change) in the direct engine's resource spec -(e.g. model_serving_endpoints.description). Editing such a field replans as a recreate, -which the update invariant would wrongly flag as a bug, so we skip it and pick a mutable -field elsewhere (or report none). +Some resources classify `description` as immutable (recreate on change) in the direct +engine spec (e.g. model_serving_endpoints); editing that replans as a recreate the update +invariant would wrongly flag, so skip it and pick a mutable field (or report none). gen_fuzz_config.py emits one scalar per line as `key: `, so a regex match suffices (no YAML dependency for the edit itself); the immutable set is read from resources.yml. @@ -31,18 +29,16 @@ NEW_VALUE = '"fuzz_edited_value"' -# resources.yml is the source of truth for field mutability; acceptance/bin sits two -# levels below the repo root, and the real dir (not a copy) is on PATH, so __file__ -# resolves here. +# resources.yml is the source of truth for field mutability. acceptance/bin is on PATH as +# the real dir (not a copy), so __file__ resolves two levels below the repo root. RESOURCES_YML = Path(__file__).resolve().parents[2] / "bundle" / "direct" / "dresources" / "resources.yml" def immutable_fields(): """Map resource type -> set of fields that recreate on change (immutable). - resources.yml has a fixed two-space layout (`resources:` -> ` :` -> - ` recreate_on_changes:` -> ` - field: `), so a small line parser - avoids a YAML dependency the harness's Python does not have. + Hand-rolled line parser over resources.yml's fixed two-space layout, avoiding a YAML + dependency the harness's Python lacks. """ result = {} current_type = None diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 47c6b715819..dfde17d14b0 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -9,18 +9,12 @@ START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" -# Per-seed wall-clock cap. Normal seeds finish in seconds, so one that blows past this -# generous budget is stuck, not slow: SIGQUIT it first (Go dumps goroutines, so a real -# deadlock is diagnosable from the seed's LOG.*) then SIGKILL, and report it as a hang -# instead of letting it silently consume the whole per-test timeout. Enforced with GNU -# `timeout`; where that is absent (e.g. macOS/Windows) seeds run uncapped as before. +# Per-seed cap: a seed past this generous budget is stuck, not slow, so SIGQUIT (Go dumps +# goroutines for triage) then SIGKILL and flag a hang. Needs GNU timeout; else uncapped. SEED_TIMEOUT="${FUZZ_SEED_TIMEOUT:-180}" -# Optional overall wall-clock budget in seconds. A slow-but-progressing variant (mutate -# with drift-checking runs ~30s/seed) can exceed the per-test timeout across a large -# FUZZ_SEED_COUNT; when set, stop launching new seeds past this many seconds and exit -# cleanly having tested as many as fit, rather than being force-killed and read as a -# failure. Unset (the committed run) means no cap. +# Optional overall budget (seconds): stop starting new seeds past it and exit cleanly, so +# a slow-but-progressing variant isn't force-killed and read as a failure. Unset = no cap. BUDGET="${FUZZ_TIME_BUDGET:-}" if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then @@ -49,16 +43,13 @@ seed_body() { source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" } -# `timeout` spawns a fresh bash that does not inherit shell functions, so export the -# acceptance helpers (trace, title, envsubst, readplanarg, ...) and seed_body for it. +# timeout spawns a fresh bash without our shell functions, so export them (and seed_body). export -f $(compgen -A function) -# Run one seed, capped at SEED_TIMEOUT when `timeout` is available. A subshell (or the -# fresh bash under timeout) keeps a generator crash or invariant failure contained. +# Run one seed, capped at SEED_TIMEOUT when `timeout` is available. run_seed() { local dir="$1" seed="$2" - # SEED_TIMEOUT=0 (or no `timeout` binary) runs the seed uncapped, matching the - # reproduce hint that disables the watchdog so a hang can be inspected live. + # SEED_TIMEOUT=0 or no timeout binary: run uncapped (matches the hang reproduce hint). if [ "$SEED_TIMEOUT" != "0" ] && command -v timeout > /dev/null 2>&1; then timeout --signal=QUIT --kill-after=10s "$SEED_TIMEOUT" \ bash -euo pipefail -c 'seed_body "$@"' _ "$dir" "$seed" > "$dir/LOG.check" 2>&1 @@ -67,18 +58,15 @@ run_seed() { fi } -# Append one machine-readable line per seed to LOG.summary so a run can be tallied -# (deployed / rejected / gap / hang / bug) without grepping every seed's logs. It is a -# file, not stdout, so the committed run's empty-output assertion is unaffected. +# One machine-readable line per seed so a run is tallyable without grepping logs. A file, +# not stdout, so the committed run's empty-output assertion holds. record() { echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate} count=${FUZZ_RESOURCE_COUNT:-1}" >> LOG.summary } for ((offset = 0; offset < COUNT; offset++)); do - # Stop before the per-test timeout kills us mid-seed; the variant then passes, - # having tested as many seeds as the budget allowed. This is a clean stop, not a - # failure, so keep it out of the compared stdout/stderr (the committed run asserts - # empty output); the nightly can read it from the preserved work dir. + # Stop before the per-test timeout kills us mid-seed; a clean stop, not a failure, so + # log to a file rather than the compared stdout/stderr. if [ -n "$BUDGET" ] && [ "$SECONDS" -ge "$BUDGET" ]; then echo "fuzz: stopping after $offset/$COUNT seeds; hit FUZZ_TIME_BUDGET=${BUDGET}s" > LOG.budget break @@ -98,9 +86,8 @@ for ((offset = 0; offset < COUNT; offset++)); do continue fi - # GNU timeout exits 124 when it had to signal the seed (137 if the SIGKILL backstop - # fired): the seed hung past SEED_TIMEOUT. Report it as a hang, distinct from a drift - # bug, with any goroutine dump preserved in the seed's LOG.* for triage. + # timeout exits 124 (137 if the SIGKILL backstop fired): the seed hung. Report a hang, + # distinct from a drift bug; any goroutine dump is preserved in the seed's LOG.*. if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then record hang "$seed" echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 @@ -126,9 +113,8 @@ for ((offset = 0; offset < COUNT; offset++)); do exit 1 fi - # Not a bug. A 501 "No stub found" means the config hit a route the testserver does - # not model (a coverage gap, see IgnoreUnhandledRequests); anything else failing - # before INPUT_CONFIG_OK is a genuine config rejection. + # A 501 "No stub found" is a testserver coverage gap (see IgnoreUnhandledRequests); + # anything else before INPUT_CONFIG_OK is a genuine config rejection. if grep -qs "No stub found for pattern" "$dir"/LOG.*; then record gap "$seed" else @@ -136,9 +122,8 @@ for ((offset = 0; offset < COUNT; offset++)); do fi done -# Per-variant tally so the nightly can sum outcomes across variants at a glance rather -# than re-deriving them from every seed's logs. Only reached on a clean run; a bug or -# hang exits above, and the per-seed lines already recorded remain for triage. +# Per-variant tally for at-a-glance triage. Reached only on a clean run; a bug/hang exits +# above with the per-seed lines already recorded. if [ -f LOG.summary ]; then # Snapshot the counts before appending the header, else awk would also count it. totals=$(awk '{print $1}' LOG.summary | sort | uniq -c) diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 0985bd7a2fd..afd3a0bcc5c 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -2,14 +2,12 @@ # generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] -# Raise the inherited 10m cap so the nightly FUZZ_TIME_BUDGET (script) has room to -# stop cleanly, plus the tail of a final in-flight seed, before this fires. The -# committed run (5 seeds, no drift) finishes in seconds regardless. +# Raise the inherited 10m cap so the nightly FUZZ_TIME_BUDGET (script) has room, plus the +# final seed's tail. The committed run (5 seeds, no drift) finishes in seconds regardless. Timeout = '20m' -# The fuzzer can emit resource types the testserver does not model. A missing -# handler is a coverage gap, not a CLI bug, so return 501 (the config is then -# rejected) instead of failing the whole run. +# The fuzzer can emit resource types the testserver doesn't model; a missing handler is a +# coverage gap, so return 501 (config rejected) instead of failing the whole run. IgnoreUnhandledRequests = true # Run the real invariant test script for each target. migrate ignores diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index a815b76aba5..37386c78e15 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -83,10 +83,9 @@ type TestConfig struct { // out.requests.txt RecordRequests *bool - // Return 501 for requests with no registered handler instead of failing the - // test. Used by the schema fuzzer, which generates resource types the - // testserver may not model: a missing handler is a coverage gap that should - // reject the config, not a CLI bug that fails the run. + // Return 501 for a request with no handler instead of failing the test. The fuzzer + // emits resource types the testserver may not model; a missing handler is a gap, not + // a bug. IgnoreUnhandledRequests *bool // List of request headers to include when recording requests. diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 72be4612417..475c92164b9 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -74,11 +74,9 @@ type Server struct { RequestCallback func(request *Request) ResponseCallback func(request *Request, response *EncodedResponse) - // IgnoreUnhandledRequests turns a request with no registered handler into a - // plain 501 instead of a test failure. The schema fuzzer generates resource - // types the testserver may not model; a missing handler there is a coverage - // gap, so the caller sees the 501 and rejects the config rather than failing - // the whole run. Curated tests leave this false so real gaps stay loud. + // IgnoreUnhandledRequests returns 501 for a request with no handler instead of failing + // the test: the fuzzer emits resource types the testserver may not model, so the caller + // rejects the config. Curated tests leave it false so real gaps stay loud. IgnoreUnhandledRequests bool } @@ -286,8 +284,7 @@ func New(t testutil.TestingT) *Server { } if s.IgnoreUnhandledRequests { - // Coverage gap, not a CLI bug: log for visibility but let the 501 - // below flow back so the caller can reject the config. + // Coverage gap, not a CLI bug: log it but return the 501 so the caller can reject. t.Logf("No handler for URL (ignored): %s", r.URL) } else { t.Errorf(`No handler for URL: %s From a35eb84e16c2820a73884cd04f46481ee8c80276 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 14 Jul 2026 07:31:44 +0000 Subject: [PATCH 42/53] invariant: regenerate out.test.toml for inherited GOOSOnPR and job_run config The parent test.toml sets GOOSOnPR.windows/darwin=false and adds job_run.yml.tmpl, but the per-directory out.test.toml snapshots were stale, failing the CI "detected changed or new files" check. --- acceptance/bundle/invariant/canonical/out.test.toml | 3 +++ acceptance/bundle/invariant/destroy_recreate/out.test.toml | 3 +++ acceptance/bundle/invariant/fuzz/out.test.toml | 2 ++ acceptance/bundle/invariant/redeploy/out.test.toml | 3 +++ acceptance/bundle/invariant/update/out.test.toml | 3 +++ 5 files changed, 14 insertions(+) diff --git a/acceptance/bundle/invariant/canonical/out.test.toml b/acceptance/bundle/invariant/canonical/out.test.toml index 35535594224..8901423c85f 100644 --- a/acceptance/bundle/invariant/canonical/out.test.toml +++ b/acceptance/bundle/invariant/canonical/out.test.toml @@ -1,6 +1,8 @@ Local = true Cloud = true RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", @@ -22,6 +24,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_task.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_recreate/out.test.toml b/acceptance/bundle/invariant/destroy_recreate/out.test.toml index 35535594224..8901423c85f 100644 --- a/acceptance/bundle/invariant/destroy_recreate/out.test.toml +++ b/acceptance/bundle/invariant/destroy_recreate/out.test.toml @@ -1,6 +1,8 @@ Local = true Cloud = true RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", @@ -22,6 +24,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_task.yml.tmpl", diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index d8d85f175d1..868a98f0bb8 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -1,6 +1,8 @@ Local = true Cloud = true RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.FUZZ_MODE = ["generate", "mutate"] EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] diff --git a/acceptance/bundle/invariant/redeploy/out.test.toml b/acceptance/bundle/invariant/redeploy/out.test.toml index 35535594224..8901423c85f 100644 --- a/acceptance/bundle/invariant/redeploy/out.test.toml +++ b/acceptance/bundle/invariant/redeploy/out.test.toml @@ -1,6 +1,8 @@ Local = true Cloud = true RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", @@ -22,6 +24,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_task.yml.tmpl", diff --git a/acceptance/bundle/invariant/update/out.test.toml b/acceptance/bundle/invariant/update/out.test.toml index 35535594224..8901423c85f 100644 --- a/acceptance/bundle/invariant/update/out.test.toml +++ b/acceptance/bundle/invariant/update/out.test.toml @@ -1,6 +1,8 @@ Local = true Cloud = true RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", @@ -22,6 +24,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", "job_run_job_ref.yml.tmpl", "job_with_depends_on.yml.tmpl", "job_with_task.yml.tmpl", From 5e8a03e73e072bfe10442549698b89b1d879c0da Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 15 Jul 2026 13:22:03 +0000 Subject: [PATCH 43/53] acc/fuzz: real oracle, coverage, less rejection waste, clean truncation Four improvements to the bundle schema fuzzer: - Oracle: in fuzz mode (SKIP_DRIFT_CHECK) no_drift/redeploy now assert plan determinism (two consecutive `plan` reads must be byte-identical) instead of only no-panic. This is independent of fake-server fidelity, unlike no-drift or no-destructive-drift (an unchanged config's recreate is just a local-vs-remote representation mismatch on an immutable field). redeploy also panic-scans its second deploy while tolerating fidelity-driven redeploy failures. - Coverage: add `task test-fuzz-cover` to run the corpus under -cover and report per-package coverage (0.0% = never exercised), plus opt-in FUZZ_CORPUS_DIR to persist deployable configs as a generator-independent regression corpus. - Rejection waste: emit non-ASCII scalars as literal UTF-8 (ensure_ascii=False) so hostile values reach bundle logic instead of dying at the YAML parser as an invalid surrogate-pair escape (was the single largest rejection bucket). - Truncation: FUZZ_TIME_BUDGET now defaults on (900s, under the 20m Timeout) so a slow variant stops cleanly and passes rather than being force-killed into a false failure; FUZZ_TIME_BUDGET=0 restores an uncapped run. --- Taskfile.yml | 39 ++++++++++++++++++--- acceptance/bin/gen_fuzz_config.py | 16 +++++++-- acceptance/bundle/invariant/fuzz/script | 23 +++++++++--- acceptance/bundle/invariant/no_drift/script | 34 +++++++++++++----- acceptance/bundle/invariant/redeploy/script | 25 ++++++++++++- 5 files changed, 116 insertions(+), 21 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index e50d62a1b09..0737ca612f6 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -737,18 +737,49 @@ tasks: cmds: - | # Wider window than the committed run, with drift checking on; a repro can - # narrow it via FUZZ_SEED_START/COUNT. + # narrow it via FUZZ_SEED_START/COUNT. Slow variants can't finish 200 seeds inside + # the per-variant Timeout, but the fuzz script's default FUZZ_TIME_BUDGET stops each + # variant cleanly (as many seeds as fit, then pass) rather than being force-killed. export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" - # Slow variants can't finish 200 seeds inside the per-variant Timeout, so bound - # each variant's wall clock: it tests as many seeds as fit and passes, not killed. - export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" {{.GO_TOOL}} gotestsum \ --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ --no-summary=skipped \ --packages ./acceptance/... \ -- -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz" + test-fuzz-cover: + desc: Run the schema fuzzer under coverage and report which CLI packages it exercises + # No `sources:` fingerprint: like test-fuzz, the window depends on FUZZ_* env vars. + cmds: + - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/ + - mkdir -p ./acceptance/build/cover-fuzz-merged/ + - | + # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, + # so every fuzzed `bundle` invocation drops counter files we can aggregate. Drift + # off (no FUZZ_CHECK_DRIFT) so the run exercises the full deploy/plan path for as + # many seeds as possible instead of stopping on the first fake-server drift. + export CLI_GOCOVERDIR=build/cover-fuzz + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}" + export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" + unset FUZZ_CHECK_DRIFT || true + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./acceptance/... \ + -- -timeout=${LOCAL_TIMEOUT:-40m} -run "TestAccept/bundle/invariant/fuzz" || true + - "go tool covdata merge -i $(printf '%s,' acceptance/build/cover-fuzz/* | sed 's/,$//') -o acceptance/build/cover-fuzz-merged/" + - go tool covdata textfmt -i acceptance/build/cover-fuzz-merged -o coverage-fuzz.txt + - | + echo "== total CLI coverage exercised by the fuzz corpus ==" + go tool cover -func=coverage-fuzz.txt | awk '/^total:/{print $NF}' + echo + echo "== bundle/cmd packages by coverage (ascending; 0.0% = never exercised) ==" + go tool covdata percent -i=acceptance/build/cover-fuzz-merged \ + | awk '/coverage:/{p=$3; gsub(/%/,"",p); sub("github.com/databricks/cli/","",$1); printf "%6.1f%% %s\n", p, $1}' \ + | grep -E ' (bundle|cmd/bundle)/' \ + | sort -n + # --- Integration tests --- integration: diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 147930fbb5e..9aca9166cce 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -485,7 +485,7 @@ def to_yaml(obj, indent=0, list_item=False): if isinstance(v, (dict, list)) and v: out += f"{prefix}{k}:\n" + to_yaml(v, child_indent) else: - out += f"{prefix}{k}: {json.dumps(v)}\n" + out += f"{prefix}{k}: {dump_scalar(v)}\n" first = False return out if isinstance(obj, list): @@ -496,9 +496,19 @@ def to_yaml(obj, indent=0, list_item=False): if isinstance(item, (dict, list)): out += to_yaml(item, indent, list_item=True) else: - out += f"{pad}- {json.dumps(item)}\n" + out += f"{pad}- {dump_scalar(item)}\n" return out - return f"{pad}{json.dumps(obj)}\n" + return f"{pad}{dump_scalar(obj)}\n" + + +def dump_scalar(v): + # ensure_ascii=False keeps non-ASCII as literal UTF-8. The default would escape an + # astral char (e.g. the 🚀 probe) into a UTF-16 surrogate pair (\ud83d\ude80), which + # YAML's parser rejects as an "invalid Unicode character escape code" -- so the config + # dies at parse time and never reaches bundle logic. A literal UTF-8 scalar is valid + # YAML and exercises the CLI's actual unicode handling instead. Control chars (\n, \t) + # are still escaped by json.dumps regardless, which YAML accepts. + return json.dumps(v, ensure_ascii=False) def main(): diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index dfde17d14b0..86cb6bfea05 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -13,9 +13,14 @@ COUNT="${FUZZ_SEED_COUNT:-5}" # goroutines for triage) then SIGKILL and flag a hang. Needs GNU timeout; else uncapped. SEED_TIMEOUT="${FUZZ_SEED_TIMEOUT:-180}" -# Optional overall budget (seconds): stop starting new seeds past it and exit cleanly, so -# a slow-but-progressing variant isn't force-killed and read as a failure. Unset = no cap. -BUDGET="${FUZZ_TIME_BUDGET:-}" +# Overall budget (seconds): stop starting new seeds past it and exit cleanly, so a +# slow-but-progressing variant (the mutate/deploy combos never finish a large seed window) +# is not force-killed at the per-script Timeout and read as a failure. Defaults on rather +# than opt-in so a direct `go test` run truncates cleanly too, not just `task test-fuzz`. +# 900s leaves margin under the 20m Timeout in test.toml for the last-started seed (capped at +# SEED_TIMEOUT) plus teardown; keep it comfortably below Timeout - SEED_TIMEOUT. Set +# FUZZ_TIME_BUDGET=0 to disable the cap (e.g. a completionist run with a raised Timeout). +BUDGET="${FUZZ_TIME_BUDGET:-900}" if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then export SKIP_DRIFT_CHECK=1 @@ -66,8 +71,8 @@ record() { for ((offset = 0; offset < COUNT; offset++)); do # Stop before the per-test timeout kills us mid-seed; a clean stop, not a failure, so - # log to a file rather than the compared stdout/stderr. - if [ -n "$BUDGET" ] && [ "$SECONDS" -ge "$BUDGET" ]; then + # log to a file rather than the compared stdout/stderr. BUDGET=0 disables the cap. + if [ "$BUDGET" != "0" ] && [ "$SECONDS" -ge "$BUDGET" ]; then echo "fuzz: stopping after $offset/$COUNT seeds; hit FUZZ_TIME_BUDGET=${BUDGET}s" > LOG.budget break fi @@ -83,6 +88,14 @@ for ((offset = 0; offset < COUNT; offset++)); do if [ "$rc" -eq 0 ]; then record deployed "$seed" + # Optional generator-independent regression corpus: persist configs that actually + # deployed, so runs accumulate known-good inputs that stay valid (and replayable) + # even after the generator changes. Unset by default, so the committed run and its + # empty-output assertion are unaffected. + if [ -n "${FUZZ_CORPUS_DIR:-}" ]; then + mkdir -p "$FUZZ_CORPUS_DIR" + cp "$dir/databricks.yml" "$FUZZ_CORPUS_DIR/${FUZZ_MODE:-generate}-${FUZZ_TARGET:-no_drift}-count${FUZZ_RESOURCE_COUNT:-1}-seed${seed}.yml" + fi continue fi diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index b5d2a569d2d..bb5b54f331d 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -43,8 +43,13 @@ cleanup() { trap cleanup EXIT -$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null +# Only compute a plan up front when the READPLAN variant actually consumes it via --plan. +# Without READPLAN, deploy computes its own plan internally, so a separate `bundle plan` +# invocation is pure waste. +if [[ -n "$READPLAN" ]]; then + $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err + cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null +fi trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy deploy_rc=$? @@ -59,14 +64,27 @@ deployed=1 echo INPUT_CONFIG_OK # A fuzzed config can deploy yet legitimately differ from the fake server, so the -# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. +# fuzzer sets SKIP_DRIFT_CHECK to swap the exact no-drift check for the weaker but +# fidelity-independent plan-determinism oracle; curated configs check full drift. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # Check both text and JSON plan for no changes - # Note, expect that there maybe more than one resource unchanged + # JSON plan asserts every action is "skip" -- a strict superset of the text + # renderer's "Plan: 0 to add, 0 to change, 0 to delete" summary. $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null verify_no_drift.py LOG.planjson - - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan - cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null +else + # Fuzz mode: the exact no-drift check false-positives because the fake server does + # not round-trip every field, and "no destructive re-plan" is no better -- for an + # unchanged config any recreate is just a local-vs-remote representation mismatch on + # an immutable field (fake-server fidelity), not real churn. So assert the one + # invariant that is independent of server fidelity: planning is deterministic. Two + # consecutive plans of the same deployed state must be byte-identical; a diff means + # nondeterministic planning/serialization (unstable map order, per-run randomness), a + # real bug. `|| true`: a plan that fails on an unstubbed read is a coverage gap, not a + # bug -- both plans are then empty and the diff trivially passes. + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err || true + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err || true + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff fi diff --git a/acceptance/bundle/invariant/redeploy/script b/acceptance/bundle/invariant/redeploy/script index 18382b44b4a..af5bddd50e1 100644 --- a/acceptance/bundle/invariant/redeploy/script +++ b/acceptance/bundle/invariant/redeploy/script @@ -59,7 +59,8 @@ deployed=1 echo INPUT_CONFIG_OK # A fuzzed config can deploy yet legitimately fail to redeploy or differ, so the fuzzer -# sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check the no-op. +# sets SKIP_DRIFT_CHECK to swap the exact no-op check for the weaker but +# fidelity-independent plan-determinism oracle; curated configs check the no-op. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then trace $CLI bundle deploy &> LOG.redeploy cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null @@ -71,4 +72,26 @@ if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null +else + # Fuzz mode: the exact no-op check false-positives because the fake server does not + # round-trip every field, so a redeploy can legitimately want a destructive recreate + # (e.g. an immutable field the fake server reformats) and then fail non-interactively. + # That is a fidelity gap, not a bug, so tolerate a non-zero redeploy; a panic in it is + # still caught below. On a clean redeploy, assert the one fidelity-independent + # invariant: planning is deterministic -- two consecutive plans of the redeployed state + # must be byte-identical. A diff means nondeterministic planning/serialization + # (unstable map order, per-run randomness), a real bug. + redeploy_rc=0 + trace $CLI bundle deploy &> LOG.redeploy || redeploy_rc=$? + cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null + + if [ "$redeploy_rc" -eq 0 ]; then + # `|| true`: a plan that fails on an unstubbed read is a coverage gap, not a bug -- + # both plans are then empty and the diff trivially passes. + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err || true + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err || true + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + fi fi From 61d1d9cc7381d1e56bacd21e2247dc06df224262 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 17 Jul 2026 10:57:13 +0000 Subject: [PATCH 44/53] Ignore root build/ (fuzz coverage + harness terraform artifacts) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 63a15332d02..a3e54eb7d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,10 @@ dist/ # Local fuzz driver scratch (see .fuzztmp/run_fuzz.sh) .fuzztmp/ +# Fuzz coverage output and harness-downloaded terraform (task test-fuzz-cover +# sets CLI_GOCOVERDIR=build/cover-fuzz, resolved against the repo root) +/build/ + # Go workspace file go.work go.work.sum From d80a27d208209554fb921c4461aca74dbc68c05d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 22 Jul 2026 08:14:57 +0000 Subject: [PATCH 45/53] acc/fuzz: pin parent_path to a valid workspace folder --- acceptance/bin/gen_fuzz_config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 9aca9166cce..6e67140d680 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -137,6 +137,11 @@ # existence/extension check a bare token would fail. NOTEBOOK_PATH = "/Shared/notebook" +# parent_path is a workspace folder; a dangerous value is rejected by the backend and, on +# read, the CLI re-adds the /Workspace prefix, so a mismatch plans a spurious recreate. Pin +# it to a valid folder so the fuzzer exercises deploy instead. +PARENT_PATH = "/Workspace/Shared" + # Fields declared as string in the schema but parsed as google.protobuf.Duration at # config load (e.g. suspend_timeout_duration, ttl); a bare token fails to parse. DURATION_VALUE = "3600s" @@ -339,6 +344,8 @@ def gen_scalar(self, schema, name): return NOTEBOOK_PATH if name == "source_code_path": return APP_SOURCE_CODE_PATH + if name == "parent_path": + return PARENT_PATH if name == "file_path": return FILE_PATH_BY_RESOURCE.get(self.rtype, self.token()) if name.endswith("_duration") or name == "ttl": From 691ac599dd8ddcf1976f11ac3ca2d674b04cc18c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 22 Jul 2026 13:16:50 +0000 Subject: [PATCH 46/53] acc/fuzz: regenerate out.test.toml for instance_pool.yml.tmpl --- acceptance/bundle/invariant/canonical/out.test.toml | 1 + acceptance/bundle/invariant/destroy_recreate/out.test.toml | 1 + acceptance/bundle/invariant/redeploy/out.test.toml | 1 + acceptance/bundle/invariant/update/out.test.toml | 1 + 4 files changed, 4 insertions(+) diff --git a/acceptance/bundle/invariant/canonical/out.test.toml b/acceptance/bundle/invariant/canonical/out.test.toml index 8901423c85f..19a27f26934 100644 --- a/acceptance/bundle/invariant/canonical/out.test.toml +++ b/acceptance/bundle/invariant/canonical/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_recreate/out.test.toml b/acceptance/bundle/invariant/destroy_recreate/out.test.toml index 8901423c85f..19a27f26934 100644 --- a/acceptance/bundle/invariant/destroy_recreate/out.test.toml +++ b/acceptance/bundle/invariant/destroy_recreate/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/redeploy/out.test.toml b/acceptance/bundle/invariant/redeploy/out.test.toml index 8901423c85f..19a27f26934 100644 --- a/acceptance/bundle/invariant/redeploy/out.test.toml +++ b/acceptance/bundle/invariant/redeploy/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/update/out.test.toml b/acceptance/bundle/invariant/update/out.test.toml index 8901423c85f..19a27f26934 100644 --- a/acceptance/bundle/invariant/update/out.test.toml +++ b/acceptance/bundle/invariant/update/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", From 3a467cadde18fa0117cfdf36615103a860ac2952 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 22 Jul 2026 15:04:17 +0000 Subject: [PATCH 47/53] Remove canonical, update, and destroy_recreate fuzz invariants --- acceptance/bin/gen_fuzz_config.py | 2 +- acceptance/bin/verify_plan_action.py | 60 ------------- acceptance/bundle/invariant/README.md | 3 - .../bundle/invariant/canonical/out.test.toml | 59 ------------- .../bundle/invariant/canonical/output.txt | 1 - acceptance/bundle/invariant/canonical/script | 43 ---------- .../invariant/destroy_recreate/out.test.toml | 59 ------------- .../invariant/destroy_recreate/output.txt | 1 - .../bundle/invariant/destroy_recreate/script | 85 ------------------- .../bundle/invariant/fuzz/out.test.toml | 9 +- acceptance/bundle/invariant/fuzz/test.toml | 2 +- .../bundle/invariant/update/out.test.toml | 59 ------------- acceptance/bundle/invariant/update/output.txt | 1 - acceptance/bundle/invariant/update/script | 84 ------------------ 14 files changed, 3 insertions(+), 465 deletions(-) delete mode 100755 acceptance/bin/verify_plan_action.py delete mode 100644 acceptance/bundle/invariant/canonical/out.test.toml delete mode 100644 acceptance/bundle/invariant/canonical/output.txt delete mode 100644 acceptance/bundle/invariant/canonical/script delete mode 100644 acceptance/bundle/invariant/destroy_recreate/out.test.toml delete mode 100644 acceptance/bundle/invariant/destroy_recreate/output.txt delete mode 100644 acceptance/bundle/invariant/destroy_recreate/script delete mode 100644 acceptance/bundle/invariant/update/out.test.toml delete mode 100644 acceptance/bundle/invariant/update/output.txt delete mode 100644 acceptance/bundle/invariant/update/script diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 6e67140d680..04249b549d9 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -314,7 +314,7 @@ def gen_permissions(self): def gen_scalar(self, schema, name): t = schema.get("type") if t == "boolean": - # destroy_recreate invariant requires destroy to succeed. + # The invariant cleanup traps destroy the bundle, which must succeed. if name == "prevent_destroy": return False return self.rng.choice([True, False]) diff --git a/acceptance/bin/verify_plan_action.py b/acceptance/bin/verify_plan_action.py deleted file mode 100755 index d08c0113875..00000000000 --- a/acceptance/bin/verify_plan_action.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -""" -Check that a `bundle plan -o json` shows an expected action, for invariants beyond -no-drift. - - verify_plan_action.py PATH update every changed resource is an in-place update - (not a recreate) and at least one changed - verify_plan_action.py PATH create every resource is a create (e.g. a re-plan - after destroy must recreate everything) - -Action vocabulary mirrors bundle/deployplan/action.go. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from util import load_plan - -# update_id/resize keep the resource (no recreate), so they count as in-place updates. -ALLOWED = { - "update": {"update", "update_id", "resize"}, - "create": {"create"}, -} -# After a destroy, a "skip" means the resource survived (orphaned state), so skip is -# only tolerated for the update check, where unrelated siblings may be unchanged. -SKIP_OK = {"update": True, "create": False} - - -def main(): - path, expected = sys.argv[1], sys.argv[2] - allowed = ALLOWED[expected] - skip_ok = SKIP_OK[expected] - - data, raw = load_plan(path) - - matched = 0 - bad = 0 - for key, value in data["plan"].items(): - action = value.get("action") - if action == "skip" and skip_ok: - continue - if action in allowed: - matched += 1 - else: - print(f"Unexpected {action=} for {key} (expected {expected})") - bad += 1 - - if matched == 0: - print(f"plan shows no {expected} action; expected at least one") - bad += 1 - - if bad: - print(raw, flush=True) - sys.exit(10) - - -if __name__ == "__main__": - main() diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 3f653324498..6ea27785957 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -19,9 +19,6 @@ int32/int64 boundaries) to probe the CLI's input handling. - `no_drift` -- deploy, then no drift - `migrate` -- Terraform deploy, migrate to direct, then no drift - `redeploy` -- deploy twice; the second deploy must be a no-op -- `canonical` -- `validate -o json` must be byte-identical across two runs -- `update` -- edit a comment/description; the redeploy must update in place (not recreate) -- `destroy_recreate` -- deploy then destroy; a re-plan must recreate everything Since the schema comes from the CLI under test, an unrelated struct change can shift a seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), diff --git a/acceptance/bundle/invariant/canonical/out.test.toml b/acceptance/bundle/invariant/canonical/out.test.toml deleted file mode 100644 index 19a27f26934..00000000000 --- a/acceptance/bundle/invariant/canonical/out.test.toml +++ /dev/null @@ -1,59 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.INPUT_CONFIG = [ - "alert.yml.tmpl", - "app.yml.tmpl", - "catalog.yml.tmpl", - "cluster.yml.tmpl", - "cluster_apply_policy_default_values.yml.tmpl", - "dashboard.yml.tmpl", - "job_apply_policy_default_values_job_cluster.yml.tmpl", - "job_apply_policy_default_values_task_cluster.yml.tmpl", - "job_apply_policy_default_values_for_each_task.yml.tmpl", - "database_catalog.yml.tmpl", - "database_instance.yml.tmpl", - "experiment.yml.tmpl", - "external_location.yml.tmpl", - "genie_space.yml.tmpl", - "instance_pool.yml.tmpl", - "job.yml.tmpl", - "job_pydabs_10_tasks.yml.tmpl", - "job_pydabs_1000_tasks.yml.tmpl", - "job_cross_resource_ref.yml.tmpl", - "job_permission_ref.yml.tmpl", - "job_run.yml.tmpl", - "job_run_job_ref.yml.tmpl", - "job_with_depends_on.yml.tmpl", - "job_with_task.yml.tmpl", - "model.yml.tmpl", - "model_with_permissions.yml.tmpl", - "model_serving_endpoint.yml.tmpl", - "pipeline.yml.tmpl", - "pipeline_apply_policy_default_values.yml.tmpl", - "pipeline_config_dots.yml.tmpl", - "postgres_branch.yml.tmpl", - "postgres_catalog.yml.tmpl", - "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", - "postgres_project.yml.tmpl", - "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", - "registered_model.yml.tmpl", - "schema.yml.tmpl", - "schema_grant_ref.yml.tmpl", - "schema_uppercase_name.yml.tmpl", - "secret_scope.yml.tmpl", - "secret_scope_default_backend_type.yml.tmpl", - "sql_warehouse.yml.tmpl", - "synced_database_table.yml.tmpl", - "vector_search_endpoint.yml.tmpl", - "vector_search_index.yml.tmpl", - "volume.yml.tmpl", - "volume_external.yml.tmpl", - "volume_path_job_ref.yml.tmpl", - "volume_uppercase_name.yml.tmpl" -] diff --git a/acceptance/bundle/invariant/canonical/output.txt b/acceptance/bundle/invariant/canonical/output.txt deleted file mode 100644 index 7a28cb73a58..00000000000 --- a/acceptance/bundle/invariant/canonical/output.txt +++ /dev/null @@ -1 +0,0 @@ -INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/canonical/script b/acceptance/bundle/invariant/canonical/script deleted file mode 100644 index fe8acc5eb42..00000000000 --- a/acceptance/bundle/invariant/canonical/script +++ /dev/null @@ -1,43 +0,0 @@ -# Invariant to test: `bundle validate -o json` is deterministic -- two runs must be -# byte-identical. Catches unstable map ordering / serialization in config loading. -# No deploy, so no cleanup or cloud state. - -if [ -n "${FUZZ_SEED:-}" ]; then - fuzz_gen_config.py > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - cp databricks.yml LOG.config -else - # Copy data files to test directory - cp -r "$TESTDIR/../data/." . &> LOG.cp - - # Run init script if present - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" - if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init - fi - - envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - - cp databricks.yml LOG.config -fi - -$CLI bundle validate -o json > validate1.json 2>LOG.validate1.err -validate_rc=$? -cat LOG.validate1.err | contains.py '!panic:' '!internal error' > /dev/null - -# A config that fails to validate is an invalid fuzz config, not a bug, so stop before -# the marker (curated tests already aborted above under `bash -e`). -if [ "$validate_rc" -ne 0 ]; then - exit "$validate_rc" -fi - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK - -$CLI bundle validate -o json > validate2.json 2>LOG.validate2.err -cat LOG.validate2.err | contains.py '!panic:' '!internal error' > /dev/null - -# Determinism is cloud-independent and cheap, so it always runs (no SKIP_DRIFT_CHECK -# gate): identical input must yield identical output. A diff is a real bug. -diff validate1.json validate2.json > LOG.validate.diff diff --git a/acceptance/bundle/invariant/destroy_recreate/out.test.toml b/acceptance/bundle/invariant/destroy_recreate/out.test.toml deleted file mode 100644 index 19a27f26934..00000000000 --- a/acceptance/bundle/invariant/destroy_recreate/out.test.toml +++ /dev/null @@ -1,59 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.INPUT_CONFIG = [ - "alert.yml.tmpl", - "app.yml.tmpl", - "catalog.yml.tmpl", - "cluster.yml.tmpl", - "cluster_apply_policy_default_values.yml.tmpl", - "dashboard.yml.tmpl", - "job_apply_policy_default_values_job_cluster.yml.tmpl", - "job_apply_policy_default_values_task_cluster.yml.tmpl", - "job_apply_policy_default_values_for_each_task.yml.tmpl", - "database_catalog.yml.tmpl", - "database_instance.yml.tmpl", - "experiment.yml.tmpl", - "external_location.yml.tmpl", - "genie_space.yml.tmpl", - "instance_pool.yml.tmpl", - "job.yml.tmpl", - "job_pydabs_10_tasks.yml.tmpl", - "job_pydabs_1000_tasks.yml.tmpl", - "job_cross_resource_ref.yml.tmpl", - "job_permission_ref.yml.tmpl", - "job_run.yml.tmpl", - "job_run_job_ref.yml.tmpl", - "job_with_depends_on.yml.tmpl", - "job_with_task.yml.tmpl", - "model.yml.tmpl", - "model_with_permissions.yml.tmpl", - "model_serving_endpoint.yml.tmpl", - "pipeline.yml.tmpl", - "pipeline_apply_policy_default_values.yml.tmpl", - "pipeline_config_dots.yml.tmpl", - "postgres_branch.yml.tmpl", - "postgres_catalog.yml.tmpl", - "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", - "postgres_project.yml.tmpl", - "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", - "registered_model.yml.tmpl", - "schema.yml.tmpl", - "schema_grant_ref.yml.tmpl", - "schema_uppercase_name.yml.tmpl", - "secret_scope.yml.tmpl", - "secret_scope_default_backend_type.yml.tmpl", - "sql_warehouse.yml.tmpl", - "synced_database_table.yml.tmpl", - "vector_search_endpoint.yml.tmpl", - "vector_search_index.yml.tmpl", - "volume.yml.tmpl", - "volume_external.yml.tmpl", - "volume_path_job_ref.yml.tmpl", - "volume_uppercase_name.yml.tmpl" -] diff --git a/acceptance/bundle/invariant/destroy_recreate/output.txt b/acceptance/bundle/invariant/destroy_recreate/output.txt deleted file mode 100644 index 7a28cb73a58..00000000000 --- a/acceptance/bundle/invariant/destroy_recreate/output.txt +++ /dev/null @@ -1 +0,0 @@ -INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/destroy_recreate/script b/acceptance/bundle/invariant/destroy_recreate/script deleted file mode 100644 index 7e615904f5f..00000000000 --- a/acceptance/bundle/invariant/destroy_recreate/script +++ /dev/null @@ -1,85 +0,0 @@ -# Invariant to test: after deploy then destroy, a re-plan wants to CREATE everything -# again -- proving destroy cleared all tracked state. A resource destroy forgets shows -# up as "skip" (still present), a bug. -# Additional checks: no internal errors / panics in validate/plan/deploy/destroy - -if [ -n "${FUZZ_SEED:-}" ]; then - fuzz_gen_config.py > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - cp databricks.yml LOG.config -else - # Copy data files to test directory - cp -r "$TESTDIR/../data/." . &> LOG.cp - - # Run init script if present - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" - if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init - fi - - envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - - cp databricks.yml LOG.config -fi - -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate - -cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null - -cleanup() { - # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. - if [ -z "${deployed:-}" ]; then - return - fi - - # This test destroys to LOG.destroy itself, so the trap logs elsewhere to keep - # the body's destroy output (and any panic) intact for the post-run scan. - trace $CLI bundle destroy --auto-approve &> LOG.destroy_cleanup - cat LOG.destroy_cleanup | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -trace $CLI bundle deploy &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null -if [ "$deploy_rc" -ne 0 ]; then - exit "$deploy_rc" -fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK - -# Destroy unconditionally so any panic lands in LOG.destroy for the post-scan; -# completeness (re-plan recreates everything) is gated below. -trace $CLI bundle destroy --auto-approve &> LOG.destroy -destroy_rc=$? -cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - -# A clean destroy leaves nothing, so stop the cleanup trap from destroying again -# (which would hit unstubbed URLs on the fake server). -if [ "$destroy_rc" -eq 0 ]; then - deployed="" -fi - -# A fuzzed config can deploy yet legitimately leave state a re-plan reads differently, -# so the fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check -# that a re-plan recreates everything. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - if [ "$destroy_rc" -ne 0 ]; then - exit "$destroy_rc" - fi - - $CLI bundle plan -o json > LOG.recreate_plan.json 2>LOG.recreate_plan.err - cat LOG.recreate_plan.err | contains.py '!panic:' '!internal error' > /dev/null - verify_plan_action.py LOG.recreate_plan.json create -fi diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index 868a98f0bb8..dac5d294c22 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -6,12 +6,5 @@ GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.FUZZ_MODE = ["generate", "mutate"] EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] -EnvMatrix.FUZZ_TARGET = [ - "no_drift", - "migrate", - "redeploy", - "canonical", - "update", - "destroy_recreate" -] +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy"] EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index afd3a0bcc5c..6d109051c9e 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -12,7 +12,7 @@ IgnoreUnhandledRequests = true # Run the real invariant test script for each target. migrate ignores # DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. -EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy"] EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] # generate = build a config from the schema (gen_fuzz_config.py); mutate = perturb a diff --git a/acceptance/bundle/invariant/update/out.test.toml b/acceptance/bundle/invariant/update/out.test.toml deleted file mode 100644 index 19a27f26934..00000000000 --- a/acceptance/bundle/invariant/update/out.test.toml +++ /dev/null @@ -1,59 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.INPUT_CONFIG = [ - "alert.yml.tmpl", - "app.yml.tmpl", - "catalog.yml.tmpl", - "cluster.yml.tmpl", - "cluster_apply_policy_default_values.yml.tmpl", - "dashboard.yml.tmpl", - "job_apply_policy_default_values_job_cluster.yml.tmpl", - "job_apply_policy_default_values_task_cluster.yml.tmpl", - "job_apply_policy_default_values_for_each_task.yml.tmpl", - "database_catalog.yml.tmpl", - "database_instance.yml.tmpl", - "experiment.yml.tmpl", - "external_location.yml.tmpl", - "genie_space.yml.tmpl", - "instance_pool.yml.tmpl", - "job.yml.tmpl", - "job_pydabs_10_tasks.yml.tmpl", - "job_pydabs_1000_tasks.yml.tmpl", - "job_cross_resource_ref.yml.tmpl", - "job_permission_ref.yml.tmpl", - "job_run.yml.tmpl", - "job_run_job_ref.yml.tmpl", - "job_with_depends_on.yml.tmpl", - "job_with_task.yml.tmpl", - "model.yml.tmpl", - "model_with_permissions.yml.tmpl", - "model_serving_endpoint.yml.tmpl", - "pipeline.yml.tmpl", - "pipeline_apply_policy_default_values.yml.tmpl", - "pipeline_config_dots.yml.tmpl", - "postgres_branch.yml.tmpl", - "postgres_catalog.yml.tmpl", - "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", - "postgres_project.yml.tmpl", - "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", - "registered_model.yml.tmpl", - "schema.yml.tmpl", - "schema_grant_ref.yml.tmpl", - "schema_uppercase_name.yml.tmpl", - "secret_scope.yml.tmpl", - "secret_scope_default_backend_type.yml.tmpl", - "sql_warehouse.yml.tmpl", - "synced_database_table.yml.tmpl", - "vector_search_endpoint.yml.tmpl", - "vector_search_index.yml.tmpl", - "volume.yml.tmpl", - "volume_external.yml.tmpl", - "volume_path_job_ref.yml.tmpl", - "volume_uppercase_name.yml.tmpl" -] diff --git a/acceptance/bundle/invariant/update/output.txt b/acceptance/bundle/invariant/update/output.txt deleted file mode 100644 index 7a28cb73a58..00000000000 --- a/acceptance/bundle/invariant/update/output.txt +++ /dev/null @@ -1 +0,0 @@ -INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/update/script b/acceptance/bundle/invariant/update/script deleted file mode 100644 index 7a91c38cef9..00000000000 --- a/acceptance/bundle/invariant/update/script +++ /dev/null @@ -1,84 +0,0 @@ -# Invariant to test: editing a comment/description redeploys as an in-place update, not -# a recreate, and converges. Exercises the update (PATCH) path create-only deploys never -# touch. -# Additional checks: no internal errors / panics in validate/plan/deploy - -if [ -n "${FUZZ_SEED:-}" ]; then - fuzz_gen_config.py > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - cp databricks.yml LOG.config -else - # Copy data files to test directory - cp -r "$TESTDIR/../data/." . &> LOG.cp - - # Run init script if present - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" - if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init - fi - - envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - - cp databricks.yml LOG.config -fi - -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate - -cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null - -cleanup() { - # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -trace $CLI bundle deploy &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null -if [ "$deploy_rc" -ne 0 ]; then - exit "$deploy_rc" -fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK - -# A fuzzed config can deploy yet legitimately differ on update, so the fuzzer sets -# SKIP_DRIFT_CHECK to assert only no-panic; curated configs check the update path. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - # Only configs with an editable comment/description exercise the update path; - # others just verify the deploy above (edit_fuzz_config.py --detect exits 1). - if edit_fuzz_config.py databricks.yml --detect 2>LOG.detect.err; then - # Change the comment/description; the re-plan must show an in-place update. - edit_fuzz_config.py databricks.yml 2>LOG.edit.err - cat LOG.edit.err | contains.py '!Traceback' > /dev/null - - $CLI bundle plan -o json > LOG.update_plan.json 2>LOG.update_plan.err - cat LOG.update_plan.err | contains.py '!panic:' '!internal error' > /dev/null - - trace $CLI bundle deploy &> LOG.redeploy - cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null - - # The edit must update in place, not recreate. - verify_plan_action.py LOG.update_plan.json update - - # And the applied update must converge: a re-plan shows no further changes. - $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null - verify_no_drift.py LOG.planjson - fi -fi From a72e5f938afb7e7eddc958962a9cd47bd2d08aa9 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 23 Jul 2026 09:29:19 +0000 Subject: [PATCH 48/53] acc/fuzz: inject valid optional fields in mutate mode Add a schema-aware additive op to mutate_fuzz_config: alongside deleting and perturbing existing fields, it now injects a valid optional field the curated base omits, valued by the schema generator. Destructive mutations stay within the base's field set (only reject/panic bugs); adding a valid optional field to a still-deploying config is what reaches reconcile/drift bugs. The no-schema path is unchanged (RNG stream and selftest golden preserved). --- acceptance/bin/fuzz_gen_config.py | 6 +- acceptance/bin/mutate_fuzz_config.py | 120 +++++++++++++++++++-- acceptance/bin/mutate_fuzz_config_check.py | 38 +++++++ 3 files changed, 153 insertions(+), 11 deletions(-) diff --git a/acceptance/bin/fuzz_gen_config.py b/acceptance/bin/fuzz_gen_config.py index a228bb0c82a..62b842443ef 100755 --- a/acceptance/bin/fuzz_gen_config.py +++ b/acceptance/bin/fuzz_gen_config.py @@ -55,7 +55,11 @@ def mutate_base(seed): with open(path) as f: rendered = substitute_variables(f.read()) config = load_yaml(rendered) - return to_yaml(mutate(config, seed)) + # The schema lets mutate inject valid optional fields, not just perturb existing ones. + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + unique = f"{os.environ['UNIQUE_NAME']}-{seed}" + return to_yaml(mutate(config, seed, schema=schema, unique=unique)) def main(): diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py index cd9c9336801..de861e518ab 100755 --- a/acceptance/bin/mutate_fuzz_config.py +++ b/acceptance/bin/mutate_fuzz_config.py @@ -1,34 +1,47 @@ #!/usr/bin/env python3 """ -Mutate a known-good bundle config by deleting and perturbing random fields. +Mutate a known-good bundle config by deleting, perturbing, and adding random fields. Complements gen_fuzz_config.py (generate-from-scratch via schema walk): instead of building a config from the schema, this starts from a curated invariant config that -already deploys and applies a few seeded mutations (delete a field, replace a scalar -with a fuzz token, a boundary/dangerous value, or an empty container). It exercises the -CLI's handling of perturbed-but-realistic input, and reaches a much higher deploy rate -than the schema walk, since the base already resolves. +already deploys and applies a few seeded mutations. It exercises the CLI's handling of +perturbed-but-realistic input, and reaches a much higher deploy rate than the schema +walk, since the base already resolves. + +Two kinds of mutation, chosen per step: + +- destructive (always): delete a field, or replace it with a fuzz token, a + boundary/dangerous value, or an empty container. Probes the reject/no-panic path. +- additive (only with a schema): inject a valid optional field the base omits, valued by + the schema generator. Destructive ops stay within the base's field set, so they only + find reject/panic bugs; adding a valid optional field to a still-deploying config is + what reaches reconcile/drift bugs (the field space the schema walk explores). Reads the base databricks.yml (already envsubst-rendered) from stdin, writes the mutated -config to stdout. --seed makes the mutation reproducible. +config to stdout. --seed makes the mutation reproducible; --schema enables the additive op. The invariant harness only asserts no-panic on fuzzed configs (SKIP_DRIFT_CHECK), so a mutation that makes the config invalid is fine: the CLI must reject it cleanly, not crash. """ import argparse +import json import os import random import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, to_yaml +from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, Generator, resource_types, to_yaml # Same near-range-end and dangerous-character probes the schema-walk generator injects into # free-form scalars; here we drop them onto any field (see mutate_once). DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS +# Chance a step injects a field rather than perturbing one. Biased high: injection is the +# path to drift bugs, and destructive coverage is already dense (1-3 steps per seed). +ADD_PROB = 0.6 + def tokenize(text): # (indent, content) per non-blank, non-comment line. Only full-line comments are @@ -155,8 +168,83 @@ def mutate_once(rng, roots): container[key] = rng.choice([{}, [], None]) -def mutate(config, seed): +def resource_element(gen, type_schema): + # The instance schema is the map's object-branch additionalProperties (as gen_resource). + map_schema = gen.resolve(type_schema) + obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") + return obj["additionalProperties"] + + +def collect_insertions(gen, node, schema, rtype, out): + # Record every writable optional field absent from an existing object, walking the node + # alongside its schema so nested objects (not just the top level) are candidates too. + schema = gen.resolve(schema) + if not isinstance(schema, dict): + return + + branches = schema.get("oneOf") or schema.get("anyOf") + if branches: + # Pick the branch matching the node we actually have, not a random one. + picked = None + for branch in branches: + resolved = gen.resolve(branch) + if isinstance(node, dict) and ( + resolved.get("type") == "object" or "properties" in resolved or gen.is_map(resolved) + ): + picked = resolved + break + if isinstance(node, list) and resolved.get("type") == "array": + picked = resolved + break + if picked is None: + return + schema = picked + + if isinstance(node, dict): + props = schema.get("properties", {}) + for name, prop_schema in props.items(): + if name not in node and not gen.should_skip_property(name, prop_schema): + out.append((node, name, prop_schema, rtype)) + for key, value in node.items(): + if key in props and isinstance(value, (dict, list)): + collect_insertions(gen, value, props[key], rtype, out) + if gen.is_map(schema): + for value in node.values(): + if isinstance(value, (dict, list)): + collect_insertions(gen, value, schema["additionalProperties"], rtype, out) + elif isinstance(node, list): + items = schema.get("items") + if items: + for value in node: + if isinstance(value, (dict, list)): + collect_insertions(gen, value, items, rtype, out) + + +def add_field(gen, rng, config): + # Inject one valid optional field, absent from the base, into a random insertion point. + types = resource_types(gen.root, gen) + points = [] + for rtype, instances in config.get("resources", {}).items(): + if rtype not in types or not isinstance(instances, dict): + continue + element = resource_element(gen, types[rtype]) + for instance in instances.values(): + if isinstance(instance, dict): + gen.rtype = rtype + collect_insertions(gen, instance, element, rtype, points) + if not points: + return + node, name, prop_schema, rtype = rng.choice(points) + # rtype drives grants/permissions/typed-string generation (see gen_scalar/gen_grants). + gen.rtype = rtype + value = gen.gen(prop_schema, 1, name) + if value is not None: + node[name] = value + + +def mutate(config, seed, schema=None, unique="fuzz"): rng = random.Random(seed) + gen = Generator(schema, rng, unique) if schema is not None else None # Mutate only inside resource instances: keep bundle/name and the # resources.. skeleton so there is always something to deploy, while @@ -167,7 +255,12 @@ def mutate(config, seed): roots.extend(v for v in instances.values() if isinstance(v, (dict, list))) for _ in range(rng.randint(1, 3)): - mutate_once(rng, roots) + # gen is None short-circuits before rng is touched, so the no-schema path keeps its + # exact RNG stream (and committed selftest output) unchanged. + if gen is not None and rng.random() < ADD_PROB: + add_field(gen, rng, config) + else: + mutate_once(rng, roots) return config @@ -175,13 +268,20 @@ def mutate(config, seed): def main(): parser = argparse.ArgumentParser() parser.add_argument("--seed", type=int, required=True, help="RNG seed (for reproducibility)") + parser.add_argument("--schema", help="Path to bundle JSON schema; enables valid-optional-field injection") + parser.add_argument("--unique", default="fuzz", help="Unique suffix for injected field values") args = parser.parse_args() config = load_yaml(sys.stdin.read()) if not isinstance(config, dict): sys.exit("mutate_fuzz_config: base config did not parse to a mapping") - sys.stdout.write(to_yaml(mutate(config, args.seed))) + schema = None + if args.schema: + with open(args.schema) as f: + schema = json.load(f) + + sys.stdout.write(to_yaml(mutate(config, args.seed, schema=schema, unique=args.unique))) if __name__ == "__main__": diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py index 6df04885dc6..a56101abde4 100755 --- a/acceptance/bin/mutate_fuzz_config_check.py +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -12,6 +12,7 @@ as an output diff. """ +import json import os import sys @@ -19,9 +20,11 @@ from envsubst import substitute_variables from fuzz_gen_config import MUTATE_BASES +from gen_fuzz_config import SKIP_PROPERTY_NAMES from mutate_fuzz_config import load_yaml, mutate, to_yaml CONFIGS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "invariant", "configs") +SCHEMA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "bundle", "schema", "jsonschema.json") def render(name): @@ -29,6 +32,13 @@ def render(name): return substitute_variables(f.read()) +def instance(config): + # The curated bases are single-resource; return that one resource instance. + (instances,) = config["resources"].values() + (value,) = instances.values() + return value + + def main(): # Fixed so the printed configs are stable regardless of the harness's unique name. os.environ["UNIQUE_NAME"] = "check" @@ -58,6 +68,34 @@ def main(): sys.stdout.write(f"=== volume seed={seed} ===\n") sys.stdout.write(to_yaml(mutate(load_yaml(render("volume")), seed))) + # Assert-only (no stdout) so this golden doesn't churn as the schema grows. The + # registered_model base sets none of its optional fields, so any added field is injected. + with open(SCHEMA) as f: + schema = json.load(f) + + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check")) + b = to_yaml(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check")) + if a != b: + sys.stderr.write(f"seed {seed}: schema-aware mutation is not deterministic\n") + failed = True + + base_fields = set(instance(load_yaml(render("registered_model")))) + injected = False + for seed in range(30): + fields = set(instance(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check"))) + added = fields - base_fields + if added: + injected = True + # Injecting an output-only field (SKIP_PROPERTY_NAMES) would manufacture false drift. + leaked = SKIP_PROPERTY_NAMES & added + if leaked: + sys.stderr.write(f"seed {seed}: injected output-only field(s): {sorted(leaked)}\n") + failed = True + if not injected: + sys.stderr.write("schema-aware mutation never injected an optional field\n") + failed = True + if failed: sys.exit(1) From 3b2b189aa9371971f236a13196637c1a040ba78d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 23 Jul 2026 13:55:52 +0000 Subject: [PATCH 49/53] =?UTF-8?q?acc/fuzz:=20address=20review=20=E2=80=94?= =?UTF-8?q?=20drop=20dead=20edit=20harness,=20testserver=20catalog=20fix,?= =?UTF-8?q?=20gitignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove the orphaned edit_fuzz_config.py chain (edit script, its check, and selftest); the update invariant it served was removed earlier in this branch. Decouple gen_fuzz_config_check.py from it (the one-line scalar contract is now attributed to mutate_fuzz_config's loader, which actually relies on it). - Revert the libs/testserver/catalogs.go create-payload round-trip; it is a standalone fake-server fidelity change unrelated to the fuzz harness. - Drop the .gitignore /build/ and .fuzztmp/ entries (stale/misleading paths). - Regenerate fuzz/out.test.toml (drops stale GOOSOnPR lines). --- .gitignore | 7 -- acceptance/bin/edit_fuzz_config.py | 109 ------------------ acceptance/bin/edit_fuzz_config_check.py | 84 -------------- acceptance/bin/gen_fuzz_config_check.py | 18 +-- .../bundle/invariant/fuzz/out.test.toml | 2 - .../selftest/edit_fuzz_config/out.test.toml | 3 - .../selftest/edit_fuzz_config/output.txt | 4 - acceptance/selftest/edit_fuzz_config/script | 1 - libs/testserver/catalogs.go | 24 ++-- 9 files changed, 15 insertions(+), 237 deletions(-) delete mode 100755 acceptance/bin/edit_fuzz_config.py delete mode 100755 acceptance/bin/edit_fuzz_config_check.py delete mode 100644 acceptance/selftest/edit_fuzz_config/out.test.toml delete mode 100644 acceptance/selftest/edit_fuzz_config/output.txt delete mode 100644 acceptance/selftest/edit_fuzz_config/script diff --git a/.gitignore b/.gitignore index a3e54eb7d2b..543b638a537 100644 --- a/.gitignore +++ b/.gitignore @@ -67,13 +67,6 @@ dist/ # Per-module golangci-lint TMPDIR (configured in Taskfile.yml) /.tmp/ -# Local fuzz driver scratch (see .fuzztmp/run_fuzz.sh) -.fuzztmp/ - -# Fuzz coverage output and harness-downloaded terraform (task test-fuzz-cover -# sets CLI_GOCOVERDIR=build/cover-fuzz, resolved against the repo root) -/build/ - # Go workspace file go.work go.work.sum diff --git a/acceptance/bin/edit_fuzz_config.py b/acceptance/bin/edit_fuzz_config.py deleted file mode 100755 index a71cf82771f..00000000000 --- a/acceptance/bin/edit_fuzz_config.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -""" -Edit a `comment`/`description` scalar in a generated databricks.yml so a redeploy is an -in-place update, not a recreate. Used by the `update` invariant. - -Some resources classify `description` as immutable (recreate on change) in the direct -engine spec (e.g. model_serving_endpoints); editing that replans as a recreate the update -invariant would wrongly flag, so skip it and pick a mutable field (or report none). - -gen_fuzz_config.py emits one scalar per line as `key: `, so a regex match suffices -(no YAML dependency for the edit itself); the immutable set is read from resources.yml. - - edit_fuzz_config.py PATH edit in place; exit 1 if no editable field - edit_fuzz_config.py PATH --detect exit 0 if an editable field exists, else 1 -""" - -import argparse -import re -import sys -from pathlib import Path - -# Allow an optional "- " so a comment/description that is the first key of a list-item -# dict still matches; the captured prefix is preserved verbatim on rewrite. -FIELD_RE = re.compile(r'^(\s*(?:- )?)(comment|description): (".*")\s*$') - -# A resource type header directly under `resources:` (two-space indent, as emitted by -# gen_fuzz_config.py and the curated templates). -TYPE_RE = re.compile(r"^ ([\w-]+):\s*$") - -NEW_VALUE = '"fuzz_edited_value"' - -# resources.yml is the source of truth for field mutability. acceptance/bin is on PATH as -# the real dir (not a copy), so __file__ resolves two levels below the repo root. -RESOURCES_YML = Path(__file__).resolve().parents[2] / "bundle" / "direct" / "dresources" / "resources.yml" - - -def immutable_fields(): - """Map resource type -> set of fields that recreate on change (immutable). - - Hand-rolled line parser over resources.yml's fixed two-space layout, avoiding a YAML - dependency the harness's Python lacks. - """ - result = {} - current_type = None - in_recreate = False - for raw in RESOURCES_YML.read_text().splitlines(): - if not raw.strip() or raw.lstrip().startswith("#"): - continue - indent = len(raw) - len(raw.lstrip()) - stripped = raw.strip() - if indent == 2 and stripped.endswith(":"): - current_type = stripped[:-1] - in_recreate = False - elif indent == 4 and stripped.endswith(":"): - in_recreate = stripped == "recreate_on_changes:" - elif in_recreate and current_type: - m = re.match(r"-\s*field:\s*(\S+)", stripped) - if m: - result.setdefault(current_type, set()).add(m.group(1)) - return result - - -def find_line(lines, immutable): - current_type = None - in_resources = False - for i, line in enumerate(lines): - stripped = line.rstrip("\n") - # Track the enclosing resource type so an immutable comment/description is skipped. - if stripped == "resources:": - in_resources = True - current_type = None - continue - if in_resources: - m_type = TYPE_RE.match(stripped) - if m_type: - current_type = m_type.group(1) - elif stripped and not stripped[0].isspace(): - in_resources = False - current_type = None - m = FIELD_RE.match(line) - if m and m.group(2) not in immutable.get(current_type, ()): - return i, m - return -1, None - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("path") - parser.add_argument("--detect", action="store_true", help="only check, don't edit") - args = parser.parse_args() - - with open(args.path) as f: - lines = f.readlines() - - i, m = find_line(lines, immutable_fields()) - if m is None: - sys.exit(1) - if args.detect: - return - - prefix, key, _ = m.groups() - lines[i] = f"{prefix}{key}: {NEW_VALUE}\n" - with open(args.path, "w") as f: - f.writelines(lines) - sys.stderr.write(f"edited {key} at line {i + 1}\n") - - -if __name__ == "__main__": - main() diff --git a/acceptance/bin/edit_fuzz_config_check.py b/acceptance/bin/edit_fuzz_config_check.py deleted file mode 100755 index b8160dbe627..00000000000 --- a/acceptance/bin/edit_fuzz_config_check.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -""" -Contract check for edit_fuzz_config's field selection (the harness diffs stdout; a -non-zero exit marks a violation on stderr): - -- A comment/description that recreates on change for its resource type (per - resources.yml, e.g. model_serving_endpoints.description) is never chosen, so the - update invariant does not assert an in-place update the backend cannot perform. -- A mutable comment/description is still chosen, even when an immutable one appears - first. -- The immutable map actually loads and reflects resources.yml. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from edit_fuzz_config import find_line, immutable_fields - -IMMUTABLE_ONLY = """\ -resources: - model_serving_endpoints: - foo: - name: "test-endpoint" - description: "old" -""" - -IMMUTABLE_THEN_MUTABLE = """\ -resources: - model_serving_endpoints: - foo: - description: "immutable" - jobs: - bar: - description: "mutable" -""" - -MUTABLE_ONLY = """\ -resources: - jobs: - bar: - description: "mutable" -""" - - -def choose(text, immutable): - i, m = find_line(text.splitlines(keepends=True), immutable) - return None if m is None else i - - -def main(): - immutable = immutable_fields() - failed = False - - # Guards the loader and the classification the update invariant relies on. - serving_immutable = "description" in immutable.get("model_serving_endpoints", set()) - if not serving_immutable: - sys.stderr.write("expected model_serving_endpoints.description to be immutable in resources.yml\n") - failed = True - - if choose(IMMUTABLE_ONLY, immutable) is not None: - sys.stderr.write("picked an immutable description\n") - failed = True - - if choose(IMMUTABLE_THEN_MUTABLE, immutable) != 6: - sys.stderr.write("expected to skip the immutable description and pick the mutable one\n") - failed = True - - if choose(MUTABLE_ONLY, immutable) != 3: - sys.stderr.write("expected to pick the mutable description\n") - failed = True - - print(f"model_serving_endpoints.description immutable: {serving_immutable}") - print(f"IMMUTABLE_ONLY: {choose(IMMUTABLE_ONLY, immutable)}") - print(f"IMMUTABLE_THEN_MUTABLE: {choose(IMMUTABLE_THEN_MUTABLE, immutable)}") - print(f"MUTABLE_ONLY: {choose(MUTABLE_ONLY, immutable)}") - - if failed: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py index 8649f4c90b5..ea8b9db943a 100755 --- a/acceptance/bin/gen_fuzz_config_check.py +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """ Contract check for gen_fuzz_config.to_yaml: every scalar is on its own line as -`key: `. edit_fuzz_config.py relies on this to edit a field by regex, not a YAML -parser. Prints each case's YAML (diffed by the harness) and exits non-zero on a violation. +`key: `. mutate_fuzz_config.py's line-based loader relies on this. Prints each +case's YAML (diffed by the harness) and exits non-zero on a violation. """ import json @@ -12,7 +12,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from edit_fuzz_config import FIELD_RE from gen_fuzz_config import DANGEROUS_STRINGS, SKIP_PROPERTY_NAMES, gen_config, to_yaml # Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, empty containers. @@ -53,17 +52,12 @@ def main(): sys.stderr.write(f"contract violation: not `key: `: {line!r}\n") failed = True - # edit_fuzz_config's FIELD_RE must still match when the value contains a colon (CASES[0]). - if not any(FIELD_RE.match(line) for line in to_yaml(CASES[0]).splitlines()): - sys.stderr.write("FIELD_RE did not match a comment/description line\n") - failed = True - - # Generate mode now emits DANGEROUS_STRINGS into free-form fields; each must still - # serialize to a single `key: ` line so edit_fuzz_config can rewrite it in place. + # Generate mode emits DANGEROUS_STRINGS into free-form fields; each must still + # serialize to a single `key: ` line so the line-based loader can parse it. for i, val in enumerate(DANGEROUS_STRINGS): line = to_yaml({"description": val}).rstrip("\n") - if "\n" in line or not FIELD_RE.match(line): - sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line comment/description contract: {line!r}\n") + if "\n" in line or not SCALAR.fullmatch(line): + sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line scalar contract: {line!r}\n") failed = True # Multi-resource configs merge types under resources., and one resource diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index dac5d294c22..46d7d2e5a2a 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.FUZZ_MODE = ["generate", "mutate"] EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] diff --git a/acceptance/selftest/edit_fuzz_config/out.test.toml b/acceptance/selftest/edit_fuzz_config/out.test.toml deleted file mode 100644 index f784a183258..00000000000 --- a/acceptance/selftest/edit_fuzz_config/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/edit_fuzz_config/output.txt b/acceptance/selftest/edit_fuzz_config/output.txt deleted file mode 100644 index 4037d0995cf..00000000000 --- a/acceptance/selftest/edit_fuzz_config/output.txt +++ /dev/null @@ -1,4 +0,0 @@ -model_serving_endpoints.description immutable: True -IMMUTABLE_ONLY: None -IMMUTABLE_THEN_MUTABLE: 6 -MUTABLE_ONLY: 3 diff --git a/acceptance/selftest/edit_fuzz_config/script b/acceptance/selftest/edit_fuzz_config/script deleted file mode 100644 index 78dfbef7825..00000000000 --- a/acceptance/selftest/edit_fuzz_config/script +++ /dev/null @@ -1 +0,0 @@ -edit_fuzz_config_check.py diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 26a5740a3b5..351c5a76499 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -30,21 +30,15 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { StorageRoot: createRequest.StorageRoot, ProviderName: createRequest.ProviderName, ShareName: createRequest.ShareName, - // Round-trip these so a re-read matches the deployed config: connection_name is - // recreate_on_changes (immutable), so dropping it re-planned as a perpetual - // recreate, and managed_encryption_settings showed as drift. - ConnectionName: createRequest.ConnectionName, - ManagedEncryptionSettings: createRequest.ManagedEncryptionSettings, - CustomMaxRetentionHours: createRequest.CustomMaxRetentionHours, - Options: createRequest.Options, - Properties: createRequest.Properties, - FullName: createRequest.Name, - CreatedAt: nowMilli(), - CreatedBy: s.CurrentUser().UserName, - UpdatedBy: s.CurrentUser().UserName, - MetastoreId: nextUUID(), - Owner: s.CurrentUser().UserName, - CatalogType: catalog.CatalogTypeManagedCatalog, + Options: createRequest.Options, + Properties: createRequest.Properties, + FullName: createRequest.Name, + CreatedAt: nowMilli(), + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + Owner: s.CurrentUser().UserName, + CatalogType: catalog.CatalogTypeManagedCatalog, } catalogInfo.UpdatedAt = catalogInfo.CreatedAt if catalogInfo.Properties == nil && createRequest.Name == catalogNameManagedDefaults { From e6d0ec9092acb802c402564bdebc4c750d56c2fe Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 24 Jul 2026 09:17:13 +0000 Subject: [PATCH 50/53] acc/fuzz: drop redeploy invariant and multi-resource fuzzing no_drift's post-deploy plan already dry-runs what a redeploy would apply, so it catches the same field-level non-idempotency without the cost of a second deploy; the only surface redeploy adds (plan/apply divergence) never surfaced a bug across the nightly runs. Multi-resource generation only added cross-resource reference injection, which likewise found no distinct defect (every bug reproduces at a single resource) while lowering acceptance. Remove both, shrinking the fuzz matrix from 12 leaf variants to 4 ({no_drift, migrate} x {generate, mutate}). Regenerate out.test.toml and update the README accordingly. --- acceptance/bin/fuzz_gen_config.py | 5 +- acceptance/bin/gen_fuzz_config.py | 104 ++---------------- acceptance/bin/gen_fuzz_config_check.py | 49 +-------- acceptance/bundle/invariant/README.md | 12 +- .../bundle/invariant/fuzz/out.test.toml | 3 +- acceptance/bundle/invariant/fuzz/script | 8 +- acceptance/bundle/invariant/fuzz/test.toml | 13 +-- .../bundle/invariant/redeploy/out.test.toml | 59 ---------- .../bundle/invariant/redeploy/output.txt | 1 - acceptance/bundle/invariant/redeploy/script | 97 ---------------- 10 files changed, 29 insertions(+), 322 deletions(-) delete mode 100644 acceptance/bundle/invariant/redeploy/out.test.toml delete mode 100644 acceptance/bundle/invariant/redeploy/output.txt delete mode 100644 acceptance/bundle/invariant/redeploy/script diff --git a/acceptance/bin/fuzz_gen_config.py b/acceptance/bin/fuzz_gen_config.py index 62b842443ef..250fe776604 100755 --- a/acceptance/bin/fuzz_gen_config.py +++ b/acceptance/bin/fuzz_gen_config.py @@ -9,7 +9,7 @@ (mutate_fuzz_config.py). Reads its inputs from the environment the invariant scripts already export: FUZZ_SEED, -FUZZ_SCHEMA, UNIQUE_NAME, FUZZ_RESOURCES, FUZZ_RESOURCE_COUNT, TESTDIR. +FUZZ_SCHEMA, UNIQUE_NAME, FUZZ_RESOURCES, TESTDIR. """ import json @@ -45,8 +45,7 @@ def generate(seed): schema = json.load(f) allowed = {r.strip() for r in os.environ.get("FUZZ_RESOURCES", "").split(",") if r.strip()} unique = f"{os.environ['UNIQUE_NAME']}-{seed}" - count = int(os.environ.get("FUZZ_RESOURCE_COUNT", "1")) - return to_yaml(gen_config(schema, seed, unique, allowed, count)) + return to_yaml(gen_config(schema, seed, unique, allowed)) def mutate_base(seed): diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 04249b549d9..00ff26211d0 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -3,11 +3,9 @@ Generate a random bundle config from the bundle JSON schema. Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf -branches) and emits one or more random resources as databricks.yml, seeded by --seed. -With --resource-count > 1 it also links resources with ${resources.*} references (each -resource referencing an earlier one) so the interpolation and deploy-ordering machinery is -exercised. Free-form scalars are occasionally replaced with dangerous / near-range-end -values (DANGEROUS_STRINGS, DANGEROUS_INTS) to probe the CLI's input handling. Feeds the +branches) and emits one random resource as databricks.yml, seeded by --seed. Free-form +scalars are occasionally replaced with dangerous / near-range-end values +(DANGEROUS_STRINGS, DANGEROUS_INTS) to probe the CLI's input handling. Feeds the invariant tests; the harness filters out configs the CLI rejects, so output may be structurally-random but sometimes invalid. """ @@ -375,7 +373,7 @@ def resource_types(schema, gen): return obj["properties"] -def gen_resource(schema, gen, types, candidates, seed, unique, index): +def gen_resource(schema, gen, types, candidates, seed, unique): rtype = gen.rng.choice(sorted(candidates)) # Each resource type is a map ref; the element schema is the object branch's @@ -384,77 +382,14 @@ def gen_resource(schema, gen, types, candidates, seed, unique, index): obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") element = obj["additionalProperties"] - # The first resource keeps the bare key/name so a seed produces the same first - # resource regardless of --resource-count; later resources are index-suffixed to - # stay unique within the config. - if index == 0: - key = f"fuzz_{rtype}_{seed}" - gen.unique = unique - else: - key = f"fuzz_{rtype}_{seed}_{index}" - gen.unique = f"{unique}-{index}" + key = f"fuzz_{rtype}_{seed}" + gen.unique = unique gen.rtype = rtype instance = gen.gen(element, 0) - return rtype, key, instance, gen.resolve(element) - - -def object_properties(gen, schema): - # The resource element is oneOf[object, ${...} string]; return the object - # branch's properties, matching the branch gen() picks to build the instance. - schema = gen.resolve(schema) - if "properties" in schema: - return schema["properties"] - for key in ("oneOf", "anyOf"): - for branch in schema.get(key, []): - resolved = gen.resolve(branch) - if "properties" in resolved: - return resolved["properties"] - return {} - - -def cross_ref_field(gen, element): - # A free-text scalar safe to overwrite with a reference; both names cover most - # resource types (jobs use "description", UC resources use "comment"). - props = object_properties(gen, element) - for field in ("description", "comment"): - if field in props: - return field - return None - - -def target_ref_field(instance): - # Reference the target's identity field: a string, so the type stays compatible - # with the description/comment field it lands in, and an input (not an output like - # ".id") so it resolves for every resource type and converges without drift. - # Output-field references are covered by the curated cross-ref configs. - for field in ("name", "display_name"): - if isinstance(instance.get(field), str): - return field - return None - - -def inject_cross_ref(gen, records): - # Link resources so deploy has to order them and resolve the references. A - # record may only reference an earlier one, so the reference graph stays - # acyclic: deploy must topologically order resources, and a cycle can't be - # ordered (the config would be rejected instead of exercising the invariant). - if len(records) < 2: - return - for i, source in enumerate(records): - if not source["ref_field"]: - continue - targets = [t for t in records[:i] if target_ref_field(t["instance"])] - if not targets: - continue - target = gen.rng.choice(targets) - field = target_ref_field(target["instance"]) - source["instance"][source["ref_field"]] = f"${{resources.{target['rtype']}.{target['key']}.{field}}}" - - -def gen_config(schema, seed, unique, allowed, resource_count=1): - if resource_count < 1: - sys.exit(f"gen_fuzz_config: --resource-count must be >= 1, got {resource_count}") + return rtype, key, instance + +def gen_config(schema, seed, unique, allowed): gen = Generator(schema, random.Random(seed), unique) types = resource_types(schema, gen) @@ -462,20 +397,11 @@ def gen_config(schema, seed, unique, allowed, resource_count=1): if not candidates: sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") - records = [] - for index in range(resource_count): - rtype, key, instance, element = gen_resource(schema, gen, types, candidates, seed, unique, index) - records.append({"rtype": rtype, "key": key, "instance": instance, "ref_field": cross_ref_field(gen, element)}) - - inject_cross_ref(gen, records) - - resources = {} - for record in records: - resources.setdefault(record["rtype"], {})[record["key"]] = record["instance"] + rtype, key, instance = gen_resource(schema, gen, types, candidates, seed, unique) return { "bundle": {"name": f"fuzz-{unique}"}, - "resources": resources, + "resources": {rtype: {key: instance}}, } @@ -528,19 +454,13 @@ def main(): default="", help="Comma-separated allow-list of resource types (default: all)", ) - parser.add_argument( - "--resource-count", - type=int, - default=1, - help="Number of resources to emit (default: 1)", - ) args = parser.parse_args() with open(args.schema) as f: schema = json.load(f) allowed = {r.strip() for r in args.resources.split(",") if r.strip()} - config = gen_config(schema, args.seed, args.unique, allowed, args.resource_count) + config = gen_config(schema, args.seed, args.unique, allowed) sys.stdout.write(to_yaml(config)) diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py index ea8b9db943a..f85b6963f0b 100755 --- a/acceptance/bin/gen_fuzz_config_check.py +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -60,57 +60,10 @@ def main(): sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line scalar contract: {line!r}\n") failed = True - # Multi-resource configs merge types under resources., and one resource - # references another's name so the interpolation/ordering path is exercised. - def resource_type(field): - return { - "oneOf": [ - { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": {"name": {"type": "string"}, field: {"type": "string"}}, - "required": ["name"], - }, - } - ] - } - - multi = gen_config( - { - "$defs": {}, - "properties": { - "resources": { - "oneOf": [ - { - "type": "object", - "properties": { - "jobs": resource_type("description"), - "volumes": resource_type("comment"), - }, - } - ] - } - }, - }, - seed=42, - unique="check", - allowed=set(), - resource_count=2, - ) - if len(multi["resources"]) < 1 or sum(len(v) for v in multi["resources"].values()) != 2: - sys.stderr.write("gen_config did not emit two resources\n") - failed = True - - values = [v for insts in multi["resources"].values() for inst in insts.values() for v in inst.values()] - if not any(isinstance(v, str) and v.startswith("${resources.") for v in values): - sys.stderr.write("gen_config did not inject a cross-resource reference\n") - failed = True - schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../bundle/schema/jsonschema.json") with open(schema_path) as f: schema = json.load(f) - seed24 = gen_config(schema, seed=24, unique="check", allowed={"registered_models"}, resource_count=1) + seed24 = gen_config(schema, seed=24, unique="check", allowed={"registered_models"}) rm = seed24["resources"]["registered_models"]["fuzz_registered_models_24"] if SKIP_PROPERTY_NAMES & set(rm): sys.stderr.write( diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 6ea27785957..4ca41a3cd62 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -8,20 +8,14 @@ In order to add a new test, add a config to configs/ and include it in test.toml The fuzz/ test generates random configs from the live `databricks bundle schema` (see fuzz/script) and runs each one through a real invariant test script. The target is selected by `FUZZ_TARGET` (matrixed in fuzz/test.toml); each target is also a curated -invariant test that runs over the `INPUT_CONFIG` matrix. `FUZZ_RESOURCE_COUNT` (also -matrixed in fuzz/test.toml) controls how many resources each generated config contains; -with more than one, the generator links them with `${resources.*}` references (each -resource referencing an earlier one, so the graph stays acyclic) so the interpolation and -deploy-ordering paths are exercised. Free-form scalars are occasionally replaced with -dangerous / near-range-end values (empty, whitespace, over-long, control characters, -int32/int64 boundaries) to probe the CLI's input handling. +invariant test that runs over the `INPUT_CONFIG` matrix. Free-form scalars are occasionally +replaced with dangerous / near-range-end values (empty, whitespace, over-long, control +characters, int32/int64 boundaries) to probe the CLI's input handling. - `no_drift` -- deploy, then no drift - `migrate` -- Terraform deploy, migrate to direct, then no drift -- `redeploy` -- deploy twice; the second deploy must be a no-op Since the schema comes from the CLI under test, an unrelated struct change can shift a seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), not flakiness; reproduce with `FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_TARGET=no_drift task test-fuzz`. -For a multi-resource repro, add `FUZZ_RESOURCE_COUNT=2`. diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml index 46d7d2e5a2a..6260c0bd6ac 100644 --- a/acceptance/bundle/invariant/fuzz/out.test.toml +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -3,6 +3,5 @@ Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.FUZZ_MODE = ["generate", "mutate"] -EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] -EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy"] +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate"] EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 86cb6bfea05..9afdc3e1194 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -66,7 +66,7 @@ run_seed() { # One machine-readable line per seed so a run is tallyable without grepping logs. A file, # not stdout, so the committed run's empty-output assertion holds. record() { - echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate} count=${FUZZ_RESOURCE_COUNT:-1}" >> LOG.summary + echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate}" >> LOG.summary } for ((offset = 0; offset < COUNT; offset++)); do @@ -94,7 +94,7 @@ for ((offset = 0; offset < COUNT; offset++)); do # empty-output assertion are unaffected. if [ -n "${FUZZ_CORPUS_DIR:-}" ]; then mkdir -p "$FUZZ_CORPUS_DIR" - cp "$dir/databricks.yml" "$FUZZ_CORPUS_DIR/${FUZZ_MODE:-generate}-${FUZZ_TARGET:-no_drift}-count${FUZZ_RESOURCE_COUNT:-1}-seed${seed}.yml" + cp "$dir/databricks.yml" "$FUZZ_CORPUS_DIR/${FUZZ_MODE:-generate}-${FUZZ_TARGET:-no_drift}-seed${seed}.yml" fi continue fi @@ -103,7 +103,7 @@ for ((offset = 0; offset < COUNT; offset++)); do # distinct from a drift bug; any goroutine dump is preserved in the seed's LOG.*. if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then record hang "$seed" - echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 + echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} task test-fuzz" >&2 exit 1 fi @@ -122,7 +122,7 @@ for ((offset = 0; offset < COUNT; offset++)); do if [ -n "$bug" ]; then record bug "$seed" - echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 + echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} task test-fuzz" >&2 exit 1 fi diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 6d109051c9e..24d27a9915e 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -12,14 +12,13 @@ IgnoreUnhandledRequests = true # Run the real invariant test script for each target. migrate ignores # DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. -EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy"] -EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] +# +# There is no redeploy target: no_drift's post-deploy plan already dry-runs what a redeploy +# would apply, so it catches the same field-level non-idempotency without the cost of a +# second deploy; the only surface redeploy adds (plan/apply divergence) never surfaced a bug +# across the nightly runs. +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate"] # generate = build a config from the schema (gen_fuzz_config.py); mutate = perturb a # curated invariant config (mutate_fuzz_config.py). Dispatched by fuzz_gen_config.py. EnvMatrix.FUZZ_MODE = ["generate", "mutate"] - -# mutate starts from a single-resource base and ignores FUZZ_RESOURCE_COUNT, so only run -# it once rather than duplicating the same output across the count matrix. -EnvMatrixExclude.mutate_count2 = ["FUZZ_MODE=mutate", "FUZZ_RESOURCE_COUNT=2"] -EnvMatrixExclude.mutate_count3 = ["FUZZ_MODE=mutate", "FUZZ_RESOURCE_COUNT=3"] diff --git a/acceptance/bundle/invariant/redeploy/out.test.toml b/acceptance/bundle/invariant/redeploy/out.test.toml deleted file mode 100644 index 19a27f26934..00000000000 --- a/acceptance/bundle/invariant/redeploy/out.test.toml +++ /dev/null @@ -1,59 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.INPUT_CONFIG = [ - "alert.yml.tmpl", - "app.yml.tmpl", - "catalog.yml.tmpl", - "cluster.yml.tmpl", - "cluster_apply_policy_default_values.yml.tmpl", - "dashboard.yml.tmpl", - "job_apply_policy_default_values_job_cluster.yml.tmpl", - "job_apply_policy_default_values_task_cluster.yml.tmpl", - "job_apply_policy_default_values_for_each_task.yml.tmpl", - "database_catalog.yml.tmpl", - "database_instance.yml.tmpl", - "experiment.yml.tmpl", - "external_location.yml.tmpl", - "genie_space.yml.tmpl", - "instance_pool.yml.tmpl", - "job.yml.tmpl", - "job_pydabs_10_tasks.yml.tmpl", - "job_pydabs_1000_tasks.yml.tmpl", - "job_cross_resource_ref.yml.tmpl", - "job_permission_ref.yml.tmpl", - "job_run.yml.tmpl", - "job_run_job_ref.yml.tmpl", - "job_with_depends_on.yml.tmpl", - "job_with_task.yml.tmpl", - "model.yml.tmpl", - "model_with_permissions.yml.tmpl", - "model_serving_endpoint.yml.tmpl", - "pipeline.yml.tmpl", - "pipeline_apply_policy_default_values.yml.tmpl", - "pipeline_config_dots.yml.tmpl", - "postgres_branch.yml.tmpl", - "postgres_catalog.yml.tmpl", - "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", - "postgres_project.yml.tmpl", - "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", - "registered_model.yml.tmpl", - "schema.yml.tmpl", - "schema_grant_ref.yml.tmpl", - "schema_uppercase_name.yml.tmpl", - "secret_scope.yml.tmpl", - "secret_scope_default_backend_type.yml.tmpl", - "sql_warehouse.yml.tmpl", - "synced_database_table.yml.tmpl", - "vector_search_endpoint.yml.tmpl", - "vector_search_index.yml.tmpl", - "volume.yml.tmpl", - "volume_external.yml.tmpl", - "volume_path_job_ref.yml.tmpl", - "volume_uppercase_name.yml.tmpl" -] diff --git a/acceptance/bundle/invariant/redeploy/output.txt b/acceptance/bundle/invariant/redeploy/output.txt deleted file mode 100644 index 7a28cb73a58..00000000000 --- a/acceptance/bundle/invariant/redeploy/output.txt +++ /dev/null @@ -1 +0,0 @@ -INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/redeploy/script b/acceptance/bundle/invariant/redeploy/script deleted file mode 100644 index af5bddd50e1..00000000000 --- a/acceptance/bundle/invariant/redeploy/script +++ /dev/null @@ -1,97 +0,0 @@ -# Invariant to test: a second deploy of an unchanged config is a clean no-op -# Additional checks: no internal errors / panics in validate/plan/deploy -# -# Catches create handlers that don't round-trip their inputs (or mutators that -# re-derive a field), which surface as a redeploy wanting to change or recreate. - -if [ -n "${FUZZ_SEED:-}" ]; then - fuzz_gen_config.py > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - cp databricks.yml LOG.config -else - # Copy data files to test directory - cp -r "$TESTDIR/../data/." . &> LOG.cp - - # Run init script if present - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" - if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init - fi - - envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - - cp databricks.yml LOG.config -fi - -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate - -cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null - -cleanup() { - # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -trace $CLI bundle deploy &> LOG.deploy -deploy_rc=$? -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null -if [ "$deploy_rc" -ne 0 ]; then - exit "$deploy_rc" -fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK - -# A fuzzed config can deploy yet legitimately fail to redeploy or differ, so the fuzzer -# sets SKIP_DRIFT_CHECK to swap the exact no-op check for the weaker but -# fidelity-independent plan-determinism oracle; curated configs check the no-op. -if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then - trace $CLI bundle deploy &> LOG.redeploy - cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null - - # Check both text and JSON plan for no changes - $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err - cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null - verify_no_drift.py LOG.planjson - - $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan - cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null -else - # Fuzz mode: the exact no-op check false-positives because the fake server does not - # round-trip every field, so a redeploy can legitimately want a destructive recreate - # (e.g. an immutable field the fake server reformats) and then fail non-interactively. - # That is a fidelity gap, not a bug, so tolerate a non-zero redeploy; a panic in it is - # still caught below. On a clean redeploy, assert the one fidelity-independent - # invariant: planning is deterministic -- two consecutive plans of the redeployed state - # must be byte-identical. A diff means nondeterministic planning/serialization - # (unstable map order, per-run randomness), a real bug. - redeploy_rc=0 - trace $CLI bundle deploy &> LOG.redeploy || redeploy_rc=$? - cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null - - if [ "$redeploy_rc" -eq 0 ]; then - # `|| true`: a plan that fails on an unstubbed read is a coverage gap, not a bug -- - # both plans are then empty and the diff trivially passes. - $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err || true - cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null - $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err || true - cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null - diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff - fi -fi From d389ca6144099edd91341ce9ec5fde95f6be7a78 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 24 Jul 2026 11:47:44 +0000 Subject: [PATCH 51/53] acc/fuzz: fix set -e deploy-code capture, gate validate, guard determinism diff The deploy_rc capture in no_drift/script and migrate/script was dead under `bash -euo pipefail`: a failing deploy aborted at the `trace` line before the capture, so the panic check never ran on a failed deploy. Wrap the deploy in `set +e`/`set -e` so the code is captured, the panic check runs even on failure (a panicking-but-rejected config is a bug, not a rejection), and only then do we exit with the deploy's code. Gate the pre-deploy `bundle validate` on FUZZ_SEED: deploy runs the same validate pipeline, so it is redundant for curated configs and only added value as an isolated panic surface for fuzzed configs. This also restores the curated tests to their prior behavior. Guard the fuzz-mode plan-determinism oracle to diff only when both plans succeed: a plan that fails on an unstubbed read is a coverage gap, and its partial output can differ run-to-run, which would false-positive. Reword the load_plan comment to drop the inaccurate fuzzer reference (only the curated drift check reaches it). --- acceptance/bin/util.py | 5 ++-- acceptance/bundle/invariant/migrate/script | 5 ++++ acceptance/bundle/invariant/no_drift/script | 33 +++++++++++++++------ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/acceptance/bin/util.py b/acceptance/bin/util.py index da100803191..9fbea462f68 100644 --- a/acceptance/bin/util.py +++ b/acceptance/bin/util.py @@ -34,8 +34,9 @@ def run(cmd): def load_plan(path): - # Empty or invalid output means `bundle plan` failed; exit cleanly (no traceback) - # so the fuzzer treats it as a rejected config, not a bug. Returns (data, raw). + # Empty or invalid output means `bundle plan` failed; exit cleanly with the reason + # (no traceback) rather than raising, so the failure reads as a plain message. + # Returns (data, raw). with open(path) as fobj: raw = fobj.read() if not raw.strip(): diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index a4357474be3..bcb9309d894 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -40,8 +40,13 @@ cleanup() { trap cleanup EXIT +# Disable set -e only around deploy so a failed deploy doesn't abort before we capture +# its code: the panic check below must run even on failure (a panicking-but-rejected +# config is a bug, not a rejection), and only then do we exit with the deploy's code. +set +e trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy deploy_rc=$? +set -e cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null if [ "$deploy_rc" -ne 0 ]; then exit "$deploy_rc" diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index bb5b54f331d..348dde6e981 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -20,10 +20,13 @@ else cp databricks.yml LOG.config fi -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate - -cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null +# Fuzz-only: give random configs an isolated validate panic check before deploy. Curated +# configs skip it -- deploy runs the same validate pipeline, so it would only be redundant. +# Output is redirected rather than recorded because a fuzzed config may produce warnings. +if [ -n "${FUZZ_SEED:-}" ]; then + trace $CLI bundle validate &> LOG.validate + cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null +fi cleanup() { # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. @@ -51,8 +54,13 @@ if [[ -n "$READPLAN" ]]; then cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi +# Disable set -e only around deploy so a failed deploy doesn't abort before we capture +# its code: the panic check below must run even on failure (a panicking-but-rejected +# config is a bug, not a rejection), and only then do we exit with the deploy's code. +set +e trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy deploy_rc=$? +set -e cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null if [ "$deploy_rc" -ne 0 ]; then exit "$deploy_rc" @@ -80,11 +88,18 @@ else # invariant that is independent of server fidelity: planning is deterministic. Two # consecutive plans of the same deployed state must be byte-identical; a diff means # nondeterministic planning/serialization (unstable map order, per-run randomness), a - # real bug. `|| true`: a plan that fails on an unstubbed read is a coverage gap, not a - # bug -- both plans are then empty and the diff trivially passes. - $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err || true + # real bug. Only compare when both plans succeed: a plan that fails on an unstubbed + # read is a coverage gap, not a bug, and its partial output can differ run-to-run, so + # diffing it would false-positive. Capture each plan's code under set +e for the guard. + set +e + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err + plan1_rc=$? + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err + plan2_rc=$? + set -e cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null - $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err || true cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null - diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + if [ "$plan1_rc" -eq 0 ] && [ "$plan2_rc" -eq 0 ]; then + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + fi fi From a0612c7054f960adcb0bff67dc74df883ae4868e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 24 Jul 2026 12:34:41 +0000 Subject: [PATCH 52/53] acc/fuzz: rename dispatcher, extract shared prologue, tighten comments Rename fuzz_gen_config.py to emit_fuzz_config.py so it no longer collides with the schema-walk generator gen_fuzz_config.py. Extract the shared config-render, cleanup, and deploy-capture prologue out of the no_drift and migrate scripts into prologue.sh. Shorten the comments added in this PR. --- .github/workflows/push.yml | 16 +- Taskfile.yml | 17 +-- acceptance/bin/check_schema_types.py | 2 +- ...fuzz_gen_config.py => emit_fuzz_config.py} | 20 ++- acceptance/bin/gen_fuzz_config.py | 138 ++++++++---------- acceptance/bin/gen_fuzz_config_check.py | 7 +- acceptance/bin/mutate_fuzz_config.py | 59 ++++---- acceptance/bin/mutate_fuzz_config_check.py | 18 +-- acceptance/bin/util.py | 5 +- acceptance/bundle/invariant/fuzz/script | 53 +++---- acceptance/bundle/invariant/fuzz/test.toml | 18 +-- acceptance/bundle/invariant/migrate/script | 52 +------ acceptance/bundle/invariant/no_drift/script | 82 ++--------- acceptance/bundle/invariant/prologue.sh | 58 ++++++++ acceptance/internal/config.go | 3 +- libs/testserver/server.go | 4 +- 16 files changed, 224 insertions(+), 328 deletions(-) rename acceptance/bin/{fuzz_gen_config.py => emit_fuzz_config.py} (68%) create mode 100644 acceptance/bundle/invariant/prologue.sh diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 50ce84d627c..5bc7aff581d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -412,9 +412,8 @@ jobs: needs: - cleanups - # Wide rotating seed window with drift checking on: too slow for every PR, so - # nightly only and not part of test-result. The committed acceptance test still - # checks the no-panic invariant on a small fixed window per PR. + # Wide rotating seed window with drift on: too slow for every PR, so nightly only and + # not part of test-result. The committed acceptance test still checks no-panic per PR. if: ${{ github.event_name == 'schedule' }} name: "task test-fuzz" runs-on: @@ -428,7 +427,7 @@ jobs: permissions: id-token: write contents: read - # Failure-reporting step comments on the PR that introduced the failing commit. + # For the failure-reporting step's PR comment. pull-requests: write steps: @@ -444,13 +443,12 @@ jobs: env: FUZZ_SEED_COUNT: "25" run: | - # start = monotonic GITHUB_RUN_NUMBER * COUNT keeps each nightly window - # non-overlapping, so CI explores new configs every run. + # GITHUB_RUN_NUMBER * COUNT keeps each nightly window non-overlapping, so CI + # explores new configs every run. export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) go tool -modfile=tools/task/go.mod task test-fuzz - # Not in test-result, so surface failures by commenting on the PR that - # introduced the commit under test. + # Not in test-result, so surface failures by commenting on the PR under test. - name: Report failure if: ${{ failure() }} env: @@ -472,7 +470,7 @@ jobs: EOF ) - # The commit's pulls endpoint returns the merged PR that introduced it. + # The commit's pulls endpoint returns the PR that introduced it. pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$COMMIT/pulls" --jq '.[0].number // empty') if [ -n "$pr" ]; then gh pr comment "$pr" --body "$body" diff --git a/Taskfile.yml b/Taskfile.yml index 0737ca612f6..739e7ba2b69 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -732,14 +732,12 @@ tasks: test-fuzz: desc: Run schema fuzz invariant tests (random configs, direct engine) - # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't - # see, so always run rather than no-op a repro or shifted nightly window. + # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't see. cmds: - | - # Wider window than the committed run, with drift checking on; a repro can - # narrow it via FUZZ_SEED_START/COUNT. Slow variants can't finish 200 seeds inside - # the per-variant Timeout, but the fuzz script's default FUZZ_TIME_BUDGET stops each - # variant cleanly (as many seeds as fit, then pass) rather than being force-killed. + # Wider window than the committed run, with drift on; a repro narrows it via + # FUZZ_SEED_START/COUNT. FUZZ_TIME_BUDGET stops each variant cleanly if 200 seeds + # don't fit the Timeout, rather than letting it be force-killed. export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" {{.GO_TOOL}} gotestsum \ @@ -755,10 +753,9 @@ tasks: - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/ - mkdir -p ./acceptance/build/cover-fuzz-merged/ - | - # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, - # so every fuzzed `bundle` invocation drops counter files we can aggregate. Drift - # off (no FUZZ_CHECK_DRIFT) so the run exercises the full deploy/plan path for as - # many seeds as possible instead of stopping on the first fake-server drift. + # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, so + # every fuzzed `bundle` invocation drops aggregatable counter files. Drift off so the + # run exercises the full deploy/plan path instead of stopping on the first drift. export CLI_GOCOVERDIR=build/cover-fuzz export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}" export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" diff --git a/acceptance/bin/check_schema_types.py b/acceptance/bin/check_schema_types.py index 6f9e72c0f8d..40c9a9fb7d6 100755 --- a/acceptance/bin/check_schema_types.py +++ b/acceptance/bin/check_schema_types.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Assert every `type` in the bundle schema is one gen_fuzz_config.py can generate, so a new -libs/jsonschema.Type fails loudly here instead of being silently skipped by the fuzz loop. +libs/jsonschema.Type fails loudly here instead of being silently skipped by the fuzzer. """ import argparse diff --git a/acceptance/bin/fuzz_gen_config.py b/acceptance/bin/emit_fuzz_config.py similarity index 68% rename from acceptance/bin/fuzz_gen_config.py rename to acceptance/bin/emit_fuzz_config.py index 250fe776604..4d83088e5e1 100755 --- a/acceptance/bin/fuzz_gen_config.py +++ b/acceptance/bin/emit_fuzz_config.py @@ -1,15 +1,13 @@ #!/usr/bin/env python3 """ Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from -FUZZ_MODE so the invariant target scripts don't each duplicate the branch: +FUZZ_MODE so the invariant scripts don't each duplicate the branch: - generate (default) - build a config from scratch by walking `bundle schema` - (gen_fuzz_config.py). - mutate - start from a curated invariant config and perturb it - (mutate_fuzz_config.py). + generate (default) - build from scratch by walking `bundle schema` (gen_fuzz_config.py). + mutate - perturb a curated invariant config (mutate_fuzz_config.py). -Reads its inputs from the environment the invariant scripts already export: FUZZ_SEED, -FUZZ_SCHEMA, UNIQUE_NAME, FUZZ_RESOURCES, TESTDIR. +Reads its inputs from the environment the invariant scripts export: FUZZ_SEED, FUZZ_SCHEMA, +UNIQUE_NAME, FUZZ_RESOURCES, TESTDIR. """ import json @@ -22,9 +20,9 @@ from gen_fuzz_config import gen_config, to_yaml from mutate_fuzz_config import load_yaml, mutate -# Curated single-resource configs that deploy standalone against the fake server (only -# $UNIQUE_NAME, no init script). All are also in the invariant INPUT_CONFIG matrix, so -# they stay deploy-verified. The seed selects one; mutate_fuzz_config perturbs it. +# Curated single-resource configs that deploy standalone (only $UNIQUE_NAME, no init +# script). All are in the invariant INPUT_CONFIG matrix, so they stay deploy-verified. +# The seed selects one; mutate_fuzz_config perturbs it. MUTATE_BASES = [ "catalog", "external_location", @@ -69,7 +67,7 @@ def main(): elif mode == "mutate": sys.stdout.write(mutate_base(seed)) else: - sys.exit(f"fuzz_gen_config: unknown FUZZ_MODE {mode!r}") + sys.exit(f"emit_fuzz_config: unknown FUZZ_MODE {mode!r}") if __name__ == "__main__": diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py index 00ff26211d0..b8716f203ba 100755 --- a/acceptance/bin/gen_fuzz_config.py +++ b/acceptance/bin/gen_fuzz_config.py @@ -2,12 +2,10 @@ """ Generate a random bundle config from the bundle JSON schema. -Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf -branches) and emits one random resource as databricks.yml, seeded by --seed. Free-form -scalars are occasionally replaced with dangerous / near-range-end values -(DANGEROUS_STRINGS, DANGEROUS_INTS) to probe the CLI's input handling. Feeds the -invariant tests; the harness filters out configs the CLI rejects, so output may be -structurally-random but sometimes invalid. +Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf branches) +and emits one random resource, seeded by --seed. Free-form scalars are sometimes replaced +with dangerous values (DANGEROUS_STRINGS/INTS) to probe input handling. The harness drops +configs the CLI rejects, so output may be structurally random but invalid. """ import argparse @@ -28,19 +26,14 @@ SCALAR_TYPES = {"boolean", "integer", "number", "string"} HANDLED_TYPES = SCALAR_TYPES | {"object", "array"} -# Cross-resource references must resolve to objects that exist on every -# workspace (the fake test server and real UC alike). "main"/"default" are the -# standard seeded catalog/schema; these mirror acceptance/bundle/invariant/configs. -# Without pinning, the generator emits random names that the fake server accepts -# but real UC rejects (e.g. CATALOG_DOES_NOT_EXIST), so the config is dropped at -# deploy and never exercises the invariant. +# Cross-resource refs must resolve on every workspace (fake server and real UC). +# "main"/"default" are the standard seeded catalog/schema; a random name deploys on the +# fake server but real UC rejects it (CATALOG_DOES_NOT_EXIST), dropping the config. DEFAULT_CATALOG = "main" DEFAULT_SCHEMA = "default" -# "account users" is a group present on every workspace, plus one privilege UC -# accepts for each grant-bearing securable type (from the curated configs). Real -# UC rejects an unknown principal or a privilege that doesn't apply to the -# securable, so a random grant would deploy on the fake server yet fail on cloud. +# "account users" exists on every workspace; each securable gets one privilege UC accepts. +# A random principal or inapplicable privilege deploys on the fake server but fails on UC. DEFAULT_PRINCIPAL = "account users" GRANT_PRIVILEGE = { "catalogs": "USE_CATALOG", @@ -51,8 +44,8 @@ "vector_search_indexes": "SELECT", } -# Permissions blocks cannot be variable references; each entry needs a concrete -# principal and a level valid for the resource type (see invariant configs). +# Permissions can't be variable refs; each entry needs a concrete principal and a level +# valid for the resource type. DEFAULT_PERMISSION_GROUP = "users" PERMISSION_LEVEL = { "alerts": "CAN_MANAGE", @@ -72,18 +65,15 @@ "vector_search_endpoints": "CAN_USE", } -# Fields the bundle schema still lists but the user never sets (backend output / -# computed). Emitting them causes false drift after terraform→direct migrate. -# Keep in sync with bundle/direct/dresources/resources.yml output_only and -# backend_defaults where the field is not user-writable. +# Output-only/computed fields the schema lists but users never set; emitting them causes +# false drift after migrate. Mirrors output_only/backend_defaults in dresources/resources.yml. SKIP_PROPERTY_NAMES = frozenset( { "browse_only", "created_at", "created_by", "creator_name", - # An etag is a read value the backend assigns; the CLI rejects one set in - # bundle config (e.g. "genie space ... has an etag set. Etags must not be set"). + # Backend-assigned; the CLI rejects an etag set in bundle config. "etag", "full_name", "metastore_id", @@ -94,9 +84,8 @@ } ) -# Resource types whose schema omits required[] (or whose required[] can't be honored -# in bundle YAML) but which need these fields to deploy. See RESOURCE_FIELD_ALLOWLIST -# and the *_BY_RESOURCE tables below for the values these fields take. +# Fields these resources need to deploy but that the schema's required[] omits (or can't +# express in YAML). Values come from the *_BY_RESOURCE tables below. RESOURCE_REQUIRED_FIELDS = { "registered_models": frozenset({"catalog_name", "name", "schema_name"}), "dashboards": frozenset({"display_name", "file_path", "warehouse_id"}), @@ -105,24 +94,22 @@ "genie_spaces": frozenset({"serialized_space", "title", "warehouse_id"}), } -# Fields to drop for a specific resource type because they conflict with the field -# set we do emit. Dashboards and Genie spaces take their body from file_path XOR an -# inline serialized_* field, so emitting both is rejected ("both ... are set"). +# Fields that conflict with the set we emit. Dashboards/Genie spaces take their body from +# file_path XOR an inline serialized_* field; emitting both is rejected. RESOURCE_SKIP_FIELDS = { "dashboards": frozenset({"serialized_dashboard"}), "genie_spaces": frozenset({"file_path"}), "apps": frozenset({"git_repository", "git_source"}), } -# Resource types where only a fixed field set is allowed in bundle YAML. Alerts read -# their spec from the .dbalert.json referenced by file_path; the CLI rejects any other -# field (see bundle/config/mutator/load_dbalert_files.go allowedInYAML). +# Resources allowing only a fixed field set in YAML. Alerts read their spec from the +# .dbalert.json at file_path; the CLI rejects other fields (load_dbalert_files.go). RESOURCE_FIELD_ALLOWLIST = { "alerts": frozenset({"display_name", "file_path", "lifecycle", "permissions", "warehouse_id"}), } -# file_path points at a serialized-body fixture copied into every seed dir from -# acceptance/bundle/invariant/data (see fuzz/script). The extension selects the parser. +# Serialized-body fixtures copied into each seed dir from invariant/data; the extension +# selects the parser. FILE_PATH_BY_RESOURCE = { "dashboards": "./dashboard.lvdash.json", "alerts": "./alert.dbalert.json", @@ -135,19 +122,17 @@ # existence/extension check a bare token would fail. NOTEBOOK_PATH = "/Shared/notebook" -# parent_path is a workspace folder; a dangerous value is rejected by the backend and, on -# read, the CLI re-adds the /Workspace prefix, so a mismatch plans a spurious recreate. Pin -# it to a valid folder so the fuzzer exercises deploy instead. +# parent_path is a workspace folder; pin it to a valid one. The CLI re-adds the /Workspace +# prefix on read, so a mismatched value plans a spurious recreate. PARENT_PATH = "/Workspace/Shared" -# Fields declared as string in the schema but parsed as google.protobuf.Duration at -# config load (e.g. suspend_timeout_duration, ttl); a bare token fails to parse. +# String in the schema but parsed as protobuf.Duration at load (suspend_timeout_duration, +# ttl); a bare token fails to parse. DURATION_VALUE = "3600s" -# Dangerous / near-range-end probes injected into free-form scalars: empty and -# whitespace-only strings, an over-long string, embedded newlines/tabs, non-ASCII, quotes, -# a dangling ${...} reference, a path-traversal string, and int32/int64 boundaries. The CLI -# must reject or round-trip these without panicking; mutate_fuzz_config.py reuses both lists. +# Dangerous/near-range-end probes for free-form scalars: empty, whitespace, over-long, +# newlines/tabs, non-ASCII, quotes, a dangling ${...} ref, path traversal, int boundaries. +# The CLI must reject or round-trip these without panicking; mutate_fuzz_config reuses them. DANGEROUS_STRINGS = [ "", " ", @@ -166,8 +151,8 @@ -1, ] -# Only inject a dangerous value some of the time: a fuzzed field mostly keeps a plausible -# value so the config still deploys and exercises the invariant, not just the reject path. +# Inject a dangerous value only sometimes, so the config usually still deploys and exercises +# the invariant, not just the reject path. DANGEROUS_PROB = 0.15 @@ -176,13 +161,12 @@ def __init__(self, schema, rng, unique): self.root = schema self.rng = rng self.unique = unique - # Set to the top-level resource type before generating its element, so - # grants can pick a privilege valid for that securable. + # Top-level resource type, set before generating its element so grants/permissions + # can pick a value valid for that securable. self.rtype = None def resolve(self, schema): - # Follow $ref chains, e.g. "#/$defs/github.com/.../resources.Job", nested - # under $defs by "/"-separated path segments. + # Follow $ref chains ("#/$defs/.../resources.Job"), indexing $defs by path segment. while isinstance(schema, dict) and "$ref" in schema: cur = self.root["$defs"] for part in schema["$ref"].split("/")[2:]: @@ -220,8 +204,8 @@ def should_skip_property(self, prop_name, prop_schema): return False def gen(self, schema, depth, name=""): - # A Genie space body is a free-form interface{}; the backend rejects unknown - # keys, so emit the minimal accepted body instead of a random object. + # A Genie space body is free-form but the backend rejects unknown keys, so emit + # the minimal accepted body instead of a random object. if name == "serialized_space": return {"version": 1} @@ -263,14 +247,14 @@ def gen_object(self, schema, depth): result = {} for prop_name, prop_schema in props.items(): - # A restricted resource (e.g. alerts) rejects any field outside its - # allow-list, even a schema-required one supplied via the file instead. + # A restricted resource (e.g. alerts) rejects any field outside its allow-list, + # even a schema-required one it reads from the file instead. if allowlist is not None and prop_name not in allowlist: continue if self.should_skip_property(prop_name, prop_schema): continue - # Always emit required fields; emit optional ones less often as we go - # deeper to keep configs from exploding. + # Always emit required fields; emit optional ones less often deeper down to keep + # configs from exploding. keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) if not keep: continue @@ -278,8 +262,8 @@ def gen_object(self, schema, depth): if value is not None: result[prop_name] = value - # Map type (additionalProperties, no fixed properties): synthesize a few - # random keys, e.g. resources. or string maps like tags. + # Map type (additionalProperties, no fixed properties): synthesize a few random + # keys, e.g. resources. or string maps like tags. if self.is_map(schema): for _ in range(self.rng.randint(1, 2)): key = self.token() @@ -294,16 +278,16 @@ def gen_array(self, schema, depth, name): return [self.gen(items, depth + 1, name) for _ in range(self.rng.randint(1, 3))] def gen_grants(self): - # One known-good grant for the current securable. Skip grants for a type - # we have no valid privilege for, rather than emit one real UC rejects. + # One known-good grant for the securable; skip types we have no valid privilege for + # rather than emit one UC rejects. privilege = GRANT_PRIVILEGE.get(self.rtype) if privilege is None: return [] return [{"principal": DEFAULT_PRINCIPAL, "privileges": [privilege]}] def gen_permissions(self): - # One known-good permission for the current resource. Skip types we have - # no valid level for, rather than emit a random principal or ${...} ref. + # One known-good permission for the resource; skip types we have no valid level for + # rather than emit a random principal or ${...} ref. level = PERMISSION_LEVEL.get(self.rtype) if level is None: return [] @@ -312,13 +296,12 @@ def gen_permissions(self): def gen_scalar(self, schema, name): t = schema.get("type") if t == "boolean": - # The invariant cleanup traps destroy the bundle, which must succeed. + # Cleanup must be able to destroy the bundle. if name == "prevent_destroy": return False return self.rng.choice([True, False]) if t == "integer": - # The field is in hours, but UC validates it as a window of 0 or 7-30 - # days; only 0 or 168-720 (hours) are accepted. + # In hours, but UC accepts only a window of 0 or 7-30 days (0 or 168-720 hours). if name == "custom_max_retention_hours": return self.rng.choice([0, self.rng.randint(168, 720)]) if self.rng.random() < DANGEROUS_PROB: @@ -329,9 +312,8 @@ def gen_scalar(self, schema, name): # Fail loud on an unknown type; a missing type is "any" and falls through to string. if t is not None and t not in SCALAR_TYPES: sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") - # string (default) - # Pin cross-resource references and typed-string fields to values the backend - # accepts; a random token fails format/existence validation and drops the config. + # string (default). Pin cross-resource refs and typed-string fields to accepted + # values; a random token fails format/existence validation and drops the config. if name == "catalog_name": return DEFAULT_CATALOG if name == "schema_name": @@ -349,15 +331,13 @@ def gen_scalar(self, schema, name): if name.endswith("_duration") or name == "ttl": return DURATION_VALUE if name == "name" and self.rtype == "vector_search_indexes": - # UC requires the full three-level catalog.schema.table name, and each - # part accepts only alphanumerics and underscores. + # UC requires the full catalog.schema.table name; each part is alphanumeric+_. table = re.sub(r"[^0-9a-zA-Z_]", "_", f"fuzz_index_{self.unique}") return f"{DEFAULT_CATALOG}.{DEFAULT_SCHEMA}.{table}" if name in ("name", "display_name"): return f"fuzz-{name}-{self.unique}" - # A free-form string with no pinned meaning (e.g. description, comment, tag value): - # probe dangerous / near-range-end input here, where a rejected or normalized value - # doesn't just fail the field-format check a pinned field above guards against. + # Free-form string with no pinned meaning (description, comment, tag): safe to probe + # dangerous input here, unlike the pinned fields above. if self.rng.random() < DANGEROUS_PROB: return self.rng.choice(DANGEROUS_STRINGS) return self.token() @@ -376,8 +356,7 @@ def resource_types(schema, gen): def gen_resource(schema, gen, types, candidates, seed, unique): rtype = gen.rng.choice(sorted(candidates)) - # Each resource type is a map ref; the element schema is the object branch's - # additionalProperties. + # Each type is a map; the element schema is the object branch's additionalProperties. map_schema = gen.resolve(types[rtype]) obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") element = obj["additionalProperties"] @@ -435,12 +414,9 @@ def to_yaml(obj, indent=0, list_item=False): def dump_scalar(v): - # ensure_ascii=False keeps non-ASCII as literal UTF-8. The default would escape an - # astral char (e.g. the 🚀 probe) into a UTF-16 surrogate pair (\ud83d\ude80), which - # YAML's parser rejects as an "invalid Unicode character escape code" -- so the config - # dies at parse time and never reaches bundle logic. A literal UTF-8 scalar is valid - # YAML and exercises the CLI's actual unicode handling instead. Control chars (\n, \t) - # are still escaped by json.dumps regardless, which YAML accepts. + # ensure_ascii=False keeps non-ASCII as literal UTF-8. The default escapes astral chars + # (e.g. the 🚀 probe) into surrogate pairs that YAML rejects, killing the config at parse + # time before it reaches bundle logic. Control chars stay escaped by json.dumps (YAML ok). return json.dumps(v, ensure_ascii=False) diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py index f85b6963f0b..adae2f15018 100755 --- a/acceptance/bin/gen_fuzz_config_check.py +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """ Contract check for gen_fuzz_config.to_yaml: every scalar is on its own line as -`key: `. mutate_fuzz_config.py's line-based loader relies on this. Prints each -case's YAML (diffed by the harness) and exits non-zero on a violation. +`key: `, which mutate_fuzz_config's line-based loader relies on. Prints each case's +YAML (diffed by the harness) and exits non-zero on a violation. """ import json @@ -52,8 +52,7 @@ def main(): sys.stderr.write(f"contract violation: not `key: `: {line!r}\n") failed = True - # Generate mode emits DANGEROUS_STRINGS into free-form fields; each must still - # serialize to a single `key: ` line so the line-based loader can parse it. + # Each DANGEROUS_STRINGS probe must still serialize to a single `key: ` line. for i, val in enumerate(DANGEROUS_STRINGS): line = to_yaml({"description": val}).rstrip("\n") if "\n" in line or not SCALAR.fullmatch(line): diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py index de861e518ab..a53494312fb 100755 --- a/acceptance/bin/mutate_fuzz_config.py +++ b/acceptance/bin/mutate_fuzz_config.py @@ -2,26 +2,19 @@ """ Mutate a known-good bundle config by deleting, perturbing, and adding random fields. -Complements gen_fuzz_config.py (generate-from-scratch via schema walk): instead of -building a config from the schema, this starts from a curated invariant config that -already deploys and applies a few seeded mutations. It exercises the CLI's handling of -perturbed-but-realistic input, and reaches a much higher deploy rate than the schema -walk, since the base already resolves. - -Two kinds of mutation, chosen per step: - -- destructive (always): delete a field, or replace it with a fuzz token, a - boundary/dangerous value, or an empty container. Probes the reject/no-panic path. -- additive (only with a schema): inject a valid optional field the base omits, valued by - the schema generator. Destructive ops stay within the base's field set, so they only - find reject/panic bugs; adding a valid optional field to a still-deploying config is - what reaches reconcile/drift bugs (the field space the schema walk explores). - -Reads the base databricks.yml (already envsubst-rendered) from stdin, writes the mutated -config to stdout. --seed makes the mutation reproducible; --schema enables the additive op. - -The invariant harness only asserts no-panic on fuzzed configs (SKIP_DRIFT_CHECK), so a -mutation that makes the config invalid is fine: the CLI must reject it cleanly, not crash. +Complements gen_fuzz_config.py: instead of building from the schema, this perturbs a +curated invariant config that already deploys, so it reaches a much higher deploy rate. + +Two mutation kinds, chosen per step: + +- destructive (always): delete a field or replace it with a token, a dangerous value, or + an empty container. Stays within the base's fields, so it finds only reject/panic bugs. +- additive (with --schema): inject a valid optional field the base omits, valued by the + schema generator. This is what reaches reconcile/drift bugs. + +Reads the base databricks.yml (envsubst-rendered) from stdin, writes the mutated config to +stdout. --seed makes it reproducible. The harness only asserts no-panic on fuzzed configs, +so an invalid mutation is fine: the CLI must reject it cleanly, not crash. """ import argparse @@ -34,18 +27,17 @@ from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, Generator, resource_types, to_yaml -# Same near-range-end and dangerous-character probes the schema-walk generator injects into -# free-form scalars; here we drop them onto any field (see mutate_once). +# The same probes gen_fuzz_config injects into free-form scalars, dropped onto any field. DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS # Chance a step injects a field rather than perturbing one. Biased high: injection is the -# path to drift bugs, and destructive coverage is already dense (1-3 steps per seed). +# path to drift bugs, and destructive coverage is already dense. ADD_PROB = 0.6 def tokenize(text): # (indent, content) per non-blank, non-comment line. Only full-line comments are - # stripped; the curated bases don't use trailing "#" in values. + # stripped; the curated bases have no trailing "#" in values. out = [] for raw in text.splitlines(): stripped = raw.lstrip(" ") @@ -135,7 +127,7 @@ def load_yaml(text): def collect(node, out): - # (container, key) for every child, so a mutation can delete or replace it in place. + # (container, key) per child, so a mutation can delete or replace it in place. if isinstance(node, dict): for k, v in node.items(): out.append((node, k)) @@ -169,22 +161,22 @@ def mutate_once(rng, roots): def resource_element(gen, type_schema): - # The instance schema is the map's object-branch additionalProperties (as gen_resource). + # The instance schema is the map's object-branch additionalProperties. map_schema = gen.resolve(type_schema) obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") return obj["additionalProperties"] def collect_insertions(gen, node, schema, rtype, out): - # Record every writable optional field absent from an existing object, walking the node - # alongside its schema so nested objects (not just the top level) are candidates too. + # Record every writable optional field absent from an object, walking node and schema + # together so nested objects are candidates too. schema = gen.resolve(schema) if not isinstance(schema, dict): return branches = schema.get("oneOf") or schema.get("anyOf") if branches: - # Pick the branch matching the node we actually have, not a random one. + # Pick the branch matching the node we have, not a random one. picked = None for branch in branches: resolved = gen.resolve(branch) @@ -235,7 +227,7 @@ def add_field(gen, rng, config): if not points: return node, name, prop_schema, rtype = rng.choice(points) - # rtype drives grants/permissions/typed-string generation (see gen_scalar/gen_grants). + # rtype drives grants/permissions/typed-string generation. gen.rtype = rtype value = gen.gen(prop_schema, 1, name) if value is not None: @@ -246,9 +238,8 @@ def mutate(config, seed, schema=None, unique="fuzz"): rng = random.Random(seed) gen = Generator(schema, rng, unique) if schema is not None else None - # Mutate only inside resource instances: keep bundle/name and the - # resources.. skeleton so there is always something to deploy, while - # every field of the instance (including required ones) is fair game. + # Mutate only inside resource instances: keep the bundle/name and resources skeleton so + # there is always something to deploy, while every instance field is fair game. roots = [] for instances in config.get("resources", {}).values(): if isinstance(instances, dict): @@ -256,7 +247,7 @@ def mutate(config, seed, schema=None, unique="fuzz"): for _ in range(rng.randint(1, 3)): # gen is None short-circuits before rng is touched, so the no-schema path keeps its - # exact RNG stream (and committed selftest output) unchanged. + # exact RNG stream unchanged. if gen is not None and rng.random() < ADD_PROB: add_field(gen, rng, config) else: diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py index a56101abde4..28f815ecf74 100755 --- a/acceptance/bin/mutate_fuzz_config_check.py +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -3,13 +3,11 @@ Contract check for mutate_fuzz_config's minimal YAML loader and mutation engine (the harness diffs stdout; a non-zero exit marks a violation on stderr): -- The loader round-trips every curated base config: load -> to_yaml -> load is a - fixed point, so a base template the loader can't represent is caught here rather - than as a confusing fuzz failure. +- The loader round-trips every curated base: load -> to_yaml -> load is a fixed point, so + a base the loader can't represent is caught here, not as a confusing fuzz failure. - Mutation is deterministic for a fixed seed (reproducible repros). -It also prints a few mutated configs so an accidental change to the algorithm shows up -as an output diff. +It also prints a few mutated configs so an algorithm change shows up as an output diff. """ import json @@ -19,7 +17,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from envsubst import substitute_variables -from fuzz_gen_config import MUTATE_BASES +from emit_fuzz_config import MUTATE_BASES from gen_fuzz_config import SKIP_PROPERTY_NAMES from mutate_fuzz_config import load_yaml, mutate, to_yaml @@ -51,7 +49,7 @@ def main(): sys.stderr.write(f"{name}: base did not parse to a config with resources\n") failed = True continue - # load -> emit -> load must be a fixed point. + # load -> emit -> load is a fixed point. if load_yaml(to_yaml(parsed)) != parsed: sys.stderr.write(f"{name}: loader is not a round-trip fixed point\n") failed = True @@ -68,8 +66,8 @@ def main(): sys.stdout.write(f"=== volume seed={seed} ===\n") sys.stdout.write(to_yaml(mutate(load_yaml(render("volume")), seed))) - # Assert-only (no stdout) so this golden doesn't churn as the schema grows. The - # registered_model base sets none of its optional fields, so any added field is injected. + # Assert-only (no stdout) so this doesn't churn as the schema grows. The registered_model + # base sets no optional fields, so any added field must have been injected. with open(SCHEMA) as f: schema = json.load(f) @@ -87,7 +85,7 @@ def main(): added = fields - base_fields if added: injected = True - # Injecting an output-only field (SKIP_PROPERTY_NAMES) would manufacture false drift. + # Injecting an output-only field would manufacture false drift. leaked = SKIP_PROPERTY_NAMES & added if leaked: sys.stderr.write(f"seed {seed}: injected output-only field(s): {sorted(leaked)}\n") diff --git a/acceptance/bin/util.py b/acceptance/bin/util.py index 9fbea462f68..84185eaf55f 100644 --- a/acceptance/bin/util.py +++ b/acceptance/bin/util.py @@ -34,9 +34,8 @@ def run(cmd): def load_plan(path): - # Empty or invalid output means `bundle plan` failed; exit cleanly with the reason - # (no traceback) rather than raising, so the failure reads as a plain message. - # Returns (data, raw). + # Empty or invalid output means `bundle plan` failed; exit with a plain message instead + # of a traceback. Returns (data, raw). with open(path) as fobj: raw = fobj.read() if not raw.strip(): diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script index 9afdc3e1194..bd0d9d3ba7a 100644 --- a/acceptance/bundle/invariant/fuzz/script +++ b/acceptance/bundle/invariant/fuzz/script @@ -1,25 +1,20 @@ -# Invariant fuzzing: generate a random config per seed and run the real invariant test -# script (no_drift/script or migrate/script). Rejection vs bug: those scripts print -# INPUT_CONFIG_OK once a config deploys, so a non-zero result before the marker is a -# rejection; a panic anywhere, or a failure after it, is a bug. -# +# Invariant fuzzing: generate a random config per seed and run the real invariant script +# (no_drift/migrate). Those print INPUT_CONFIG_OK once a config deploys, so a non-zero +# result before the marker is a rejection; a panic anywhere, or a failure after it, is a bug. # Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet legitimately # differ from the fake server, so the committed run asserts only no-panic. START="${FUZZ_SEED_START:-0}" COUNT="${FUZZ_SEED_COUNT:-5}" -# Per-seed cap: a seed past this generous budget is stuck, not slow, so SIGQUIT (Go dumps -# goroutines for triage) then SIGKILL and flag a hang. Needs GNU timeout; else uncapped. +# Per-seed cap: a seed past this budget is stuck, not slow, so SIGQUIT (for Go's goroutine +# dump) then SIGKILL and flag a hang. Needs GNU timeout; else uncapped. SEED_TIMEOUT="${FUZZ_SEED_TIMEOUT:-180}" -# Overall budget (seconds): stop starting new seeds past it and exit cleanly, so a -# slow-but-progressing variant (the mutate/deploy combos never finish a large seed window) -# is not force-killed at the per-script Timeout and read as a failure. Defaults on rather -# than opt-in so a direct `go test` run truncates cleanly too, not just `task test-fuzz`. -# 900s leaves margin under the 20m Timeout in test.toml for the last-started seed (capped at -# SEED_TIMEOUT) plus teardown; keep it comfortably below Timeout - SEED_TIMEOUT. Set -# FUZZ_TIME_BUDGET=0 to disable the cap (e.g. a completionist run with a raised Timeout). +# Overall budget (seconds): stop starting new seeds past it and exit cleanly, so a slow-but- +# progressing variant isn't force-killed at the per-script Timeout and read as a failure. +# On by default so a direct `go test` truncates cleanly too. 900s leaves margin under the +# 20m test.toml Timeout for the last seed plus teardown. Set FUZZ_TIME_BUDGET=0 to disable. BUDGET="${FUZZ_TIME_BUDGET:-900}" if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then @@ -36,19 +31,18 @@ cat LOG.schema.err | contains.py '!panic:' '!internal error' > /dev/null # Fail loud on a schema type the generator can't produce (the loop below would hide it). check_schema_types.py --schema schema.json -# One seed's worth of work, factored out so it can run either directly or under `timeout`. +# One seed's worth of work, factored out so it can run directly or under `timeout`. seed_body() { cd "$1" - # The generator points file_path/source_code_path fields at these fixtures - # (see gen_fuzz_config.py *_BY_RESOURCE), so make them resolvable from the - # bundle root, matching how the curated invariant scripts stage data/. + # Stage the fixtures the generator's file_path/source_code_path fields point at, as the + # curated scripts do. cp -r "$TESTDIR/../data/." . export FUZZ_SEED="$2" export FUZZ_SCHEMA="../schema.json" source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" } -# timeout spawns a fresh bash without our shell functions, so export them (and seed_body). +# timeout spawns a fresh bash without our functions, so export them (and seed_body). export -f $(compgen -A function) # Run one seed, capped at SEED_TIMEOUT when `timeout` is available. @@ -63,15 +57,15 @@ run_seed() { fi } -# One machine-readable line per seed so a run is tallyable without grepping logs. A file, -# not stdout, so the committed run's empty-output assertion holds. +# One machine-readable line per seed, tallyable without grepping logs. To a file, not +# stdout, so the committed run's empty-output assertion holds. record() { echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate}" >> LOG.summary } for ((offset = 0; offset < COUNT; offset++)); do # Stop before the per-test timeout kills us mid-seed; a clean stop, not a failure, so - # log to a file rather than the compared stdout/stderr. BUDGET=0 disables the cap. + # log to a file. BUDGET=0 disables the cap. if [ "$BUDGET" != "0" ] && [ "$SECONDS" -ge "$BUDGET" ]; then echo "fuzz: stopping after $offset/$COUNT seeds; hit FUZZ_TIME_BUDGET=${BUDGET}s" > LOG.budget break @@ -88,10 +82,8 @@ for ((offset = 0; offset < COUNT; offset++)); do if [ "$rc" -eq 0 ]; then record deployed "$seed" - # Optional generator-independent regression corpus: persist configs that actually - # deployed, so runs accumulate known-good inputs that stay valid (and replayable) - # even after the generator changes. Unset by default, so the committed run and its - # empty-output assertion are unaffected. + # Optional corpus: persist configs that deployed, so runs accumulate known-good, + # replayable inputs that survive generator changes. Unset by default. if [ -n "${FUZZ_CORPUS_DIR:-}" ]; then mkdir -p "$FUZZ_CORPUS_DIR" cp "$dir/databricks.yml" "$FUZZ_CORPUS_DIR/${FUZZ_MODE:-generate}-${FUZZ_TARGET:-no_drift}-seed${seed}.yml" @@ -100,7 +92,7 @@ for ((offset = 0; offset < COUNT; offset++)); do fi # timeout exits 124 (137 if the SIGKILL backstop fired): the seed hung. Report a hang, - # distinct from a drift bug; any goroutine dump is preserved in the seed's LOG.*. + # distinct from a drift bug; any goroutine dump is in the seed's LOG.*. if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then record hang "$seed" echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} task test-fuzz" >&2 @@ -126,8 +118,8 @@ for ((offset = 0; offset < COUNT; offset++)); do exit 1 fi - # A 501 "No stub found" is a testserver coverage gap (see IgnoreUnhandledRequests); - # anything else before INPUT_CONFIG_OK is a genuine config rejection. + # A 501 "No stub found" is a testserver coverage gap; anything else before + # INPUT_CONFIG_OK is a genuine config rejection. if grep -qs "No stub found for pattern" "$dir"/LOG.*; then record gap "$seed" else @@ -135,8 +127,7 @@ for ((offset = 0; offset < COUNT; offset++)); do fi done -# Per-variant tally for at-a-glance triage. Reached only on a clean run; a bug/hang exits -# above with the per-seed lines already recorded. +# Per-variant tally for triage. Reached only on a clean run; a bug/hang exits above. if [ -f LOG.summary ]; then # Snapshot the counts before appending the header, else awk would also count it. totals=$(awk '{print $1}' LOG.summary | sort | uniq -c) diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml index 24d27a9915e..f2eec85305b 100644 --- a/acceptance/bundle/invariant/fuzz/test.toml +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -1,8 +1,7 @@ -# Schema fuzzing (see script). Unlike the curated invariant tests, the fuzzer -# generates its own configs, so drop the inherited INPUT_CONFIG matrix. +# The fuzzer generates its own configs, so drop the inherited INPUT_CONFIG matrix. EnvMatrix.INPUT_CONFIG = [] -# Raise the inherited 10m cap so the nightly FUZZ_TIME_BUDGET (script) has room, plus the +# Raise the inherited 10m cap so the nightly FUZZ_TIME_BUDGET (script) has room plus the # final seed's tail. The committed run (5 seeds, no drift) finishes in seconds regardless. Timeout = '20m' @@ -10,15 +9,10 @@ Timeout = '20m' # coverage gap, so return 501 (config rejected) instead of failing the whole run. IgnoreUnhandledRequests = true -# Run the real invariant test script for each target. migrate ignores -# DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. -# -# There is no redeploy target: no_drift's post-deploy plan already dry-runs what a redeploy -# would apply, so it catches the same field-level non-idempotency without the cost of a -# second deploy; the only surface redeploy adds (plan/apply divergence) never surfaced a bug -# across the nightly runs. +# Run each target's real invariant script. No redeploy target: no_drift's post-deploy plan +# already dry-runs a redeploy, and the only surface a real redeploy adds (plan/apply +# divergence) never surfaced a bug in nightly runs. EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate"] -# generate = build a config from the schema (gen_fuzz_config.py); mutate = perturb a -# curated invariant config (mutate_fuzz_config.py). Dispatched by fuzz_gen_config.py. +# generate = build from the schema; mutate = perturb a curated config. See emit_fuzz_config.py. EnvMatrix.FUZZ_MODE = ["generate", "mutate"] diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index bcb9309d894..05f83a7925b 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -3,57 +3,9 @@ unset DATABRICKS_BUNDLE_ENGINE -if [ -n "${FUZZ_SEED:-}" ]; then - fuzz_gen_config.py > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - cp databricks.yml LOG.config -else - # Copy data files to test directory - cp -r "$TESTDIR/../data/." . &> LOG.cp +source "$TESTDIR/../prologue.sh" - # Run init script if present - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" - if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init - fi - - envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - - cp databricks.yml LOG.config -fi - -cleanup() { - # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -# Disable set -e only around deploy so a failed deploy doesn't abort before we capture -# its code: the panic check below must run even on failure (a panicking-but-rejected -# config is a bug, not a rejection), and only then do we exit with the deploy's code. -set +e -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy -deploy_rc=$? -set -e -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null -if [ "$deploy_rc" -ne 0 ]; then - exit "$deploy_rc" -fi -deployed=1 - -echo INPUT_CONFIG_OK +invariant_deploy DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy MIGRATE_ARGS="" # The terraform provider sorts depends_on entries alphabetically by task_key on Read diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 348dde6e981..e27a44b7e5e 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -1,51 +1,15 @@ # Invariant to test: no drift after deploy # Additional checks: no internal errors / panics in validate/plan/deploy -if [ -n "${FUZZ_SEED:-}" ]; then - fuzz_gen_config.py > databricks.yml 2>LOG.gen.err - cat LOG.gen.err | contains.py '!Traceback' > /dev/null - cp databricks.yml LOG.config -else - # Copy data files to test directory - cp -r "$TESTDIR/../data/." . &> LOG.cp - - # Run init script if present - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" - if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init - fi +source "$TESTDIR/../prologue.sh" - envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - - cp databricks.yml LOG.config -fi - -# Fuzz-only: give random configs an isolated validate panic check before deploy. Curated -# configs skip it -- deploy runs the same validate pipeline, so it would only be redundant. -# Output is redirected rather than recorded because a fuzzed config may produce warnings. +# Fuzz-only validate panic check before deploy. Curated configs skip it -- deploy runs the +# same validate pipeline. Output is redirected, not recorded, as a fuzzed config may warn. if [ -n "${FUZZ_SEED:-}" ]; then trace $CLI bundle validate &> LOG.validate cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null fi -cleanup() { - # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. - if [ -z "${deployed:-}" ]; then - return - fi - - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - # Only compute a plan up front when the READPLAN variant actually consumes it via --plan. # Without READPLAN, deploy computes its own plan internally, so a separate `bundle plan` # invocation is pure waste. @@ -54,26 +18,11 @@ if [[ -n "$READPLAN" ]]; then cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi -# Disable set -e only around deploy so a failed deploy doesn't abort before we capture -# its code: the panic check below must run even on failure (a panicking-but-rejected -# config is a bug, not a rejection), and only then do we exit with the deploy's code. -set +e -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -deploy_rc=$? -set -e -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null -if [ "$deploy_rc" -ne 0 ]; then - exit "$deploy_rc" -fi -deployed=1 - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK +invariant_deploy $CLI bundle deploy $(readplanarg plan.json) -# A fuzzed config can deploy yet legitimately differ from the fake server, so the -# fuzzer sets SKIP_DRIFT_CHECK to swap the exact no-drift check for the weaker but -# fidelity-independent plan-determinism oracle; curated configs check full drift. +# A fuzzed config can deploy yet legitimately differ from the fake server, so the fuzzer +# sets SKIP_DRIFT_CHECK to swap the exact no-drift check for the fidelity-independent +# plan-determinism oracle; curated configs check full drift. if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then # JSON plan asserts every action is "skip" -- a strict superset of the text # renderer's "Plan: 0 to add, 0 to change, 0 to delete" summary. @@ -81,16 +30,12 @@ if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null verify_no_drift.py LOG.planjson else - # Fuzz mode: the exact no-drift check false-positives because the fake server does - # not round-trip every field, and "no destructive re-plan" is no better -- for an - # unchanged config any recreate is just a local-vs-remote representation mismatch on - # an immutable field (fake-server fidelity), not real churn. So assert the one - # invariant that is independent of server fidelity: planning is deterministic. Two - # consecutive plans of the same deployed state must be byte-identical; a diff means - # nondeterministic planning/serialization (unstable map order, per-run randomness), a - # real bug. Only compare when both plans succeed: a plan that fails on an unstubbed - # read is a coverage gap, not a bug, and its partial output can differ run-to-run, so - # diffing it would false-positive. Capture each plan's code under set +e for the guard. + # Fuzz mode: the exact no-drift check false-positives (the fake server doesn't round-trip + # every field), so assert the one server-fidelity-independent invariant: planning is + # deterministic. Two consecutive plans of the same state must be byte-identical; a diff + # means nondeterministic planning/serialization, a real bug. Compare only when both + # plans succeed -- a plan that fails on an unstubbed read is a gap and can differ run to + # run. Capture each plan's code under set +e for the guard. set +e $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err plan1_rc=$? @@ -100,6 +45,7 @@ else cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null if [ "$plan1_rc" -eq 0 ] && [ "$plan2_rc" -eq 0 ]; then + # diff exits non-zero on any difference; under set -e that fails the seed as a bug. diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff fi fi diff --git a/acceptance/bundle/invariant/prologue.sh b/acceptance/bundle/invariant/prologue.sh new file mode 100644 index 00000000000..d32ef88a49a --- /dev/null +++ b/acceptance/bundle/invariant/prologue.sh @@ -0,0 +1,58 @@ +# Shared setup for the invariant target scripts (no_drift, migrate), also reached when the +# fuzzer sources them. Renders the config (fuzz-generated when FUZZ_SEED is set, curated +# otherwise), installs the destroy-on-exit trap, and defines invariant_deploy. + +if [ -n "${FUZZ_SEED:-}" ]; then + emit_fuzz_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +# Deploy via the given command (may start with VAR=val prefixes; trace applies them). +# set -e is off only around the deploy so the panic check runs even on failure (a +# panicking-but-rejected config is a bug); a clean non-zero deploy just exits as a rejection. +# On success it marks `deployed` (so cleanup destroys) and prints INPUT_CONFIG_OK, after +# which the fuzzer treats any failure as a bug. +invariant_deploy() { + set +e + trace "$@" &> LOG.deploy + deploy_rc=$? + set -e + cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null + if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" + fi + deployed=1 + echo INPUT_CONFIG_OK +} diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 37386c78e15..27f5fcf353d 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -84,8 +84,7 @@ type TestConfig struct { RecordRequests *bool // Return 501 for a request with no handler instead of failing the test. The fuzzer - // emits resource types the testserver may not model; a missing handler is a gap, not - // a bug. + // emits resource types the testserver may not model; a missing handler is a gap, not a bug. IgnoreUnhandledRequests *bool // List of request headers to include when recording requests. diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 475c92164b9..ff4fddd7744 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -75,8 +75,8 @@ type Server struct { ResponseCallback func(request *Request, response *EncodedResponse) // IgnoreUnhandledRequests returns 501 for a request with no handler instead of failing - // the test: the fuzzer emits resource types the testserver may not model, so the caller - // rejects the config. Curated tests leave it false so real gaps stay loud. + // the test: the fuzzer emits resource types the testserver may not model. Curated tests + // leave it false so real gaps stay loud. IgnoreUnhandledRequests bool } From d134418f888d430c4c1a13915e0a46d4a1a6b258 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 24 Jul 2026 12:41:30 +0000 Subject: [PATCH 53/53] acc/fuzz: fix import order after dispatcher rename The rename to emit_fuzz_config sorts before envsubst, so ruff I001 flagged the import block in mutate_fuzz_config_check.py. --- acceptance/bin/mutate_fuzz_config_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py index 28f815ecf74..a7c03f06886 100755 --- a/acceptance/bin/mutate_fuzz_config_check.py +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -16,8 +16,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from envsubst import substitute_variables from emit_fuzz_config import MUTATE_BASES +from envsubst import substitute_variables from gen_fuzz_config import SKIP_PROPERTY_NAMES from mutate_fuzz_config import load_yaml, mutate, to_yaml