diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/assets/model.bin b/acceptance/bundle/ai_runtime_task/empty_code_source/assets/model.bin
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/databricks.yml b/acceptance/bundle/ai_runtime_task/empty_code_source/databricks.yml
deleted file mode 100644
index 562924ddc06..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/databricks.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-bundle:
- name: ai-runtime-empty
-
-# An unrelated force-include elsewhere in the bundle. sync.include is added to the
-# file list regardless of the scoped walk, so it must not make an all-filtered code
-# directory look non-empty.
-sync:
- include:
- - assets/*.bin
-
-resources:
- jobs:
- train:
- name: "[${bundle.target}] AI Runtime training"
- tasks:
- - task_key: train
- environment_key: default
- ai_runtime_task:
- experiment: my-training
- code_source_path: ./src
- deployments:
- - command_path: src/command.sh
- compute:
- accelerator_type: GPU_8xH100
- accelerator_count: 8
- environments:
- - environment_key: default
- spec:
- environment_version: "5"
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml
deleted file mode 100644
index e1af1a235ad..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-Cloud = false
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
-EnvMatrix.DMS = ["", "true"]
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/output.txt b/acceptance/bundle/ai_runtime_task/empty_code_source/output.txt
deleted file mode 100644
index bab538a81e0..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/output.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-
->>> [CLI] bundle deploy
-Building air_code_source_src...
-Error: artifact tgz: no files to pack under "[TEST_TMP_DIR]" (empty, gitignored, or no `include` match)
-
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/script b/acceptance/bundle/ai_runtime_task/empty_code_source/script
deleted file mode 100644
index 6c9858c4368..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/script
+++ /dev/null
@@ -1,5 +0,0 @@
-# When code_source_path resolves to a directory whose contents are all filtered out
-# (here: a src/.gitignore of "*"), there is nothing to package. Deploy must fail
-# with an actionable message rather than shipping an empty code archive.
-# A sync.include elsewhere in the bundle must not defeat that guard (see databricks.yml).
-musterr trace $CLI bundle deploy
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/src/.gitignore b/acceptance/bundle/ai_runtime_task/empty_code_source/src/.gitignore
deleted file mode 100644
index 72e8ffc0db8..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/src/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/src/command.sh b/acceptance/bundle/ai_runtime_task/empty_code_source/src/command.sh
deleted file mode 100644
index 7ce3949767c..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/src/command.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-cd $CODE_SOURCE_PATH
-python train.py
diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/src/train.py b/acceptance/bundle/ai_runtime_task/empty_code_source/src/train.py
deleted file mode 100644
index 399ce99aa9f..00000000000
--- a/acceptance/bundle/ai_runtime_task/empty_code_source/src/train.py
+++ /dev/null
@@ -1 +0,0 @@
-print("x")
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/databricks.yml b/acceptance/bundle/ai_runtime_task/local_code_source/databricks.yml
deleted file mode 100644
index 6f91fce9d54..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/databricks.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-bundle:
- name: ai-runtime-test
-
-resources:
- jobs:
- train:
- name: "[${bundle.target}] AI Runtime training"
- tasks:
- # Two AI Runtime tasks with distinct local code dirs: each is turned into its
- # own tgz artifact and uploaded through the standard artifact path.
- - task_key: train
- environment_key: default
- ai_runtime_task:
- experiment: my-training
- code_source_path: ./src
- deployments:
- - command_path: src/command.sh
- compute:
- accelerator_type: GPU_8xH100
- accelerator_count: 8
- - task_key: train2
- environment_key: default
- ai_runtime_task:
- experiment: my-training-2
- code_source_path: ./src2
- deployments:
- - command_path: src2/command.sh
- compute:
- accelerator_type: GPU_8xH100
- accelerator_count: 8
- environments:
- - environment_key: default
- spec:
- environment_version: "5"
- dependencies:
- - torch>=2.0.0
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml
deleted file mode 100644
index e1af1a235ad..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-Cloud = false
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
-EnvMatrix.DMS = ["", "true"]
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt
deleted file mode 100644
index 476d4b1908c..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-
-=== deploy packages the local code sources as tgz artifacts and uploads them
-
->>> [CLI] bundle deploy
-Building air_code_source_src...
-Building air_code_source_src2...
-Uploading .databricks/air_code_source/air_code_source_src.tar.gz...
-Uploading .databricks/air_code_source/air_code_source_src2.tar.gz...
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files...
-Created jobs.train
-Files: 9 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
-
-=== both code_source_paths rewritten to the uploaded artifact remote paths
-
->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task.code_source_path out.requests.txt
-"/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/artifacts/.internal/air_code_source_src.tar.gz"
-"/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/artifacts/.internal/air_code_source_src2.tar.gz"
-
-=== both code tarballs uploaded
-
->>> jq -r .path
-/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/artifacts/.internal/air_code_source_src.tar.gz
-/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/artifacts/.internal/air_code_source_src2.tar.gz
-
-=== destroy removes the deployed bundle
-
->>> [CLI] bundle destroy --auto-approve
-The following resources will be deleted:
- delete resources.jobs.train
-
-All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default
-
-Destroy: 1 deleted
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/script b/acceptance/bundle/ai_runtime_task/local_code_source/script
deleted file mode 100644
index dccc1786add..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/script
+++ /dev/null
@@ -1,18 +0,0 @@
-# Two AI Runtime tasks with distinct local directory code sources. Each directory is
-# turned into a tgz artifact (see bundle/config/mutator/aicode), built and uploaded
-# through the standard artifact path, and its code_source_path is rewritten to the
-# uploaded remote path. Pip deps ride on the job's environments[].spec.dependencies.
-
-title "deploy packages the local code sources as tgz artifacts and uploads them\n"
-trace $CLI bundle deploy
-
-title "both code_source_paths rewritten to the uploaded artifact remote paths\n"
-# Filters use a leading // so Git Bash on Windows does not path-convert them.
-trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task.code_source_path' out.requests.txt
-
-title "both code tarballs uploaded\n"
-trace jq -r .path < out.requests.txt | grep import | grep '.tar.gz' | sort
-
-title "destroy removes the deployed bundle\n"
-trace $CLI bundle destroy --auto-approve
-rm out.requests.txt
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/src/command.sh b/acceptance/bundle/ai_runtime_task/local_code_source/src/command.sh
deleted file mode 100644
index 7ce3949767c..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/src/command.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-cd $CODE_SOURCE_PATH
-python train.py
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/src/train.py b/acceptance/bundle/ai_runtime_task/local_code_source/src/train.py
deleted file mode 100644
index d8b062e5dfe..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/src/train.py
+++ /dev/null
@@ -1 +0,0 @@
-print("training")
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/src2/command.sh b/acceptance/bundle/ai_runtime_task/local_code_source/src2/command.sh
deleted file mode 100644
index 7ce3949767c..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/src2/command.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-cd $CODE_SOURCE_PATH
-python train.py
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/src2/train.py b/acceptance/bundle/ai_runtime_task/local_code_source/src2/train.py
deleted file mode 100644
index 4ff3f662781..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/src2/train.py
+++ /dev/null
@@ -1 +0,0 @@
-print("train2")
diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/test.toml b/acceptance/bundle/ai_runtime_task/local_code_source/test.toml
deleted file mode 100644
index f99765d07e2..00000000000
--- a/acceptance/bundle/ai_runtime_task/local_code_source/test.toml
+++ /dev/null
@@ -1,5 +0,0 @@
-RecordRequests = true
-
-Ignore = [
- '.databricks',
-]
diff --git a/acceptance/experimental/air/convert-to-dabs/output.txt b/acceptance/experimental/air/convert-to-dabs/output.txt
index f7b48740045..fa4413ffc07 100644
--- a/acceptance/experimental/air/convert-to-dabs/output.txt
+++ b/acceptance/experimental/air/convert-to-dabs/output.txt
@@ -19,13 +19,21 @@ persistent job that is not garbage-collected. When you are done, remove the
job and its uploaded files with:
[CLI] bundle destroy
-=== emitted databricks.yml (code_source_path points at ./src; no code is copied)
+=== emitted databricks.yml (code_source packaged as a tgz artifact; no code is copied)
>>> cat databricks.yml
bundle:
name: torchrun-a10-smoke-test
sync:
paths:
- generated_artifacts
+artifacts:
+ code_source:
+ type: tgz
+ path: .
+ include:
+ - src
+ files:
+ - source: ./dist/code_source.tgz
targets:
dev:
mode: development
@@ -45,7 +53,7 @@ resources:
compute:
accelerator_type: GPU_1xA10
accelerator_count: 1
- code_source_path: ./src
+ code_source_path: ./dist/code_source.tgz
environments:
- environment_key: default
spec:
diff --git a/acceptance/experimental/air/convert-to-dabs/script b/acceptance/experimental/air/convert-to-dabs/script
index 77b6e06938a..0998acb0724 100644
--- a/acceptance/experimental/air/convert-to-dabs/script
+++ b/acceptance/experimental/air/convert-to-dabs/script
@@ -1,7 +1,7 @@
title "convert an AIR run YAML into a DABs bundle (in place, next to the source)"
trace $CLI experimental air convert-to-dabs train.yaml
-title "emitted databricks.yml (code_source_path points at ./src; no code is copied)"
+title "emitted databricks.yml (code_source packaged as a tgz artifact; no code is copied)"
trace cat databricks.yml
title "the generated command.sh carries the run command"
diff --git a/bundle/config/mutator/aicode/package_code_source.go b/bundle/config/mutator/aicode/package_code_source.go
deleted file mode 100644
index 57654dfd092..00000000000
--- a/bundle/config/mutator/aicode/package_code_source.go
+++ /dev/null
@@ -1,200 +0,0 @@
-// Package aicode routes an AI Runtime task's local-directory code_source_path through
-// the standard artifact path: it synthesizes a `tgz` artifact that packages the
-// directory and rewrites code_source_path to the tarball the artifact builds. Remote
-// values and local files (a pre-built tarball from an explicit `artifacts` block) are
-// left untouched.
-//
-// It runs before artifacts.Prepare, so the synthesized artifact is prepared, built, and
-// uploaded by the normal artifact pipeline — there is no sync-root overlay. Because it
-// only edits config (no packaging or workspace writes), it is safe in the initialize
-// phase; the tarball itself is produced later by artifacts.Build.
-package aicode
-
-import (
- "context"
- "errors"
- "fmt"
- "io/fs"
- "maps"
- "os"
- "path"
- "path/filepath"
- "strings"
-
- "github.com/databricks/cli/bundle"
- "github.com/databricks/cli/bundle/config"
- "github.com/databricks/cli/bundle/libraries"
- "github.com/databricks/cli/libs/diag"
- "github.com/databricks/cli/libs/dyn"
- "github.com/databricks/cli/libs/log"
-)
-
-// codeSourcePattern is the config location of an AI Runtime task's code_source_path. It
-// matches a direct task only — the same scope aicode.Validate operates on.
-var codeSourcePattern = dyn.NewPattern(
- dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(),
- dyn.Key("tasks"), dyn.AnyIndex(),
- dyn.Key("ai_runtime_task"), dyn.Key("code_source_path"),
-)
-
-// codeArtifactOutputDir is where synthesized tgz artifacts write their tarball. It lives
-// under .databricks (transient, not synced) so the built file is uploaded once via the
-// artifact path and never swept into a sync or into the archive it produces.
-const codeArtifactOutputDir = ".databricks/air_code_source"
-
-// codeSource is a single local code_source_path occurrence to package.
-type codeSource struct {
- configPath dyn.Path
- location dyn.Location
- // value is the raw code_source_path string as written in config.
- value string
-}
-
-func PackageCodeSource() bundle.Mutator {
- return &packageCodeSource{}
-}
-
-type packageCodeSource struct{}
-
-func (m *packageCodeSource) Name() string {
- return "aicode.PackageCodeSource"
-}
-
-func (m *packageCodeSource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
- sources, diags := collectLocalCodeSources(b)
- if diags.HasError() || len(sources) == 0 {
- return diags
- }
-
- // artifacts maps the synthesized artifact name to its spec; outputs maps each
- // code_source_path config location to the tarball it should point at. The
- // code_source_path and the artifact's `files` output share the same local path —
- // that shared path is what links them when libraries upload rewrites both to the
- // same remote location.
- artifacts := make(map[string]*config.Artifact, len(sources))
- outputs := make(map[string]string, len(sources))
- // keyDir records which relDir each artifact key was derived from. artifactKey
- // sanitizes non-alphanumerics to '_', so distinct directories ("a/b" and "a_b")
- // can collide on one key — which would collapse them to a single tarball and make
- // both tasks silently ship the same code. Detect that and error instead.
- keyDir := make(map[string]string, len(sources))
- for _, cs := range sources {
- relDir := strings.TrimPrefix(filepath.ToSlash(cs.value), "./")
- key := artifactKey(relDir)
- if prev, ok := keyDir[key]; ok && prev != relDir {
- return diags.Extend(diag.Errorf("code_source directories %q and %q map to the same artifact name %q; rename one so they differ by more than a non-alphanumeric character", prev, relDir, key))
- }
- keyDir[key] = relDir
- outRel := path.Join(codeArtifactOutputDir, key+".tar.gz")
- // Paths are set absolute: a synthesized artifact carries no config location, so
- // artifacts.Prepare cannot resolve relative paths against the bundle root for it.
- // The runtime extracts to /databricks/code_source/
, so entries must nest
- // under the directory basename — hence path = the directory's parent and
- // include = its basename, so the tgz builder names entries "/...".
- artifacts[key] = &config.Artifact{
- Type: config.ArtifactTarball,
- Path: filepath.Join(b.SyncRootPath, filepath.FromSlash(path.Dir(relDir))),
- Include: []string{path.Base(relDir)},
- Files: []config.ArtifactFile{{Source: filepath.Join(b.SyncRootPath, filepath.FromSlash(outRel))}},
- }
- // code_source_path resolves (via the sync root) to the same absolute output, so
- // libraries upload links the two and rewrites both to the same remote path.
- outputs[cs.configPath.String()] = "./" + outRel
- log.Debugf(ctx, "synthesized tgz artifact %q for code_source_path %q", key, cs.value)
- }
-
- // Rewrite code_source_path first (via the dynamic tree); the typed Artifacts set
- // below then survives to the dynamic tree on mutator exit. Doing it in the other
- // order would let Mutate's ToTyped pass drop the freshly-set artifacts.
- err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
- for _, cs := range sources {
- out := outputs[cs.configPath.String()]
- var err error
- root, err = dyn.SetByPath(root, cs.configPath, dyn.NewValue(out, []dyn.Location{cs.location}))
- if err != nil {
- return root, fmt.Errorf("failed to update code_source_path %q to %q: %w", cs.value, out, err)
- }
- }
- return root, nil
- })
- if err != nil {
- return diags.Extend(diag.FromErr(err))
- }
-
- if b.Config.Artifacts == nil {
- b.Config.Artifacts = make(map[string]*config.Artifact, len(artifacts))
- }
- maps.Copy(b.Config.Artifacts, artifacts)
-
- return diags
-}
-
-// artifactKey is a stable artifact name for a code directory (relative to the
-// bundle). Two tasks pointing at the same directory collapse to one artifact. The
-// sanitization is lossy, so distinct directories can collide on one key; the caller
-// guards against that (see keyDir in Apply).
-func artifactKey(relDir string) string {
- safe := strings.Map(func(r rune) rune {
- if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
- return r
- }
- return '_'
- }, relDir)
- return "air_code_source_" + safe
-}
-
-// collectLocalCodeSources returns every AI Runtime task code_source_path that points at
-// a local directory. Remote values and local files (handled by the artifact path) are
-// skipped.
-func collectLocalCodeSources(b *bundle.Bundle) ([]codeSource, diag.Diagnostics) {
- var sources []codeSource
- var diags diag.Diagnostics
-
- err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
- return dyn.MapByPattern(root, codeSourcePattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
- value, ok := v.AsString()
- if !ok {
- return v, fmt.Errorf("expected string, got %s", v.Kind())
- }
- if !libraries.IsLocalPath(value) {
- return v, nil
- }
- // Only package a local *directory*. A local file (e.g. a pre-built tarball
- // delivered via an `artifacts` block) is left alone so it flows through the
- // standard artifact-upload path as a file.
- localDir := filepath.Join(b.SyncRootPath, filepath.FromSlash(value))
- isDir, err := isExistingDir(localDir)
- if err != nil {
- return v, fmt.Errorf("code_source_path %q: %w", value, err)
- }
- if !isDir {
- return v, nil
- }
- sources = append(sources, codeSource{
- configPath: p,
- location: v.Location(),
- value: value,
- })
- return v, nil
- })
- })
- if err != nil {
- diags = diags.Extend(diag.FromErr(err))
- }
-
- return sources, diags
-}
-
-// isExistingDir reports whether path is an existing directory. A not-exist error is not
-// an error here (the path is simply not a directory this mutator packages), but any
-// other stat failure — notably a permission error on the parent — is surfaced.
-func isExistingDir(path string) (bool, error) {
- info, err := os.Stat(path)
- if err != nil {
- if errors.Is(err, fs.ErrNotExist) {
- return false, nil
- }
- return false, err
- }
- return info.IsDir(), nil
-}
diff --git a/bundle/config/mutator/aicode/package_code_source_test.go b/bundle/config/mutator/aicode/package_code_source_test.go
deleted file mode 100644
index 44c72bcfb3d..00000000000
--- a/bundle/config/mutator/aicode/package_code_source_test.go
+++ /dev/null
@@ -1,145 +0,0 @@
-package aicode
-
-import (
- "os"
- "path/filepath"
- "testing"
-
- "github.com/databricks/cli/bundle"
- "github.com/databricks/cli/bundle/config"
- "github.com/databricks/cli/bundle/config/resources"
- "github.com/databricks/cli/bundle/internal/bundletest"
- "github.com/databricks/cli/libs/dyn"
- "github.com/databricks/databricks-sdk-go/service/iam"
- "github.com/databricks/databricks-sdk-go/service/jobs"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-// bundleWithCodeSource builds a bundle rooted at dir whose single AI Runtime task
-// points at codeSourcePath.
-//
-// The end-to-end build/upload behavior (tarball built by artifacts.Build, uploaded by
-// libraries) runs the full pipeline and is covered by acceptance tests under
-// acceptance/bundle/ai_runtime_task. These unit tests cover the config-only seam:
-// which paths are collected, and the artifact synthesis + code_source_path rewrite.
-func bundleWithCodeSource(t *testing.T, dir, codeSourcePath string) *bundle.Bundle {
- t.Helper()
- b := &bundle.Bundle{
- BundleRootPath: dir,
- SyncRootPath: dir,
- Config: config.Root{
- Bundle: config.Bundle{Target: "default"},
- Workspace: config.Workspace{
- CurrentUser: &config.User{User: &iam.User{UserName: "me@databricks.com"}},
- },
- Resources: config.Resources{
- Jobs: map[string]*resources.Job{
- "train": {
- JobSettings: jobs.JobSettings{
- Tasks: []jobs.Task{
- {
- TaskKey: "train",
- AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: "exp", CodeSourcePath: codeSourcePath},
- },
- },
- },
- },
- },
- },
- },
- }
- bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}})
- return b
-}
-
-// A local-directory code_source_path is turned into a tgz artifact and the field is
-// rewritten to the tarball that artifact builds. path/include are chosen so archive
-// entries nest under the directory basename (the runtime's code_source layout), and
-// the rewritten path equals the artifact's files.source so the upload rail links them.
-func TestPackageCodeSourceSynthesizesArtifact(t *testing.T) {
- dir := t.TempDir()
- require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o700))
- b := bundleWithCodeSource(t, dir, "./src")
-
- diags := PackageCodeSource().Apply(t.Context(), b)
- require.Empty(t, diags)
-
- outRel := ".databricks/air_code_source/air_code_source_src.tar.gz"
- a := b.Config.Artifacts["air_code_source_src"]
- require.NotNil(t, a)
- assert.Equal(t, config.ArtifactTarball, a.Type)
- // path/files are absolute: a synthesized artifact has no location for Prepare to
- // resolve relative paths against.
- assert.Equal(t, dir, a.Path)
- assert.Equal(t, []string{"src"}, a.Include)
- require.Len(t, a.Files, 1)
- assert.Equal(t, filepath.Join(dir, filepath.FromSlash(outRel)), a.Files[0].Source)
-
- // code_source_path is rewritten to the bundle-relative output, which resolves to
- // the same absolute file so the upload rail links them.
- assert.Equal(t, "./"+outRel, b.Config.Resources.Jobs["train"].Tasks[0].AiRuntimeTask.CodeSourcePath)
-}
-
-// Two distinct code_source directories that sanitize to the same artifact key
-// ("a/b" and "a_b" both become air_code_source_a_b) are rejected rather than
-// silently collapsed into one tarball.
-func TestPackageCodeSourceErrorsOnKeyCollision(t *testing.T) {
- dir := t.TempDir()
- require.NoError(t, os.MkdirAll(filepath.Join(dir, "a", "b"), 0o700))
- require.NoError(t, os.MkdirAll(filepath.Join(dir, "a_b"), 0o700))
- b := &bundle.Bundle{
- BundleRootPath: dir,
- SyncRootPath: dir,
- Config: config.Root{
- Bundle: config.Bundle{Target: "default"},
- Resources: config.Resources{
- Jobs: map[string]*resources.Job{
- "train": {JobSettings: jobs.JobSettings{Tasks: []jobs.Task{
- {TaskKey: "t1", AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: "exp", CodeSourcePath: "./a/b"}},
- {TaskKey: "t2", AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: "exp", CodeSourcePath: "./a_b"}},
- }}},
- },
- },
- },
- }
- bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}})
-
- diags := PackageCodeSource().Apply(t.Context(), b)
- require.True(t, diags.HasError())
- assert.ErrorContains(t, diags.Error(), "map to the same artifact name")
-}
-
-func TestCollectLocalCodeSourcesFindsLocalDir(t *testing.T) {
- dir := t.TempDir()
- require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o700))
- b := bundleWithCodeSource(t, dir, "./src")
- sources, diags := collectLocalCodeSources(b)
- require.Empty(t, diags)
- require.Len(t, sources, 1)
- assert.Equal(t, "./src", sources[0].value)
-}
-
-func TestCollectLocalCodeSourcesSkipsRemotePaths(t *testing.T) {
- for _, remote := range []string{
- "/Workspace/Users/me/code.tar.gz",
- "/Volumes/main/default/code/existing.tar.gz",
- } {
- b := bundleWithCodeSource(t, t.TempDir(), remote)
- sources, diags := collectLocalCodeSources(b)
- require.Empty(t, diags)
- assert.Empty(t, sources, "remote code_source_path %q must not be collected", remote)
- }
-}
-
-// A local path that resolves to a file (not a directory) — e.g. a pre-built
-// tarball delivered via an `artifacts` block — is NOT collected: it flows through
-// the standard artifact-upload path as a file rather than being packaged here.
-func TestCollectLocalCodeSourcesSkipsLocalFile(t *testing.T) {
- dir := t.TempDir()
- require.NoError(t, os.WriteFile(filepath.Join(dir, "code.tgz"), []byte("x"), 0o600))
- b := bundleWithCodeSource(t, dir, "code.tgz")
- sources, diags := collectLocalCodeSources(b)
- require.Empty(t, diags)
- assert.Empty(t, sources, "a local tarball file must flow through artifact upload, not aicode packaging")
-}
diff --git a/bundle/config/mutator/aicode/validate.go b/bundle/config/mutator/aicode/validate.go
deleted file mode 100644
index 69e9d6300f2..00000000000
--- a/bundle/config/mutator/aicode/validate.go
+++ /dev/null
@@ -1,141 +0,0 @@
-package aicode
-
-import (
- "context"
- "fmt"
- "path/filepath"
-
- "github.com/databricks/cli/bundle"
- "github.com/databricks/cli/bundle/config"
- "github.com/databricks/cli/bundle/libraries"
- "github.com/databricks/cli/libs/diag"
- "github.com/databricks/cli/libs/dyn"
- "github.com/databricks/databricks-sdk-go/service/jobs"
-)
-
-// Validate checks AI Runtime tasks that reference a local code_source_path so
-// that misconfigurations surface at `bundle validate` time with an actionable
-// message, rather than as an obscure failure mid-deploy. It performs no uploads.
-func Validate() bundle.ReadOnlyMutator {
- return &validate{}
-}
-
-type validate struct{ bundle.RO }
-
-func (v *validate) Name() string {
- return "aicode.Validate"
-}
-
-func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
- var diags diag.Diagnostics
-
- jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
-
- for name, job := range b.Config.Resources.Jobs {
- jobPath := jobsPath.Append(dyn.Key(name))
-
- for i, task := range job.Tasks {
- taskPath := jobPath.Append(dyn.Key("tasks"), dyn.Index(i))
-
- // A local code_source_path under a for_each_task is not packaged by this
- // mutator (aicode collects only direct tasks). Reject it rather than let a
- // nested ai_runtime_task deploy an un-packaged local path.
- if task.ForEachTask != nil && task.ForEachTask.Task.AiRuntimeTask != nil {
- nestedCode := task.ForEachTask.Task.AiRuntimeTask.CodeSourcePath
- if nestedCode != "" && libraries.IsLocalPath(nestedCode) {
- p := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task"),
- dyn.Key("ai_runtime_task"), dyn.Key("code_source_path"))
- diags = diags.Append(diag.Diagnostic{
- Severity: diag.Error,
- Summary: "ai_runtime_task with a local code_source_path is not supported inside a for_each_task",
- Detail: "Set code_source_path to a workspace or volume path",
- Locations: b.Config.GetLocations(p.String()),
- Paths: []dyn.Path{p},
- })
- }
- }
-
- if task.AiRuntimeTask == nil {
- continue
- }
- codePath := taskPath.Append(dyn.Key("ai_runtime_task"), dyn.Key("code_source_path"))
- diags = diags.Extend(v.validateTask(b, job.GitSource, task.AiRuntimeTask.CodeSourcePath, codePath))
- }
- }
-
- return diags
-}
-
-func (v *validate) validateTask(b *bundle.Bundle, gitSource *jobs.GitSource, codeSourcePath string, codePath dyn.Path) diag.Diagnostics {
- // Only local code_source_path values are packaged at deploy; remote values
- // are used as-is and need no validation here.
- if codeSourcePath == "" || !libraries.IsLocalPath(codeSourcePath) {
- return nil
- }
-
- locations := b.Config.GetLocations(codePath.String())
- reject := func(summary, detail string) diag.Diagnostics {
- return diag.Diagnostics{{
- Severity: diag.Error,
- Summary: summary,
- Detail: detail,
- Locations: locations,
- Paths: []dyn.Path{codePath},
- }}
- }
-
- // The packaged directory must live inside the bundle sync root: it is uploaded as
- // part of the bundle, so a path escaping the root (e.g. "../shared") can't be
- // synced. Reject it here with a clear message rather than letting it fail later as
- // an opaque io/fs "invalid argument" when the file list is built.
- if rel, err := filepath.Rel(b.SyncRootPath, filepath.Join(b.SyncRootPath, filepath.FromSlash(codeSourcePath))); err != nil || !filepath.IsLocal(rel) {
- return reject(
- fmt.Sprintf("code_source_path %q is outside the bundle root", codeSourcePath),
- "code_source_path must point at a directory inside the bundle, or at a workspace or volume path",
- )
- }
-
- // This mutator packages a local *directory*. A local path that is not an existing
- // directory is left alone so it flows through the standard artifact-upload path:
- // a pre-built tarball delivered via an `artifacts` block is produced during the
- // build phase (so it does not exist yet at validate time) and is uploaded as a
- // file, not packaged here. Only when the path is an existing directory do the
- // packaging-specific constraints below apply. A stat failure other than not-exist
- // (e.g. unreadable parent) is surfaced rather than silently skipped.
- localDir := filepath.Join(b.SyncRootPath, filepath.FromSlash(codeSourcePath))
- isDir, err := isExistingDir(localDir)
- if err != nil {
- return reject(fmt.Sprintf("failed to inspect code_source_path %q: %v", codeSourcePath, err), "")
- }
- if !isDir {
- return nil
- }
-
- // The deploy engine retrieves task files from git when git_source is set, so
- // packaging a local directory would be silently ignored. Reject the combination.
- if gitSource != nil {
- return reject(
- "ai_runtime_task with a local code_source_path cannot be combined with git_source",
- "Remove git_source or set code_source_path to a workspace or volume path",
- )
- }
-
- // Immutable-folder deployments upload a single content-addressed snapshot and
- // do not support the per-task code packaging this mutator performs.
- if b.IsImmutableFolder() {
- return reject("ai_runtime_task with a local code_source_path is not supported with experimental.immutable_folder", "")
- }
-
- // Source-linked deployment runs jobs against the source files in place and does
- // not copy them to the workspace file path (files.Upload is a no-op). This
- // mutator relies on file sync uploading the packaged snapshot, so the two are
- // incompatible: reject rather than deploy a job pointing at an un-uploaded path.
- if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
- return reject(
- "ai_runtime_task with a local code_source_path is not supported with source-linked deployment",
- "Disable source-linked deployment, or set code_source_path to a workspace or volume path",
- )
- }
-
- return nil
-}
diff --git a/bundle/config/mutator/aicode/validate_test.go b/bundle/config/mutator/aicode/validate_test.go
deleted file mode 100644
index 1839e8c227d..00000000000
--- a/bundle/config/mutator/aicode/validate_test.go
+++ /dev/null
@@ -1,138 +0,0 @@
-package aicode
-
-import (
- "os"
- "path/filepath"
- "testing"
-
- "github.com/databricks/cli/bundle"
- "github.com/databricks/cli/bundle/config"
- "github.com/databricks/cli/bundle/config/resources"
- "github.com/databricks/cli/bundle/internal/bundletest"
- "github.com/databricks/cli/libs/dyn"
- "github.com/databricks/databricks-sdk-go/service/jobs"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func bundleForValidate(t *testing.T, codeSourcePath string, gitSource *jobs.GitSource) *bundle.Bundle {
- t.Helper()
- dir := t.TempDir()
- b := &bundle.Bundle{
- BundleRootPath: dir,
- SyncRootPath: dir,
- Config: config.Root{
- Bundle: config.Bundle{Target: "default"},
- Resources: config.Resources{
- Jobs: map[string]*resources.Job{
- "train": {
- JobSettings: jobs.JobSettings{
- GitSource: gitSource,
- Tasks: []jobs.Task{
- {
- TaskKey: "train",
- AiRuntimeTask: &jobs.AiRuntimeTask{CodeSourcePath: codeSourcePath},
- },
- },
- },
- },
- },
- },
- },
- }
- bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}})
- return b
-}
-
-// mkCodeDir creates a code_source directory (with one file) under the bundle's
-// sync root, so the path resolves to an existing directory this mutator packages.
-func mkCodeDir(t *testing.T, b *bundle.Bundle, rel string) {
- t.Helper()
- dir := filepath.Join(b.SyncRootPath, rel)
- require.NoError(t, os.MkdirAll(dir, 0o700))
- require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte("print()\n"), 0o600))
-}
-
-// A local path that is not an existing directory is left alone: it flows through
-// the standard artifact-upload path (e.g. a pre-built tarball built by an
-// `artifacts` block, which does not exist yet at validate time).
-func TestValidateNonDirectoryCodeSourceIsSkipped(t *testing.T) {
- // Missing path (nothing on disk yet).
- b := bundleForValidate(t, "does-not-exist", nil)
- assert.Empty(t, Validate().Apply(t.Context(), b))
-
- // Existing local file (a pre-built tarball), not a directory.
- b = bundleForValidate(t, "code.tgz", nil)
- require.NoError(t, os.WriteFile(filepath.Join(b.SyncRootPath, "code.tgz"), []byte("x"), 0o600))
- assert.Empty(t, Validate().Apply(t.Context(), b))
-}
-
-func TestValidateGitSourceConflict(t *testing.T) {
- b := bundleForValidate(t, "src", &jobs.GitSource{GitUrl: "https://example.invalid/repo"})
- mkCodeDir(t, b, "src")
- diags := Validate().Apply(t.Context(), b)
- require.Len(t, diags, 1)
- assert.Contains(t, diags[0].Summary, "cannot be combined with git_source")
-}
-
-func TestValidateRemoteCodeSourceIsSkipped(t *testing.T) {
- b := bundleForValidate(t, "/Volumes/main/default/code/x.tar.gz", nil)
- diags := Validate().Apply(t.Context(), b)
- assert.Empty(t, diags)
-}
-
-// A code_source_path escaping the bundle sync root is rejected with a clear
-// message (it can't be synced), rather than failing later as an opaque io/fs error.
-func TestValidateCodeSourceOutsideBundleRoot(t *testing.T) {
- b := bundleForValidate(t, "../shared", nil)
- diags := Validate().Apply(t.Context(), b)
- require.Len(t, diags, 1)
- assert.Contains(t, diags[0].Summary, "outside the bundle root")
-}
-
-// Source-linked deployment doesn't copy files to the workspace file path, so the
-// packaged snapshot would never be uploaded; the combination is rejected.
-func TestValidateSourceLinkedConflict(t *testing.T) {
- b := bundleForValidate(t, "src", nil)
- mkCodeDir(t, b, "src")
- enabled := true
- b.Config.Presets.SourceLinkedDeployment = &enabled
- diags := Validate().Apply(t.Context(), b)
- require.Len(t, diags, 1)
- assert.Contains(t, diags[0].Summary, "source-linked deployment")
-}
-
-// A local code_source_path nested under a for_each_task is not packaged by the
-// mutator, so it is rejected rather than silently skipped.
-func TestValidateForEachTaskCodeSourceRejected(t *testing.T) {
- dir := t.TempDir()
- b := &bundle.Bundle{
- BundleRootPath: dir,
- SyncRootPath: dir,
- Config: config.Root{
- Bundle: config.Bundle{Target: "default"},
- Resources: config.Resources{
- Jobs: map[string]*resources.Job{
- "train": {
- JobSettings: jobs.JobSettings{
- Tasks: []jobs.Task{
- {
- TaskKey: "fanout",
- ForEachTask: &jobs.ForEachTask{
- Task: jobs.Task{
- AiRuntimeTask: &jobs.AiRuntimeTask{CodeSourcePath: "src"},
- },
- },
- },
- },
- },
- },
- },
- },
- },
- }
- bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}})
- diags := Validate().Apply(t.Context(), b)
- require.Len(t, diags, 1)
- assert.Contains(t, diags[0].Summary, "for_each_task")
-}
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index 2942dd4d05e..7c18212846f 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -10,7 +10,6 @@ import (
"github.com/databricks/cli/bundle/artifacts"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
- "github.com/databricks/cli/bundle/config/mutator/aicode"
pythonmutator "github.com/databricks/cli/bundle/config/mutator/python"
"github.com/databricks/cli/bundle/config/validate"
"github.com/databricks/cli/bundle/deploy/metadata"
@@ -203,19 +202,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
mutator.TranslatePaths(),
- // Reads (typed): resources.jobs.*.tasks[*].ai_runtime_task.code_source_path, job git_source
- // Validates that AI Runtime tasks referencing a local code_source_path point at an existing
- // directory and are not combined with git_source or immutable-folder deployments, so these
- // misconfigurations are caught at validate time rather than mid-deploy.
- aicode.Validate(),
-
- // Turn any AI Runtime task code_source_path that points at a local directory
- // into a `tgz` artifact and rewrite the field to the tarball that artifact
- // builds, so the code is packaged and uploaded through the standard artifact
- // path. Runs before artifacts.Prepare so the synthesized artifact is prepared
- // and built like any other. Remote values and local files are left untouched.
- aicode.PackageCodeSource(),
-
// Reads (typed): b.Config.Experimental.PythonWheelWrapper, b.Config.Presets.SourceLinkedDeployment (checks Python wheel wrapper and deployment mode settings)
// Reads (dynamic): resources.jobs.*.tasks (checks for tasks with local libraries and incompatible DBR versions)
// Provides warnings when Python wheel tasks are used with DBR < 13.3 or when wheel wrapper is incompatible with source-linked deployment
diff --git a/experimental/air/cmd/convert_to_dabs.go b/experimental/air/cmd/convert_to_dabs.go
index 461071fb126..06d2e0311ac 100644
--- a/experimental/air/cmd/convert_to_dabs.go
+++ b/experimental/air/cmd/convert_to_dabs.go
@@ -28,16 +28,12 @@ import (
// env_vars.json / secret_env_vars.json / hyperparameters.yaml sidecars into
// generated_artifacts/. It does NOT package, snapshot, or upload anything.
//
-// code_source_path is emitted as the source directory relative to the bundle (the
-// bundle root defaults to the YAML's directory, which contains it). At deploy the
-// aicode mutator (bundle/config/mutator/aicode) packages that directory and uploads
-// it — so convert never touches the code. Dependencies are folded into the job's
-// environments[] spec, which the runtime installs from directly; no requirements.yaml
-// is emitted.
-//
-// When the snapshot pins a git ref or narrows to include_paths, convert instead emits
-// a `tgz` artifact (the DABs artifact snapshotter): DABs builds the tarball from that
-// ref/subset at deploy, and code_source_path points at the built tarball.
+// The code_source is emitted as a `tgz` artifact (the DABs artifact snapshotter):
+// DABs builds the tarball from the source directory at deploy and code_source_path
+// points at the built tarball — so convert never touches the code. A git ref or
+// include_paths just add `git` / a narrowed `include` to the same artifact.
+// Dependencies are folded into the job's environments[] spec, which the runtime
+// installs from directly; no requirements.yaml is emitted.
// dabsTargetName is the single default target emitted; a development-mode target
// is the conventional starting point for a generated bundle.
@@ -92,10 +88,9 @@ does not contact the workspace.`,
}
// Default the bundle to the input YAML's directory. The bundle's sync root
- // must contain the code_source so `code_source_path` resolves within it (the
- // deploy-time aicode mutator packages the source in place), and root_path is
- // resolved relative to the YAML, so the YAML's dir is the natural bundle root.
- // An explicit --output-dir overrides.
+ // must contain the code_source so the `tgz` artifact can package it, and
+ // root_path is resolved relative to the YAML, so the YAML's dir is the natural
+ // bundle root. An explicit --output-dir overrides.
dir := outputDir
if dir == "" {
dir = filepath.Dir(yamlPath)
@@ -118,7 +113,7 @@ does not contact the workspace.`,
// mapping is unit-testable in isolation. Returns the bundle root as a
// map[string]dyn.Value (ready for yamlsaver) and the loose artifacts (command.sh +
// env/secret/param sidecars) to write under generated_artifacts/. It does not touch the
-// code_source; the deploy-time aicode mutator packages it in place.
+// code_source; the emitted `tgz` artifact packages it at deploy.
func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (map[string]dyn.Value, []uploadItem, error) {
// idempotency_token is intentionally not mapped: it dedups a single runs/submit
// call, which has no analogue for a persistent, repeatedly-runnable bundle job.
@@ -140,9 +135,8 @@ func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir st
}
}
- // The source dir relative to the bundle. It becomes code_source_path directly in
- // the plain case, or the `tgz` artifact's `path` when a git ref / include_paths is
- // pinned (see codeArtifactFor).
+ // The source dir relative to the bundle, used as the `tgz` artifact's `path`
+ // (see codeArtifactFor); code_source_path points at the tarball the artifact builds.
codeDirPath, err := bundleCodeSourcePath(ctx, cfg, configPath, bundleDir)
if err != nil {
return nil, nil, err
@@ -189,11 +183,12 @@ type codeArtifact struct {
gitCommit *string
}
-// codeArtifactFor returns the artifact to emit when the snapshot pins a git ref or
-// narrows to include_paths, else nil — the plain directory case, packaged by the
-// deploy-time aicode mutator. It errors when the code dir resolves to the bundle root
-// (no basename to nest under). codeDirPath is the source dir relative to the bundle
-// ("./"-prefixed).
+// codeArtifactFor returns the `tgz` artifact to emit for a code_source, or nil when
+// there is no code_source. Every code_source is packaged as a `tgz` artifact and built
+// and uploaded through the standard artifact path; a git ref / include_paths just add
+// `git` / narrowed `include` to the same artifact. It errors when the code dir resolves
+// to the bundle root (no basename to nest under). codeDirPath is the source dir relative
+// to the bundle ("./"-prefixed).
//
// The artifact snapshotter names entries relative to `path`, and the runtime extracts
// to /databricks/code_source/, so the code dir's basename must be the top-level
@@ -201,7 +196,7 @@ type codeArtifact struct {
// subpaths, so entries come out as "/..." — the layout the air CLI produced.
func codeArtifactFor(cfg *runConfig, codeDirPath string) (*codeArtifact, error) {
snap := codeSnapshot(cfg)
- if snap == nil || (snap.Git == nil && len(snap.IncludePaths) == 0) {
+ if snap == nil {
return nil, nil
}
codeDirRel := strings.TrimPrefix(codeDirPath, "./")
@@ -211,7 +206,7 @@ func codeArtifactFor(cfg *runConfig, codeDirPath string) (*codeArtifact, error)
// generated_artifacts/, the output tarball) into it. Reject rather than emit that;
// the user should point root_path at a subdirectory.
if codeDirRel == "." {
- return nil, fmt.Errorf("code_source root_path %q resolves to the bundle root; convert-to-dabs cannot translate a git or include_paths snapshot there (no code directory to package under %s/). Point root_path at a subdirectory", snap.RootPath, runtimeCodeSourceRoot)
+ return nil, fmt.Errorf("code_source root_path %q resolves to the bundle root; convert-to-dabs cannot package code there (no code directory to nest under %s/). Point root_path at a subdirectory", snap.RootPath, runtimeCodeSourceRoot)
}
dirName := path.Base(codeDirRel)
art := &codeArtifact{
@@ -241,9 +236,9 @@ func codeSnapshot(cfg *runConfig) *snapshotSourceConfig {
}
// bundleCodeSourcePath resolves the code_source directory to a "./"-prefixed path
-// relative to the bundle dir, for emission as ai_runtime_task.code_source_path.
-// Returns "" when the config has no code_source. The path must be inside the bundle
-// (the deploy-time mutator packages it in place and only handles in-bundle dirs).
+// relative to the bundle dir, used as the `tgz` artifact's `path`. Returns "" when the
+// config has no code_source. The path must be inside the bundle (the artifact
+// snapshotter only packages in-bundle dirs).
func bundleCodeSourcePath(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (string, error) {
snap := codeSnapshot(cfg)
if snap == nil {
@@ -306,8 +301,8 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc
}
line := 3
if codeSourcePath != "" {
- // The source dir relative to the bundle; the aicode mutator packages it at
- // deploy and rewrites this field to the uploaded workspace path.
+ // Points at the `tgz` artifact's built tarball; deploy uploads it and rewrites
+ // this to the uploaded workspace path.
aiRuntimeTask["code_source_path"] = nv(codeSourcePath, line)
line++
}
@@ -335,10 +330,10 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc
}
task["ai_runtime_task"] = nv(aiRuntimeTask, taskLine)
- // environments[]: version + the dependency set. The aicode.SynthesizeRequirements
- // mutator regenerates requirements.yaml from this spec at deploy time, so the full
- // dependency set (whether authored inline or in a requirements file) must live
- // here — convert emits no requirements.yaml of its own. Resolve the version
+ // environments[]: version + the dependency set. The runtime installs deps from this
+ // spec directly, so the full dependency set (whether authored inline or in a
+ // requirements file) must live here — convert emits no requirements.yaml of its
+ // own. Resolve the version
// through the same path `air run` uses (config, else env override, else the
// default channel) so a config without an explicit version still pins the version
// the workload would have run with — not an empty spec.
@@ -378,8 +373,8 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc
"name": nv(name, 1),
}, 1),
// sync.paths replaces the default of syncing the whole bundle root. The code
- // directory is omitted deliberately: deploy still packages it into the
- // snapshot tarball, so syncing it too would upload the tree twice.
+ // directory is omitted deliberately: the `tgz` artifact packages it, so syncing
+ // it too would upload the tree twice.
"sync": nv(map[string]dyn.Value{
"paths": nv([]dyn.Value{nv(generatedArtifactsDir, 1)}, 1),
}, 2),
@@ -395,14 +390,14 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc
}, 1),
}, 5),
}
- // The `tgz` artifact snapshotter, when the snapshot pins a git ref / include_paths.
+ // The `tgz` artifact that packages the code_source (nil only when there is none).
if art != nil {
rootValue["artifacts"] = nv(buildArtifactsValue(art), 3)
}
return rootValue
}
-// buildArtifactsValue builds the `artifacts` block for a git/include snapshot: a
+// buildArtifactsValue builds the `artifacts` block for the code_source: a
// single `tgz` artifact whose `path` is the code-source root, carrying the git ref
// and/or include subpaths, and whose `files` output is the tarball code_source_path
// points at.
@@ -437,10 +432,9 @@ func buildArtifactsValue(art *codeArtifact) map[string]dyn.Value {
}
// bundleEnvironmentDeps resolves the runtime version and the inline dependency
-// list to emit in the bundle's environments[] spec. The aicode mutator synthesizes
-// requirements.yaml from that spec at deploy, so the whole set must be here.
-// Dependencies are inline-only (a requirements-file path is rejected at config
-// load), so an unset list yields no dependencies.
+// list to emit in the bundle's environments[] spec. The runtime installs deps from
+// that spec, so the whole set must be here. Dependencies are inline-only (a
+// requirements-file path is rejected at config load), so an unset list yields none.
func bundleEnvironmentDeps(ctx context.Context, cfg *runConfig) (version string, deps []string) {
cfgVersion, _ := cfg.runtimeVersion()
version = dlRuntimeImage(ctx, cfgVersion)
@@ -522,7 +516,7 @@ func buildPermissionsValue(perms []permission) dyn.Value {
// writeBundle writes the bundle into dir: databricks.yml plus the loose launch
// artifacts (command.sh + env/secret/param sidecars). It does not touch the code
-// source — the deploy-time aicode mutator packages it in place. Unless force is set
+// source — the emitted `tgz` artifact packages it at deploy. Unless force is set
// it refuses to overwrite existing files, so a re-run can't silently clobber a
// bundle the user has edited. Returns the relative paths written, for the
// next-steps message.
diff --git a/experimental/air/cmd/convert_to_dabs_test.go b/experimental/air/cmd/convert_to_dabs_test.go
index d169538fb06..0a81ddef844 100644
--- a/experimental/air/cmd/convert_to_dabs_test.go
+++ b/experimental/air/cmd/convert_to_dabs_test.go
@@ -141,9 +141,17 @@ environment:
art := task + ".ai_runtime_task"
assert.Equal(t, name, get(t, root, art+".experiment").MustString())
assert.Equal(t, "run-42", get(t, root, art+".mlflow_run").MustString())
- // code_source_path is the source dir relative to the bundle; the deploy-time
- // aicode mutator packages it in place.
- assert.Equal(t, "./src", get(t, root, art+".code_source_path").MustString())
+ // code_source is packaged as a `tgz` artifact; code_source_path points at the
+ // built tarball, and the artifact packs the code dir (path = its parent, include =
+ // its basename, so entries nest under "src/").
+ assert.Equal(t, "./dist/code_source.tgz", get(t, root, art+".code_source_path").MustString())
+ codeArt := "artifacts." + codeSourceArtifactKey
+ assert.Equal(t, "tgz", get(t, root, codeArt+".type").MustString())
+ assert.Equal(t, ".", get(t, root, codeArt+".path").MustString())
+ codeInc := get(t, root, codeArt+".include").MustSequence()
+ require.Len(t, codeInc, 1)
+ assert.Equal(t, "src", codeInc[0].MustString())
+ assert.Equal(t, "./dist/code_source.tgz", get(t, root, codeArt+".files[0].source").MustString())
dep := art + ".deployments[0]"
assert.Equal(t, "./"+generatedArtifactsDir+"/"+commandScriptName, get(t, root, dep+".command_path").MustString())
@@ -168,8 +176,8 @@ environment:
// Optional fields are omitted rather than emitted empty: no code_source means no
// code_source_path; unset retries/timeout means no wrapper fields.
// sync.paths lists only the generated-artifacts dir. The code directory must be
-// absent: deploy packages it into the snapshot tarball, so syncing it as loose
-// files too would upload the whole tree a second time.
+// absent: the `tgz` artifact packages it, so syncing it as loose files too would
+// upload the whole tree a second time.
func TestConvertToDabsSyncPathsExcludesCodeDir(t *testing.T) {
cfg := minimalConfig + `
code_source:
@@ -189,9 +197,10 @@ code_source:
require.Len(t, paths, 1)
assert.Equal(t, generatedArtifactsDir, paths[0].MustString())
- // The task still points at the code dir; only the sync set omits it.
+ // The task points at the built `tgz` artifact; the code dir is packaged through the
+ // artifact path, so the sync set omits it.
task := "resources.jobs." + loaded.ExperimentName + ".tasks[0].ai_runtime_task"
- assert.Equal(t, "./src", get(t, root, task+".code_source_path").MustString())
+ assert.Equal(t, "./dist/code_source.tgz", get(t, root, task+".code_source_path").MustString())
}
func TestConvertToDabsOmitsUnsetFields(t *testing.T) {
@@ -282,9 +291,9 @@ parameters:
assert.False(t, has(root, art+".secrets"))
}
-// code_source_path is emitted as the source dir relative to the bundle and no code
-// is copied — the deploy-time mutator packages it in place. writeBundle produces
-// only databricks.yml + launch artifacts, not a code_source copy.
+// code_source_path points at the built `tgz` artifact and no code is copied — the
+// artifact snapshotter packages the source dir at deploy. writeBundle produces only
+// databricks.yml + launch artifacts, not a code_source copy.
func TestConvertToDabsDoesNotCopyCode(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o700))
@@ -301,11 +310,11 @@ func TestConvertToDabsDoesNotCopyCode(t *testing.T) {
written, err := writeBundle(t.Context(), loaded, path, dir, false)
require.NoError(t, err)
- // code_source_path points at the existing source dir; nothing is copied.
+ // code_source_path points at the built tarball; nothing is copied.
root, _, err := convertToDabs(t.Context(), loaded, path, dir)
require.NoError(t, err)
art := "resources.jobs." + loaded.ExperimentName + ".tasks[0].ai_runtime_task"
- assert.Equal(t, "./src", get(t, root, art+".code_source_path").MustString())
+ assert.Equal(t, "./dist/code_source.tgz", get(t, root, art+".code_source_path").MustString())
assert.NotContains(t, written, "code_source/")
}
@@ -360,8 +369,8 @@ func TestConvertToDabsGitPinEmitsArtifact(t *testing.T) {
}
// A requirements-FILE dependency set (environment.dependencies is a path) is folded
-// into the environments[] spec so the deploy-time aicode mutator can regenerate
-// requirements.yaml from it. Convert emits no requirements.yaml artifact of its own.
+// into the environments[] spec, which the runtime installs deps from directly.
+// Convert emits no requirements.yaml artifact of its own.
// usage_policy_id is a resolved budget policy id and maps to the job's
// budget_policy_id (usage_policy_name, which needs resolution, is rejected).
func TestConvertToDabsMapsUsagePolicyID(t *testing.T) {