Environments: an inline build context for foreignDockerfile

code_sandboxes/environments/adapters/datalayer.py

@@ -38,6 +38,7 @@
 
 from __future__ import annotations
 
+import base64
 import json
 import os
 import shlex
@@ -210,6 +211,26 @@ def _pip_lock_command(find_links: str, *, authored: bool, foreign: bool = False)
     )
 
 
+def write_build_context(environment: Environment, root: Path) -> None:
+    """Write a `foreignDockerfile`'s inline context where its `COPY`s read it.
+
+    Checked again here, against the folder itself, because this is the step
+    that writes to disk: nothing validated elsewhere is taken on trust.
+    """
+    dockerfile = environment.spec.build.dockerfile
+    for item in (dockerfile.context if dockerfile else None) or []:
+        target = (root / item.path).resolve()
+        if not target.is_relative_to(root.resolve()) or target == root.resolve():
+            raise EnvironmentsError(
+                SPEC_INVALID,
+                f"`{item.path}` is not a path inside the build context",
+                detail={"field": "spec.build.dockerfile.context", "path": item.path},
+            )
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(base64.b64decode(item.content_base64))
+        target.chmod(0o755 if item.executable else 0o644)
+
+
 def _contract_layer(request: BuildRequest) -> list[str]:
     """What makes a foreign image a sandbox, before its lock is installed: the
     contract's files from the approved base it was resolved against, so they
@@ -573,6 +594,7 @@ def build(self, request: BuildRequest) -> ArtifactReference:
             root = Path(directory)
             (root / "Dockerfile").write_text(self.dockerfile(request), encoding="utf-8")
             (root / "lock.txt").write_text(request.lock_text, encoding="utf-8")
+            write_build_context(request.environment, root)
             metadata = root / "metadata.json"
             # A secret's value is never written under `root`: that directory
             # is also `--local context=`, sent to `buildkitd` as ordinary

code_sandboxes/environments/contract.py

@@ -40,6 +40,8 @@
     "MAX_CONTEXT_FILES",
     "MAX_CONTEXT_FILE_BYTES",
     "MAX_CONTEXT_TOTAL_BYTES",
+    "MAX_INLINE_CONTEXT_BYTES",
+    "RESERVED_CONTEXT_PATHS",
     "SANDBOX_CONTRACT_V1",
     "SUPPORTED_CONTRACTS",
     "BuildContextEntry",
@@ -537,8 +539,16 @@ def _approved_froms(text: str, bases: Mapping[str, ApprovedBase]) -> list[_Appro
     return found
 
 
+def _in_context(source: str, context: Sequence[str]) -> bool:
+    """Whether a `COPY` source names a file of the context, or a folder of it."""
+    name = source.strip("/").removeprefix("./")
+    if name in ("", "."):
+        return bool(context)
+    return any(path == name or path.startswith(f"{name}/") for path in context)
+
+
 def dockerfile_findings_for_build(
-    text: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES
+    text: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES, *, context: Sequence[str] = ()
 ) -> list[DockerfileFinding]:
     """What a Dockerfile source must also be to be built, beyond what the contract allows.
 
@@ -598,6 +608,7 @@ def dockerfile_findings_for_build(
             continue
         sources = [t for t in tokens if not t.startswith("--")][:-1]
         local = [s for s in sources if not re.match(r"^(https?|git)://|^git@", s, re.IGNORECASE)]
+        local = [s for s in local if not _in_context(s, context)]
         if local:
             findings.append(
                 DockerfileFinding(
@@ -655,7 +666,7 @@ def _foreign_froms(text: str) -> list[tuple[int, str]]:
 
 
 def foreign_dockerfile_findings(
-    text: str, *, registries: tuple[str, ...]
+    text: str, *, registries: tuple[str, ...], context: Sequence[str] = ()
 ) -> list[DockerfileFinding]:
     """What a Dockerfile on a foreign base must be: the contract's refusals, one
     base, in `registries`, and no build context.
@@ -705,7 +716,7 @@ def foreign_dockerfile_findings(
     findings += _foreign_stage_findings(text)
     findings += [
         finding
-        for finding in dockerfile_findings_for_build(text)
+        for finding in dockerfile_findings_for_build(text, context=context)
         if finding.instruction in ("COPY", "ADD", "escape")
     ]
     return sorted(findings, key=lambda finding: finding.line)
@@ -798,6 +809,11 @@ def pin_dockerfile_base(
 MAX_CONTEXT_FILES = 2000
 MAX_CONTEXT_FILE_BYTES = 50 * 1024 * 1024
 MAX_CONTEXT_TOTAL_BYTES = 100 * 1024 * 1024
+#: A context carried inside the spec, as a `foreignDockerfile` does: it is stored
+#: with the version, so it is small, and its bytes are in the spec's digest.
+MAX_INLINE_CONTEXT_BYTES = 1024 * 1024
+#: What the build's own context already holds at its root.
+RESERVED_CONTEXT_PATHS = ("Dockerfile", "lock.txt", "metadata.json", ".dockerignore")
 
 _CONTEXT_SEPARATOR = re.compile(r"[\\/]")
 

code_sandboxes/environments/spec.py

@@ -22,14 +22,15 @@
 
 from __future__ import annotations
 
+import base64
 import json
 import posixpath
 import re
 from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import Any, Literal
 
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_serializer
 from pydantic.alias_generators import to_camel
 
 from .bases import APPROVED_BASES, ApprovedBase
@@ -352,6 +353,14 @@ class DependencyFileSpec(_Model):
     lock_content: str = ""
 
 
+class ContextFile(_Model):
+    """One file of an inline build context."""
+
+    path: str
+    content_base64: str
+    executable: bool = False
+
+
 class DockerfileSpec(_Model):
     """The Dockerfile a `dockerfile`-sourced version builds from (E3-03).
 
@@ -369,6 +378,17 @@ class DockerfileSpec(_Model):
 
     #: The Dockerfile text.
     content: str = ""
+    #: The files its `COPY`s read, for a `foreignDockerfile`: inline, so the
+    #: spec's digest covers them, and bounded for it.
+    context: list[ContextFile] | None = None
+
+    @model_serializer(mode="wrap")
+    def _no_empty_context(self, handler: Any) -> dict[str, Any]:
+        """Leave `context` out when there is none: a spec's digest does not move."""
+        dumped = handler(self)
+        if self.context is None:
+            dumped.pop("context", None)
+        return dumped
 
 
 class ImageSourceSpec(_Model):
@@ -543,6 +563,38 @@ def parse_requirements_txt(text: str) -> list[str]:
     return lines
 
 
+def _context_findings(context: list[ContextFile]) -> list[SpecFinding]:
+    """An inline context: the upload's own rules, and its bound."""
+    from .contract import (
+        MAX_INLINE_CONTEXT_BYTES,
+        RESERVED_CONTEXT_PATHS,
+        BuildContextEntry,
+        validate_build_context,
+    )
+
+    field = "spec.build.dockerfile.context"
+    findings: list[SpecFinding] = []
+    entries: list[BuildContextEntry] = []
+    for item in context:
+        try:
+            size = len(base64.b64decode(item.content_base64, validate=True))
+        except ValueError:
+            findings.append(SpecFinding(field, f"`{item.path}` is not base64"))
+            continue
+        entries.append(BuildContextEntry(path=item.path, size_bytes=size))
+        if item.path in RESERVED_CONTEXT_PATHS:
+            findings.append(SpecFinding(field, f"`{item.path}` is reserved by the build"))
+    findings += [
+        SpecFinding(field, f"`{found.path}` {found.message}")
+        for found in validate_build_context(entries)
+    ]
+    if sum(entry.size_bytes for entry in entries) > MAX_INLINE_CONTEXT_BYTES:
+        findings.append(
+            SpecFinding(field, f"is over the {MAX_INLINE_CONTEXT_BYTES}-byte inline limit")
+        )
+    return findings
+
+
 def _foreign_dockerfile_findings(
     dockerfile: DockerfileSpec | None, *, registries: tuple[str, ...]
 ) -> list[SpecFinding]:
@@ -555,13 +607,19 @@ def _foreign_dockerfile_findings(
         return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")]
     from .contract import foreign_dockerfile_findings
 
-    return [
+    context = dockerfile.context or []
+    problems = _context_findings(context)
+    return problems + [
         SpecFinding(
             f"{field}.content",
             f"line {finding.line}: {finding.message}",
             finding.code or SPEC_INVALID,
         )
-        for finding in foreign_dockerfile_findings(dockerfile.content, registries=registries)
+        for finding in foreign_dockerfile_findings(
+            dockerfile.content,
+            registries=registries,
+            context=[item.path for item in context],
+        )
     ]
 
 
@@ -579,6 +637,8 @@ def _dockerfile_findings(dockerfile: DockerfileSpec | None) -> list[SpecFinding]
         return [SpecFinding(field, "is required when `spec.build.source` is `dockerfile`")]
     if not dockerfile.content.strip():
         return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")]
+    if dockerfile.context is not None:
+        return [SpecFinding(f"{field}.context", "is taken by a `foreignDockerfile` source")]
     from .contract import dockerfile_findings_for_build, validate_dockerfile
 
     # The contract's refusals, then what building it needs on top (E3-03):

docs/docs/environments/specification.mdx

@@ -42,7 +42,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with
 | `compatibility.variants.required` | `[datalayer]` | At least one of `datalayer`, `e2b`, `daytona`, `modal`. |
 | `compatibility.variants.optional` | none | The same variants, none of them also required. |
 | `compatibility.regions` | none | Region names. |
-| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused.; `foreignDockerfile` (a Dockerfile `FROM` one image in the registries `image` may import from, Datalayer builder only). An `image` or `foreignDockerfile` must be Debian-derived; the build layers `sandbox-contract/v1` on it from an approved base (the contract files, PID 1, the kernel stack and the contract user, installing over what the image holds), and installs Python from its distribution if it has none. |
+| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused.; `foreignDockerfile` (a Dockerfile `FROM` one image in the registries `image` may import from, Datalayer builder only; the files its `COPY`s read go inline in `build.dockerfile.context`, 1 MiB at most, so the spec's digest covers them). An `image` or `foreignDockerfile` must be Debian-derived; the build layers `sandbox-contract/v1` on it from an approved base (the contract files, PID 1, the kernel stack and the contract user, installing over what the image holds), and installs Python from its distribution if it has none. |
 
 ## The spec digest
 

schemas/environment-v1alpha1.json

@@ -203,6 +203,31 @@
       "title": "ContentsBuildEntry",
       "type": "object"
     },
+    "ContextFile": {
+      "additionalProperties": false,
+      "description": "One file of an inline build context.",
+      "properties": {
+        "contentBase64": {
+          "title": "Contentbase64",
+          "type": "string"
+        },
+        "executable": {
+          "default": false,
+          "title": "Executable",
+          "type": "boolean"
+        },
+        "path": {
+          "title": "Path",
+          "type": "string"
+        }
+      },
+      "required": [
+        "path",
+        "contentBase64"
+      ],
+      "title": "ContextFile",
+      "type": "object"
+    },
     "DependencyFileSpec": {
       "additionalProperties": false,
       "description": "A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda\n`environment.yml` (E3-01, E3-02).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one. ``conda`` resolves\nthe ``environment.yml`` in its own ``micromamba`` solve into an explicit\nlock, with the protected constraints merged over its ``pip:`` layer.",
@@ -239,6 +264,21 @@
           "default": "",
           "title": "Content",
           "type": "string"
+        },
+        "context": {
+          "anyOf": [
+            {
+              "items": {
+                "$ref": "#/$defs/ContextFile"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Context"
         }
       },
       "title": "DockerfileSpec",