From 1d8380bf536103f7ef55034ea5f8aa05d3fb6675 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Fri, 30 Oct 2026 18:58:40 +0100 Subject: [PATCH] Environments: a foreignDockerfile's build context, inline and bounded, so the spec's digest covers its bytes --- .../environments/adapters/datalayer.py | 22 +++++ code_sandboxes/environments/contract.py | 22 ++++- code_sandboxes/environments/spec.py | 66 ++++++++++++- docs/docs/environments/specification.mdx | 2 +- schemas/environment-v1alpha1.json | 40 ++++++++ ...test_environment_foreign_contract_layer.py | 92 ++++++++++++++++++- 6 files changed, 236 insertions(+), 8 deletions(-) diff --git a/code_sandboxes/environments/adapters/datalayer.py b/code_sandboxes/environments/adapters/datalayer.py index 20498d6..2866377 100644 --- a/code_sandboxes/environments/adapters/datalayer.py +++ b/code_sandboxes/environments/adapters/datalayer.py @@ -38,6 +38,7 @@ from __future__ import annotations +import base64 import json import os import shlex @@ -210,6 +211,26 @@ def _pip_lock_command(find_links: str, *, authored: bool, foreign: bool = False) ) +def write_build_context(environment: Environment, root: Path) -> None: + """Write a `foreignDockerfile`'s inline context where its `COPY`s read it. + + Checked again here, against the folder itself, because this is the step + that writes to disk: nothing validated elsewhere is taken on trust. + """ + dockerfile = environment.spec.build.dockerfile + for item in (dockerfile.context if dockerfile else None) or []: + target = (root / item.path).resolve() + if not target.is_relative_to(root.resolve()) or target == root.resolve(): + raise EnvironmentsError( + SPEC_INVALID, + f"`{item.path}` is not a path inside the build context", + detail={"field": "spec.build.dockerfile.context", "path": item.path}, + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(base64.b64decode(item.content_base64)) + target.chmod(0o755 if item.executable else 0o644) + + def _contract_layer(request: BuildRequest) -> list[str]: """What makes a foreign image a sandbox, before its lock is installed: the contract's files from the approved base it was resolved against, so they @@ -573,6 +594,7 @@ def build(self, request: BuildRequest) -> ArtifactReference: root = Path(directory) (root / "Dockerfile").write_text(self.dockerfile(request), encoding="utf-8") (root / "lock.txt").write_text(request.lock_text, encoding="utf-8") + write_build_context(request.environment, root) metadata = root / "metadata.json" # A secret's value is never written under `root`: that directory # is also `--local context=`, sent to `buildkitd` as ordinary diff --git a/code_sandboxes/environments/contract.py b/code_sandboxes/environments/contract.py index a0eab50..0a0e2ab 100644 --- a/code_sandboxes/environments/contract.py +++ b/code_sandboxes/environments/contract.py @@ -40,6 +40,8 @@ "MAX_CONTEXT_FILES", "MAX_CONTEXT_FILE_BYTES", "MAX_CONTEXT_TOTAL_BYTES", + "MAX_INLINE_CONTEXT_BYTES", + "RESERVED_CONTEXT_PATHS", "SANDBOX_CONTRACT_V1", "SUPPORTED_CONTRACTS", "BuildContextEntry", @@ -537,8 +539,16 @@ def _approved_froms(text: str, bases: Mapping[str, ApprovedBase]) -> list[_Appro return found +def _in_context(source: str, context: Sequence[str]) -> bool: + """Whether a `COPY` source names a file of the context, or a folder of it.""" + name = source.strip("/").removeprefix("./") + if name in ("", "."): + return bool(context) + return any(path == name or path.startswith(f"{name}/") for path in context) + + def dockerfile_findings_for_build( - text: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES + text: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES, *, context: Sequence[str] = () ) -> list[DockerfileFinding]: """What a Dockerfile source must also be to be built, beyond what the contract allows. @@ -598,6 +608,7 @@ def dockerfile_findings_for_build( continue sources = [t for t in tokens if not t.startswith("--")][:-1] local = [s for s in sources if not re.match(r"^(https?|git)://|^git@", s, re.IGNORECASE)] + local = [s for s in local if not _in_context(s, context)] if local: findings.append( DockerfileFinding( @@ -655,7 +666,7 @@ def _foreign_froms(text: str) -> list[tuple[int, str]]: def foreign_dockerfile_findings( - text: str, *, registries: tuple[str, ...] + text: str, *, registries: tuple[str, ...], context: Sequence[str] = () ) -> list[DockerfileFinding]: """What a Dockerfile on a foreign base must be: the contract's refusals, one base, in `registries`, and no build context. @@ -705,7 +716,7 @@ def foreign_dockerfile_findings( findings += _foreign_stage_findings(text) findings += [ finding - for finding in dockerfile_findings_for_build(text) + for finding in dockerfile_findings_for_build(text, context=context) if finding.instruction in ("COPY", "ADD", "escape") ] return sorted(findings, key=lambda finding: finding.line) @@ -798,6 +809,11 @@ def pin_dockerfile_base( MAX_CONTEXT_FILES = 2000 MAX_CONTEXT_FILE_BYTES = 50 * 1024 * 1024 MAX_CONTEXT_TOTAL_BYTES = 100 * 1024 * 1024 +#: A context carried inside the spec, as a `foreignDockerfile` does: it is stored +#: with the version, so it is small, and its bytes are in the spec's digest. +MAX_INLINE_CONTEXT_BYTES = 1024 * 1024 +#: What the build's own context already holds at its root. +RESERVED_CONTEXT_PATHS = ("Dockerfile", "lock.txt", "metadata.json", ".dockerignore") _CONTEXT_SEPARATOR = re.compile(r"[\\/]") diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index 68ca903..e3fdc4e 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -22,6 +22,7 @@ from __future__ import annotations +import base64 import json import posixpath import re @@ -29,7 +30,7 @@ from dataclasses import dataclass from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_serializer from pydantic.alias_generators import to_camel from .bases import APPROVED_BASES, ApprovedBase @@ -352,6 +353,14 @@ class DependencyFileSpec(_Model): lock_content: str = "" +class ContextFile(_Model): + """One file of an inline build context.""" + + path: str + content_base64: str + executable: bool = False + + class DockerfileSpec(_Model): """The Dockerfile a `dockerfile`-sourced version builds from (E3-03). @@ -369,6 +378,17 @@ class DockerfileSpec(_Model): #: The Dockerfile text. content: str = "" + #: The files its `COPY`s read, for a `foreignDockerfile`: inline, so the + #: spec's digest covers them, and bounded for it. + context: list[ContextFile] | None = None + + @model_serializer(mode="wrap") + def _no_empty_context(self, handler: Any) -> dict[str, Any]: + """Leave `context` out when there is none: a spec's digest does not move.""" + dumped = handler(self) + if self.context is None: + dumped.pop("context", None) + return dumped class ImageSourceSpec(_Model): @@ -543,6 +563,38 @@ def parse_requirements_txt(text: str) -> list[str]: return lines +def _context_findings(context: list[ContextFile]) -> list[SpecFinding]: + """An inline context: the upload's own rules, and its bound.""" + from .contract import ( + MAX_INLINE_CONTEXT_BYTES, + RESERVED_CONTEXT_PATHS, + BuildContextEntry, + validate_build_context, + ) + + field = "spec.build.dockerfile.context" + findings: list[SpecFinding] = [] + entries: list[BuildContextEntry] = [] + for item in context: + try: + size = len(base64.b64decode(item.content_base64, validate=True)) + except ValueError: + findings.append(SpecFinding(field, f"`{item.path}` is not base64")) + continue + entries.append(BuildContextEntry(path=item.path, size_bytes=size)) + if item.path in RESERVED_CONTEXT_PATHS: + findings.append(SpecFinding(field, f"`{item.path}` is reserved by the build")) + findings += [ + SpecFinding(field, f"`{found.path}` {found.message}") + for found in validate_build_context(entries) + ] + if sum(entry.size_bytes for entry in entries) > MAX_INLINE_CONTEXT_BYTES: + findings.append( + SpecFinding(field, f"is over the {MAX_INLINE_CONTEXT_BYTES}-byte inline limit") + ) + return findings + + def _foreign_dockerfile_findings( dockerfile: DockerfileSpec | None, *, registries: tuple[str, ...] ) -> list[SpecFinding]: @@ -555,13 +607,19 @@ def _foreign_dockerfile_findings( return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")] from .contract import foreign_dockerfile_findings - return [ + context = dockerfile.context or [] + problems = _context_findings(context) + return problems + [ SpecFinding( f"{field}.content", f"line {finding.line}: {finding.message}", finding.code or SPEC_INVALID, ) - for finding in foreign_dockerfile_findings(dockerfile.content, registries=registries) + for finding in foreign_dockerfile_findings( + dockerfile.content, + registries=registries, + context=[item.path for item in context], + ) ] @@ -579,6 +637,8 @@ def _dockerfile_findings(dockerfile: DockerfileSpec | None) -> list[SpecFinding] return [SpecFinding(field, "is required when `spec.build.source` is `dockerfile`")] if not dockerfile.content.strip(): return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")] + if dockerfile.context is not None: + return [SpecFinding(f"{field}.context", "is taken by a `foreignDockerfile` source")] from .contract import dockerfile_findings_for_build, validate_dockerfile # The contract's refusals, then what building it needs on top (E3-03): diff --git a/docs/docs/environments/specification.mdx b/docs/docs/environments/specification.mdx index 9ef79f7..c83ea84 100644 --- a/docs/docs/environments/specification.mdx +++ b/docs/docs/environments/specification.mdx @@ -42,7 +42,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `compatibility.variants.required` | `[datalayer]` | At least one of `datalayer`, `e2b`, `daytona`, `modal`. | | `compatibility.variants.optional` | none | The same variants, none of them also required. | | `compatibility.regions` | none | Region names. | -| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused.; `foreignDockerfile` (a Dockerfile `FROM` one image in the registries `image` may import from, Datalayer builder only). An `image` or `foreignDockerfile` must be Debian-derived; the build layers `sandbox-contract/v1` on it from an approved base (the contract files, PID 1, the kernel stack and the contract user, installing over what the image holds), and installs Python from its distribution if it has none. | +| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused.; `foreignDockerfile` (a Dockerfile `FROM` one image in the registries `image` may import from, Datalayer builder only; the files its `COPY`s read go inline in `build.dockerfile.context`, 1 MiB at most, so the spec's digest covers them). An `image` or `foreignDockerfile` must be Debian-derived; the build layers `sandbox-contract/v1` on it from an approved base (the contract files, PID 1, the kernel stack and the contract user, installing over what the image holds), and installs Python from its distribution if it has none. | ## The spec digest diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index f823f34..086299d 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -203,6 +203,31 @@ "title": "ContentsBuildEntry", "type": "object" }, + "ContextFile": { + "additionalProperties": false, + "description": "One file of an inline build context.", + "properties": { + "contentBase64": { + "title": "Contentbase64", + "type": "string" + }, + "executable": { + "default": false, + "title": "Executable", + "type": "boolean" + }, + "path": { + "title": "Path", + "type": "string" + } + }, + "required": [ + "path", + "contentBase64" + ], + "title": "ContextFile", + "type": "object" + }, "DependencyFileSpec": { "additionalProperties": false, "description": "A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda\n`environment.yml` (E3-01, E3-02).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one. ``conda`` resolves\nthe ``environment.yml`` in its own ``micromamba`` solve into an explicit\nlock, with the protected constraints merged over its ``pip:`` layer.", @@ -239,6 +264,21 @@ "default": "", "title": "Content", "type": "string" + }, + "context": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/ContextFile" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Context" } }, "title": "DockerfileSpec", diff --git a/tests/test_environment_foreign_contract_layer.py b/tests/test_environment_foreign_contract_layer.py index c063913..a128d9d 100644 --- a/tests/test_environment_foreign_contract_layer.py +++ b/tests/test_environment_foreign_contract_layer.py @@ -16,6 +16,7 @@ # ruff: noqa: S603, S607 from __future__ import annotations +import base64 import json import os import shutil @@ -31,9 +32,11 @@ TASK_WORKDIR_FILE, TINI_SHA256, Builder, + write_build_context, ) from code_sandboxes.environments.builders import BuildRequest from code_sandboxes.environments.contract import ( + MAX_INLINE_CONTEXT_BYTES, foreign_dockerfile_findings, pin_foreign_dockerfile_base, ) @@ -50,7 +53,7 @@ merge_requirements, resolve_environment, ) -from code_sandboxes.environments.spec import parse_environment, spec_findings +from code_sandboxes.environments.spec import parse_environment, spec_digest, spec_findings BASE = "datalayer/python-cpu@sha256:" + "a" * 64 FOREIGN = "docker.io/library/python@sha256:" + "b" * 64 @@ -252,6 +255,83 @@ def test_an_unset_plane_is_no_plane(self): assert plane_of({"PLANE_DOCKERFILES": "/no/such/plane"}) is None +def with_context(content, *files): + spec = dockerfile_source(content) + spec["spec"]["build"]["dockerfile"]["context"] = [ + {"path": path, "contentBase64": base64.b64encode(data).decode(), **extra} + for path, data, extra in files + ] + return spec + + +class TestABuildContext: + """Inline, so the spec's digest covers its bytes, and bounded because of it.""" + + COPIES = "FROM python:3.12\nCOPY app/run.sh /run.sh\n" + + def test_a_copy_of_what_it_holds_is_taken(self): + assert findings(with_context(self.COPIES, ("app/run.sh", b"x", {}))) == [] + + def test_a_copy_of_its_folder_is_taken(self): + content = "FROM python:3.12\nCOPY app/ /app/\n" + assert findings(with_context(content, ("app/run.sh", b"x", {}))) == [] + + def test_a_copy_of_what_it_does_not_hold_is_refused(self): + said = findings(with_context(self.COPIES, ("other.txt", b"x", {}))) + assert "build context" in said[0] + + def test_with_none_a_copy_stays_refused(self): + assert "build context" in findings(dockerfile_source(self.COPIES))[0] + + @pytest.mark.parametrize( + ("path", "said"), + [ + ("../escape", "escapes the context"), + ("/etc/passwd", "absolute path"), + ("Dockerfile", "reserved"), + ("lock.txt", "reserved"), + ], + ) + def test_a_path_that_is_not_a_context_path_is_refused(self, path, said): + spec = with_context("FROM python:3.12\n", (path, b"x", {})) + assert said in " ".join(findings(spec)) + + def test_it_is_bounded(self): + spec = with_context( + "FROM python:3.12\n", ("big", b"x" * (MAX_INLINE_CONTEXT_BYTES + 1), {}) + ) + assert "inline" in " ".join(findings(spec)) + + def test_what_is_not_base64_is_refused(self): + spec = dockerfile_source("FROM python:3.12\n") + spec["spec"]["build"]["dockerfile"]["context"] = [{"path": "a", "contentBase64": "%%"}] + assert "base64" in " ".join(findings(spec)) + + def test_only_a_foreign_dockerfile_takes_one(self): + spec = with_context("FROM datalayer/python-cpu:2026.09\n", ("a", b"x", {})) + spec["spec"]["build"]["source"] = "dockerfile" + assert "foreignDockerfile" in " ".join(findings(spec)) + + def test_its_bytes_are_in_the_digest(self): + one = parse_environment(with_context(self.COPIES, ("app/run.sh", b"one", {}))) + two = parse_environment(with_context(self.COPIES, ("app/run.sh", b"two", {}))) + assert spec_digest(one) != spec_digest(two) + + def test_a_spec_with_none_keeps_the_digest_it_had(self): + parsed = parse_environment(dockerfile_source(self.COPIES)) + dumped = parsed.spec.model_dump(by_alias=True, mode="json") + assert "context" not in dumped["build"]["dockerfile"] + + def test_the_builder_writes_it_beside_the_dockerfile(self, tmp_path): + request = request_of( + with_context(self.COPIES, ("app/run.sh", b"#!/bin/sh\n", {"executable": True})) + ) + write_build_context(request.environment, tmp_path) + written = tmp_path / "app" / "run.sh" + assert written.read_bytes() == b"#!/bin/sh\n" + assert written.stat().st_mode & 0o111 + + class TestTheRegistries: def test_the_list_is_what_it_was(self): assert DEFAULT_ALLOWED_REGISTRIES == ( @@ -367,6 +447,7 @@ def solve_and_build(approved, tmp_path, spec, base): context = tmp_path / "context" context.mkdir() (context / "lock.txt").write_text(locked["content"]) + write_build_context(parse_environment(spec), context) rendered = Builder().dockerfile(request_of(spec, lock=locked["content"], base=base)) (context / "Dockerfile").write_text(rendered) tag = f"foreign-layer:{uuid.uuid4().hex[:8]}" @@ -480,6 +561,15 @@ def test_a_distribution_python_with_no_pip_gets_its_distributions(self, approved ) self._settled(solve_and_build(approved, tmp_path, spec, base)) + def test_a_context_is_copied_into_the_image(self, approved, tmp_path): + base = self._digest("python:3.12-slim") + spec = with_context( + "FROM python:3.12-slim\nCOPY app/ /app/\nRUN /app/run.sh > /built\n", + ("app/run.sh", b"#!/bin/sh\necho from-the-context\n", {"executable": True}), + ) + tag = solve_and_build(approved, tmp_path, spec, base) + assert docker("run", "--rm", tag, "cat", "/built").strip() == "from-the-context" + def test_an_image_with_no_python_gets_its_distributions(self, approved, tmp_path): base = self._digest("ubuntu:24.04") tag = solve_and_build(approved, tmp_path, image_source("ubuntu:24.04"), base)