diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index fcfdf38..f84c53b 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -1 +1 @@ -__version__ = "1.10.0" +__version__ = "1.11.0" diff --git a/code_sandboxes/environments/adapters/datalayer.py b/code_sandboxes/environments/adapters/datalayer.py index 004c8b4..20498d6 100644 --- a/code_sandboxes/environments/adapters/datalayer.py +++ b/code_sandboxes/environments/adapters/datalayer.py @@ -61,7 +61,7 @@ owner_cache_repository, owner_repository, ) -from ..contract import SANDBOX_CONTRACT_V1, pin_dockerfile_base +from ..contract import SANDBOX_CONTRACT_V1, pin_dockerfile_base, pin_foreign_dockerfile_base from ..errors import ( ARTIFACT_MISSING, BUILD_FAILED, @@ -73,14 +73,15 @@ from ..files import files_step from ..resolve import ( WHEELHOUSE_IMAGE_PATH, - WHEELHOUSE_PATH, apt_pins_in, apt_snapshot_in, buildkit_proxy, buildkit_proxy_options, + contract_base_in, egress_findings, egress_hosts_text, egress_refused_hosts, + foreign_bootstrap_lines, locked_versions, ) from ..resolve_conda import ( @@ -90,7 +91,7 @@ micromamba_bootstrap_dockerfile_line, micromamba_install_command, ) -from ..spec import BuildSecret, Environment, command_names_secret +from ..spec import FOREIGN_SOURCES, BuildSecret, Environment, command_names_secret __all__ = [ "ECR_ENVIRONMENT_PREFIX", @@ -131,6 +132,22 @@ #: Raising this is a change to every Datalayer build, so it is one constant #: with one reason, not a literal inside the Dockerfile's own text. DOCKERFILE_FRONTEND = "docker/dockerfile:1.10" +#: PID 1 of a foreign image's sandbox, which reaps and forwards signals as the +#: contract asks. Static, so it runs on any Debian-derived image, and fetched by +#: checksum, so the layer is the same every time. +TINI_URL = "https://github.com/krallin/tini/releases/download/v0.19.0/tini-static-amd64" +TINI_SHA256 = "c5b0666b4cb676901f90dfcb37106783c5fe2077b04590973b885950611b30ee" +#: What the sandbox's server needs of the approved base it is not built on. +CONTRACT_LAYER_FILES = ( + "/opt/datalayer/bin/datalayer-sandbox", + "/opt/datalayer/constraints/", + "/opt/datalayer/wheelhouse/", + "/usr/local/share/jupyter_config.py", + "/usr/local/sbin/start-jupyter.sh", +) +#: Where a foreign image's own working directory is kept: a task's commands run +#: there, while the sandbox's stays the contract's. +TASK_WORKDIR_FILE = "/opt/datalayer/task-workdir" #: The first frontend with `--mount=type=secret,env=` (see above). SECRET_ENV_FRONTEND = (1, 10) @@ -164,27 +181,82 @@ def _head(request: BuildRequest, authored: Any) -> list[str]: f"# Lock: {request.lock_digest}", ] if authored is not None: - lines.append(pin_dockerfile_base(authored.content, request.resolved_base).rstrip("\n")) + pin = ( + pin_foreign_dockerfile_base + if request.environment.spec.build.source == "foreignDockerfile" + else pin_dockerfile_base + ) + lines.append(pin(authored.content, request.resolved_base).rstrip("\n")) else: lines.append(f"FROM {request.resolved_base}") return lines -def _pip_lock_command(find_links: str, *, authored: bool) -> str: - """Install the pip lock: `sync` to it, or `install` it over an authored Dockerfile. +def _pip_lock_command(find_links: str, *, authored: bool, foreign: bool = False) -> str: + """Install the pip lock: `sync` to it, or `install` it over what is already there. `sync` removes whatever the lock does not name, and a `dockerfile` source's lock does not name what the Dockerfile itself installed - (`DOCKERFILE_COVERAGE`). The lock's pins, the protected ones included, - win either way. + (`DOCKERFILE_COVERAGE`), nor does a foreign image's name what the image + holds. The lock's pins, the protected ones included, win either way. A + distribution's Python calls itself externally managed; a foreign image's + is the one the contract's packages go into. """ - verb, target = ("install", "-r ") if authored else ("sync", "") + verb, target = ("install", "-r ") if authored or foreign else ("sync", "") + managed = " --break-system-packages" if foreign else "" return ( - f"uv pip {verb} --system --require-hashes --find-links {find_links} " + f"uv pip {verb} --system{managed} --require-hashes --find-links {find_links} " f"{target}/opt/datalayer/lock.txt" ) +def _contract_layer(request: BuildRequest) -> list[str]: + """What makes a foreign image a sandbox, before its lock is installed: the + contract's files from the approved base it was resolved against, so they + are the channel's own, and PID 1.""" + contract_base = contract_base_in(request.lock_text) + if not contract_base: + raise EnvironmentsError( + SPEC_INVALID, + "this lock names no approved base to layer the contract from: resolve again", + detail={"field": "lock", "source": request.environment.spec.build.source}, + ) + return [ + *foreign_bootstrap_lines(), + *[f"COPY --from={contract_base} {path} {path}" for path in CONTRACT_LAYER_FILES], + f"ADD --checksum=sha256:{TINI_SHA256} {TINI_URL} /opt/datalayer/bin/tini", + "RUN chmod 0755 /opt/datalayer /opt/datalayer/bin /opt/datalayer/bin/tini " + "/opt/datalayer/bin/datalayer-sandbox /opt/datalayer/constraints " + "/opt/datalayer/wheelhouse /usr/local/sbin/start-jupyter.sh", + 'RUN python3 -m pip install --no-cache-dir "uv==0.12.11"', + ] + + +def _contract_user() -> list[str]: + """The contract's user, home and start, after everything is installed. A + uid 1000 the image already has becomes it; what the image kept in its + working directory stays usable by it, but for the reserved tree and what + holds it, which stay root's wherever that directory is.""" + contract = SANDBOX_CONTRACT_V1 + user, uid, gid, home = contract.user, contract.uid, contract.gid, contract.home + reserved, reserved_parent = contract.reserved_path, str(Path(contract.reserved_path).parent) + return [ + f"RUN pwd > {TASK_WORKDIR_FILE} && chmod 0644 {TASK_WORKDIR_FILE} " + f'&& if [ "$(pwd)" != / ]; then find "$(pwd)" -path {reserved} -prune ' + f"-o ! -path {reserved_parent} ! -path / -exec chown -h {uid}:{gid} {{}} +; fi", + f"RUN (getent group {gid} >/dev/null || groupadd -g {gid} users) " + f"&& if getent passwd {uid} >/dev/null; then " + f'usermod -l {user} -g {gid} -d {home} "$(getent passwd {uid} | cut -d: -f1)"; ' + f"else useradd -u {uid} -g {gid} -d {home} -s /bin/bash {user}; fi " + f"&& mkdir -p {home}/.local/share/jupyter/runtime/external_kernels " + f"&& chown -R {uid}:{gid} {home}", + f"ENV HOME={home} LANG={contract.locale} LC_ALL={contract.locale} " + f"LANGUAGE={contract.locale} TZ=Etc/UTC PATH=$PATH:/opt/datalayer/bin", + 'ENTRYPOINT ["/opt/datalayer/bin/tini", "-g", "--"]', + 'CMD ["/usr/local/sbin/start-jupyter.sh"]', + ] + + class Builder: """The Datalayer variant: BuildKit into the owner's ECR repository. @@ -272,7 +344,13 @@ def capabilities(self) -> CapabilitySet: # `image` resolves from an imported reference instead of an # approved base (E3-04). Neither changes how this builder itself # builds, once the resolver has done its part. - build_sources=("packages", "dependencyFile", "image", "dockerfile"), + build_sources=( + "packages", + "dependencyFile", + "image", + "dockerfile", + "foreignDockerfile", + ), package_managers=("uv", "pip"), # Nothing is forbidden by the builder itself: BuildKit is the # reference implementation of a Dockerfile, and what the contract @@ -352,7 +430,8 @@ def dockerfile(self, request: BuildRequest) -> str: """ spec = request.environment.spec apt = apt_pins_in(request.lock_text) - authored = spec.build.dockerfile if spec.build.source == "dockerfile" else None + source = spec.build.source + authored = spec.build.dockerfile if source in ("dockerfile", "foreignDockerfile") else None lines = _head(request, authored) lines += [ f'LABEL io.datalayer.environment="{request.environment.metadata.name}" \\', @@ -383,20 +462,13 @@ def dockerfile(self, request: BuildRequest) -> str: " && rm -rf /var/lib/apt/lists/*", ] ) - # An imported image (E3-04) cannot be assumed to carry `uv` or the - # wheelhouse the way an approved base does (E1-05): both are brought - # to this stage instead, the same as the resolver's own solve - # already does for the same reason. An approved base needs neither, - # so this changes nothing about a build that already works. - imported = spec.build.source == "image" - imported_wheelhouse = f"{WHEELHOUSE_IMAGE_PATH}-import" - find_links = imported_wheelhouse if imported else WHEELHOUSE_IMAGE_PATH + # A foreign image (E3-04) carries none of what an approved base does + # (E1-05): the contract is layered on it, from the approved base it + # was resolved against. An approved base needs none of this. + foreign = source in FOREIGN_SOURCES + find_links = WHEELHOUSE_IMAGE_PATH lines.append("USER root") - if imported: - lines += [ - f"COPY wheelhouse/ {imported_wheelhouse}/", - 'RUN pip install --no-cache-dir "uv==0.12.11"', - ] + lines += _contract_layer(request) if foreign else [] if is_conda_lock(request.lock_text): # A conda source (E3-02): the lock is an `@EXPLICIT` file # `micromamba install --file` installs without re-solving, and the @@ -433,9 +505,10 @@ def dockerfile(self, request: BuildRequest) -> str: # `--find-links` for what no index has — a protected pin's # own wheel, the fork's local version above all (E1-04). "RUN --mount=type=cache,target=/root/.cache/uv " - + _pip_lock_command(find_links, authored=authored is not None), + + _pip_lock_command(find_links, authored=authored is not None, foreign=foreign), ] ) + lines += _contract_user() if foreign else [] # A build secret is mounted on the postInstall commands that name it # and nowhere else (§4.1, D-11): never an `ARG` or `ENV`, which bakes a # value into the image's history, never the package-install or files @@ -500,11 +573,6 @@ def build(self, request: BuildRequest) -> ArtifactReference: root = Path(directory) (root / "Dockerfile").write_text(self.dockerfile(request), encoding="utf-8") (root / "lock.txt").write_text(request.lock_text, encoding="utf-8") - if request.environment.spec.build.source == "image": - wheelhouse = root / "wheelhouse" - wheelhouse.mkdir() - for wheel in WHEELHOUSE_PATH.glob("*.whl"): - (wheelhouse / wheel.name).write_bytes(wheel.read_bytes()) metadata = root / "metadata.json" # A secret's value is never written under `root`: that directory # is also `--local context=`, sent to `buildkitd` as ordinary diff --git a/code_sandboxes/environments/contract.py b/code_sandboxes/environments/contract.py index 5cacd7c..a0eab50 100644 --- a/code_sandboxes/environments/contract.py +++ b/code_sandboxes/environments/contract.py @@ -27,11 +27,13 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path +from typing import Any from pydantic import BaseModel, ConfigDict from .bases import APPROVED_BASES, ApprovedBase, is_approved_repository -from .errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError +from .errors import CAPABILITY_UNSUPPORTED, POLICY_DENIED, SPEC_INVALID, EnvironmentsError +from .image_import import image_registry_allowed, parse_image_reference __all__ = [ "CONTRACT_V1", @@ -52,9 +54,12 @@ "contract_markdown", "dockerfile_base", "dockerfile_findings_for_build", + "foreign_dockerfile_base", + "foreign_dockerfile_findings", "get_contract", "parse_dockerfile", "pin_dockerfile_base", + "pin_foreign_dockerfile_base", "validate_build_context", "validate_dockerfile", ] @@ -263,6 +268,8 @@ class DockerfileFinding: line: int instruction: str message: str + #: Set when this is a policy's refusal rather than a problem with the file. + code: Any = None def to_dict(self) -> dict[str, object]: return {"line": self.line, "instruction": self.instruction, "message": self.message} @@ -333,7 +340,11 @@ def _tokens(arguments: str) -> list[str]: def _check_from( - instruction: DockerfileInstruction, stages: set[str], contract: SandboxContract + instruction: DockerfileInstruction, + stages: set[str], + contract: SandboxContract, + *, + foreign: bool = False, ) -> list[DockerfileFinding]: findings: list[DockerfileFinding] = [] tokens = _tokens(instruction.arguments) @@ -372,7 +383,7 @@ def _check_from( "the approved bases; name the base itself", ) ) - elif not is_approved_repository(image): + elif not foreign and not is_approved_repository(image): findings.append( DockerfileFinding( instruction.line, @@ -394,14 +405,15 @@ def _mount_options(argument: str) -> dict[str, str]: def validate_dockerfile( - text: str, *, contract: SandboxContract = SANDBOX_CONTRACT_V1 + text: str, *, contract: SandboxContract = SANDBOX_CONTRACT_V1, foreign: bool = False ) -> list[DockerfileFinding]: """Everything in a Dockerfile the contract refuses, in line order. Refused: the instructions a provider's builder does not implement, a base that is not approved (or cannot be checked), a platform other than the contract's, privileged builds, the host network, Docker socket mounts - and host bind mounts. + and host bind mounts. With `foreign`, a base is not asked to be approved: + `foreign_dockerfile_findings` holds it to the registries instead. """ findings: list[DockerfileFinding] = [] stages: set[str] = set() @@ -413,7 +425,7 @@ def validate_dockerfile( DockerfileFinding(instruction.line, keyword, f"{keyword} is refused: {reason}") ) if keyword == "FROM": - findings.extend(_check_from(instruction, stages, contract)) + findings.extend(_check_from(instruction, stages, contract, foreign=foreign)) if "docker.sock" in instruction.arguments: findings.append( DockerfileFinding(instruction.line, keyword, "mounts the Docker socket") @@ -627,6 +639,135 @@ def dockerfile_base( ) +def _foreign_froms(text: str) -> list[tuple[int, str]]: + """Every `FROM` that names an image rather than an earlier stage: line, image.""" + stages: set[str] = set() + found: list[tuple[int, str]] = [] + for instruction in parse_dockerfile(text): + if instruction.keyword != "FROM": + continue + image, alias = _from_image(instruction) + if image and image.lower() not in stages: + found.append((instruction.line, image)) + if alias: + stages.add(alias) + return found + + +def foreign_dockerfile_findings( + text: str, *, registries: tuple[str, ...] +) -> list[DockerfileFinding]: + """What a Dockerfile on a foreign base must be: the contract's refusals, one + base, in `registries`, and no build context. + + One base because one digest is pinned and one image is layered on. Its + registry is held to the same list an `image` source is, so this source + reaches nothing an `image` cannot; there is no credential here, so a + private registry is refused. An approved base has its own source, whose + packages the layer would fight. + """ + findings = validate_dockerfile(text, foreign=True) + refused = {finding.line for finding in findings} + froms = _foreign_froms(text) + if not froms: + findings.append(DockerfileFinding(1, "FROM", "names no base image")) + for line, image in froms[1:]: + findings.append( + DockerfileFinding( + line, "FROM", f"`{image}` is a second base: the build pins and layers one" + ) + ) + for line, image in froms[:1]: + if "$" in image or line in refused: + continue + if is_approved_repository(image): + findings.append( + DockerfileFinding( + line, "FROM", f"`{image}` is an approved base: use the `dockerfile` source" + ) + ) + continue + try: + parsed = parse_image_reference(image) + except EnvironmentsError as error: + findings.append(DockerfileFinding(line, "FROM", error.message)) + continue + if not image_registry_allowed(parsed, registries): + findings.append( + DockerfileFinding( + line, + "FROM", + f"`{parsed.registry}` is not an allowed registry; allowed: " + + ", ".join(registries), + POLICY_DENIED, + ) + ) + findings += _foreign_stage_findings(text) + findings += [ + finding + for finding in dockerfile_findings_for_build(text) + if finding.instruction in ("COPY", "ADD", "escape") + ] + return sorted(findings, key=lambda finding: finding.line) + + +def _foreign_stage_findings(text: str) -> list[DockerfileFinding]: + """A `--from=` that names no earlier stage: BuildKit would pull it as an + image, past the one base the registries and the digest hold.""" + findings: list[DockerfileFinding] = [] + stages: set[str] = set() + count = 0 + for instruction in parse_dockerfile(text): + if instruction.keyword == "FROM": + _image, alias = _from_image(instruction) + stages.update({alias} if alias else set()) + count += 1 + continue + named: list[str] = [] + for token in _tokens(instruction.arguments): + if not token.startswith("--"): + break + if instruction.keyword == "COPY" and token.startswith("--from="): + named.append(token.split("=", 1)[1]) + if instruction.keyword == "RUN" and token.startswith("--mount="): + named += [_mount_options(token).get("from", "")] + for source in filter(None, named): + earlier = source.lower() in stages or (source.isdigit() and int(source) < count - 1) + if not earlier: + findings.append( + DockerfileFinding( + instruction.line, + instruction.keyword, + f"`{source}` is not an earlier stage: the one base is the only image " + "a build on a foreign base pulls", + ) + ) + return findings + + +def foreign_dockerfile_base(text: str) -> str: + """The one image a foreign Dockerfile builds on.""" + return _foreign_froms(text)[0][1] + + +def pin_foreign_dockerfile_base(text: str, reference: str) -> str: + """The Dockerfile as it is built: its base pinned to `reference`, directives gone.""" + line, image = _foreign_froms(text)[0] + # As a token, from the `FROM` on: a continuation can carry it to a later line. + token = re.compile(rf"(?= line and not pinned and token.search(raw): + raw, pinned = token.sub(lambda _match: reference, raw, count=1), True + rewritten.append(raw) + return "\n".join(rewritten) + "\n" + + def pin_dockerfile_base( text: str, reference: str, bases: Mapping[str, ApprovedBase] = APPROVED_BASES ) -> str: diff --git a/code_sandboxes/environments/resolve.py b/code_sandboxes/environments/resolve.py index 71002b7..36fae2c 100644 --- a/code_sandboxes/environments/resolve.py +++ b/code_sandboxes/environments/resolve.py @@ -54,6 +54,7 @@ from .bases import APPROVED_BASES, ApprovedBase, channel_snapshot, resolve_base from .build_secrets import resolve_build_secret +from .contract import foreign_dockerfile_base from .errors import ( CAPABILITY_UNSUPPORTED, PACKAGE_NOT_FOUND, @@ -70,9 +71,11 @@ resolve_image_digest, ) from .spec import ( + FOREIGN_SOURCES, BuildSecret, DependencyFileSpec, Environment, + ImageSourceSpec, parse_environment, parse_requirements_txt, ) @@ -138,6 +141,10 @@ APT_PIN_PREFIX = "# datalayer-apt: " #: What a lock says it does not cover, as a header line. COVERAGE_PREFIX = "# datalayer-coverage: " +#: The approved base a foreign image's contract comes from when it names none. +DEFAULT_CONTRACT_BASE = "datalayer/python-cpu" +#: The approved base a foreign image's contract layer is copied from, pinned. +CONTRACT_BASE_PREFIX = "# datalayer-contract-base: " #: A `dockerfile` source's lock (E3-03): what the resolver saw, and no more. DOCKERFILE_COVERAGE = ( "the declared packages and Datalayer's protected pins only; what the Dockerfile's own " @@ -472,6 +479,10 @@ class ResolveOutcome: apt_snapshot: str = "" """The snapshot id they were pinned against, which the builder installs from.""" + python_version: str = "" + """The interpreter the solve found in a foreign image, which the lock is + for; empty where the spec's own version was solved for.""" + class ResolveRunner(Protocol): """Where a solve runs.""" @@ -794,17 +805,24 @@ def dockerfile(self, request: ResolveRequest) -> str: # so this changes nothing about a build that already works. find_links = WHEELHOUSE_IMAGE_PATH lines = [f"FROM {request.base_reference} AS solve", "USER root", "WORKDIR /solve"] + python_version = request.python_version if request.bootstrap_uv: find_links = "/solve/wheelhouse" + # Solved for the interpreter the image has, not the one the spec + # names: the layer installs into that one, and the lock says so. + python_version = '"$(cat /solve/python-version)"' lines += [ + *foreign_bootstrap_lines(), + "RUN python3 -c 'import sys; print(\"%d.%d\" % sys.version_info[:2])' " + "> /solve/python-version", "COPY wheelhouse/ ./wheelhouse/", - 'RUN pip install --no-cache-dir "uv==0.12.11"', + 'RUN python3 -m pip install --no-cache-dir "uv==0.12.11"', ] lines += [ "COPY requirements.in constraints.txt ./", "RUN --mount=type=cache,target=/root/.cache/uv " f"uv pip compile --quiet --no-header --generate-hashes " - f"--python-version {request.python_version} --constraint constraints.txt " + f"--python-version {python_version} --constraint constraints.txt " f"--find-links {find_links} " # This whole line is one shell command once BuildKit runs it: an # index URL is a spec field a user wrote, checked only for @@ -833,6 +851,8 @@ def dockerfile(self, request: ResolveRequest) -> str: "COPY --from=solve /solve/lock.txt /lock.txt", ] ) + if request.bootstrap_uv: + lines.append("COPY --from=solve /solve/python-version /python-version") if request.apt: lines.append("COPY --from=solve /solve/apt.txt /apt.txt") return "\n".join(lines) + "\n" @@ -922,8 +942,14 @@ def solve( if snapshot else "the base channel's mirror" ) + version = out / "python-version" + solved_for = version.read_text(encoding="utf-8").strip() if version.exists() else "" return ResolveOutcome( - lock_text=lock, apt_pins=pins, apt_source=source, apt_snapshot=snapshot + lock_text=lock, + apt_pins=pins, + apt_source=source, + apt_snapshot=snapshot, + python_version=solved_for, ) def _environment(self, request: ResolveRequest) -> dict[str, str] | None: @@ -1041,6 +1067,38 @@ def apt_pins_in(lock_text: str) -> dict[str, str]: return pins +def contract_base_in(lock_text: str) -> str: + """The approved base a foreign image's contract layer comes from, or ``""``.""" + for raw in lock_text.splitlines(): + line = raw.strip() + if line.startswith(CONTRACT_BASE_PREFIX.strip()): + return line[len(CONTRACT_BASE_PREFIX.strip()) :].strip() + return "" + + +def foreign_bootstrap_lines() -> list[str]: + """What a foreign image needs before anything can install: a Debian-derived + system, which the contract asks and which is said here rather than as a + missing `apt-get`, and a Python with pip, from its own distribution when it + has none, so the versions are that image's, and the lock is solved for it. + """ + return [ + "RUN test -f /etc/debian_version || " + '{ echo "this image is not Debian-derived, which sandbox-contract/v1 requires" >&2; ' + "exit 1; }", + # `python3` is what everything after this runs, so pip goes to that one: + # its own `ensurepip`, and only a distribution's Python, which turns + # that off, or none at all, from the distribution. + "RUN python3 -m pip --version >/dev/null 2>&1 || " + "python3 -m ensurepip --default-pip >/dev/null 2>&1 || " + '{ case "$(command -v python3)" in ""|/usr/bin/python3) ' + "apt-get update -qq && apt-get install -y --no-install-recommends " + "python3 python3-pip ca-certificates && rm -rf /var/lib/apt/lists/*;; " + '*) echo "$(command -v python3) has no pip and cannot make one" >&2; exit 1;; esac; }', + "ENV PIP_BREAK_SYSTEM_PACKAGES=1", + ] + + def apt_snapshot_in(lock_text: str) -> str: """The Ubuntu snapshot a lock's apt pins were taken from, or ``""`` (D-9).""" for raw in lock_text.splitlines(): @@ -1058,6 +1116,7 @@ def lock_document( base_reference: str, merged: MergedRequirements, coverage: str | None = None, + contract_base: str = "", ) -> dict[str, Any]: """The stored lock: its text, its digest, and what a reader needs from it. @@ -1091,6 +1150,8 @@ def lock_document( header.append(f"# datalayer-apt-source: {outcome.apt_source}") if outcome.apt_pins and outcome.apt_snapshot: header.append(f"{APT_SNAPSHOT_PREFIX}{outcome.apt_snapshot}") + if contract_base: + header.append(f"{CONTRACT_BASE_PREFIX}{contract_base}") for constraint in merged.constraints: header.append(f"{PROTECTED_PIN_PREFIX}{constraint}") text = "\n".join(header) + "\n" + outcome.lock_text.lstrip("\n") @@ -1136,6 +1197,27 @@ def resolve_bases( return resolved +def contract_base_of( + environment: Environment, bases: dict[str, ApprovedBase], *, registry: str | None +) -> str: + """The approved base a foreign image's contract is layered from, pinned. + + The one its spec names, when that is one; a foreign source need not name + any (`spec.base` means nothing to it), and then the CPU base, its newest + channel. Layering from a base never makes it the image's base: only its + contract files are copied, and the kernel stack is the lock's. + """ + named = environment.spec.base + if named.ref in bases and named.channel in bases[named.ref].channels: + ref, channel = named.ref, named.channel + else: + ref = DEFAULT_CONTRACT_BASE + channel = sorted(bases[ref].channels)[-1] + return resolve_bases( + environment, ["datalayer"], bases, registry=registry, ref=ref, channel=channel + )["datalayer"] + + def resolve_image_base( environment: Environment, variants: Sequence[str], @@ -1165,6 +1247,11 @@ def resolve_image_base( needs neither. """ image = environment.spec.build.image + if environment.spec.build.source == "foreignDockerfile": + dockerfile = environment.spec.build.dockerfile + image = ImageSourceSpec( + reference=foreign_dockerfile_base(dockerfile.content) if dockerfile else "" + ) if image is None or not image.reference.strip(): raise EnvironmentsError( SPEC_INVALID, @@ -1437,6 +1524,7 @@ def resolve_environment( # A Dockerfile names its own base in `FROM` (E3-03): that is what is # pinned and solved in, never `spec.base`, which the schema still asks for. dockerfile = environment.spec.build.dockerfile if source == "dockerfile" else None + foreign = source in FOREIGN_SOURCES base_ref, base_channel = environment.spec.base.ref, environment.spec.base.channel if dockerfile is not None: from .contract import dockerfile_base @@ -1459,7 +1547,7 @@ def resolve_environment( owner_uid=owner_uid, resolve_secret=resolve_secret, ) - if source == "image" + if foreign else resolve_bases( environment, wanted, @@ -1515,17 +1603,22 @@ def resolve_environment( apt=tuple(environment.spec.packages.system.apt), base_reference=solving_in, registry_auth=_registry_auth(credential), - bootstrap_uv=(source == "image"), + bootstrap_uv=foreign, # An imported image is not an approved base, and names no channel. - apt_snapshot=("" if source == "image" else channel_snapshot(base_ref, base_channel, bases)), + apt_snapshot=("" if foreign else channel_snapshot(base_ref, base_channel, bases)), ) outcome = (runner or BuildkitResolveRunner()).solve(request, say) document = lock_document( outcome, - python_version=environment.spec.language.version, + python_version=outcome.python_version or environment.spec.language.version, base_reference=solving_in, merged=merged, - coverage=DOCKERFILE_COVERAGE if dockerfile is not None else None, + coverage=DOCKERFILE_COVERAGE if dockerfile is not None or foreign else None, + contract_base=( + contract_base_of(environment, bases, registry=_registry_of(credential)) + if foreign and "datalayer" in wanted + else "" + ), ) say( f"Locked {document['package_count']} packages" diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index 375de5a..68ca903 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -106,9 +106,20 @@ SIZE_CLASSES: tuple[str, ...] = ("small", "medium", "large", "gpu-small", "gpu-large") GPU_SIZE_CLASSES: tuple[str, ...] = ("gpu-small", "gpu-large") -BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") -#: What builds today; `dockerfile` is the one source still to come. -SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") +BUILD_SOURCES: tuple[str, ...] = ( + "packages", + "dependencyFile", + "dockerfile", + "image", + "foreignDockerfile", +) +#: What builds today. +SUPPORTED_BUILD_SOURCES: tuple[str, ...] = BUILD_SOURCES +#: The sources that start from an image Datalayer did not approve, so the +#: pipeline layers the contract on top of it at build. +FOREIGN_SOURCES: tuple[str, ...] = ("image", "foreignDockerfile") +#: The sources whose base is not `spec.base`. +FOREIGN_OR_AUTHORED_SOURCES: tuple[str, ...] = (*FOREIGN_SOURCES, "dockerfile") SUPPORTED_PACKAGE_MANAGERS: tuple[str, ...] = ("uv", "pip") #: `requirements.txt` and `pyproject.toml`/`uv.lock` are archived on the #: version they resolved (E3-01); this bounds what a spec may carry inline, @@ -384,7 +395,12 @@ class ImageSourceSpec(_Model): class BuildSpec(_Model): - source: Literal["packages", "dependencyFile", "dockerfile", "image"] = "packages" + #: `foreignDockerfile` is a Dockerfile `FROM` an image Datalayer did not + #: approve, in the registries an `image` source may import from; like an + #: `image`, it gets the contract layered on top at build. + source: Literal["packages", "dependencyFile", "dockerfile", "image", "foreignDockerfile"] = ( + "packages" + ) dependency_file: DependencyFileSpec | None = None dockerfile: DockerfileSpec | None = None image: ImageSourceSpec | None = None @@ -527,6 +543,28 @@ def parse_requirements_txt(text: str) -> list[str]: return lines +def _foreign_dockerfile_findings( + dockerfile: DockerfileSpec | None, *, registries: tuple[str, ...] +) -> list[SpecFinding]: + """A Dockerfile on a foreign base: what the contract refuses, its one base + held to the list an `image` source is, and no build context.""" + field = "spec.build.dockerfile" + if dockerfile is None: + return [SpecFinding(field, "is required when `spec.build.source` is `foreignDockerfile`")] + if not dockerfile.content.strip(): + return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")] + from .contract import foreign_dockerfile_findings + + return [ + SpecFinding( + f"{field}.content", + f"line {finding.line}: {finding.message}", + finding.code or SPEC_INVALID, + ) + for finding in foreign_dockerfile_findings(dockerfile.content, registries=registries) + ] + + def _dockerfile_findings(dockerfile: DockerfileSpec | None) -> list[SpecFinding]: """What a `dockerfile`-sourced version must carry, and what the contract refuses. @@ -682,7 +720,7 @@ def _organization_findings( findings: list[SpecFinding] = [] spec = environment.spec if ( - spec.build.source not in ("image", "dockerfile") + spec.build.source not in FOREIGN_OR_AUTHORED_SOURCES and policy.allowed_bases is not None and not policy.allows("bases", spec.base.ref) ): @@ -764,7 +802,7 @@ def spec_findings( # validated against the approved bases by `check_dockerfile`, not here): # `spec.base` names nothing Datalayer approved for either, so checking it # against the table would refuse every one for the reason they exist. - if spec.build.source not in ("image", "dockerfile"): + if spec.build.source not in FOREIGN_OR_AUTHORED_SOURCES: base = bases.get(spec.base.ref) if base is None: findings.append( @@ -796,13 +834,18 @@ def spec_findings( findings.extend(_dependency_file_findings(spec.build.dependency_file)) elif spec.build.source == "dockerfile": findings.extend(_dockerfile_findings(spec.build.dockerfile)) - elif spec.build.source == "image": + elif spec.build.source in FOREIGN_SOURCES: registries = ( DEFAULT_ALLOWED_REGISTRIES + policy.allowed_registries if policy.allowed_registries else DEFAULT_ALLOWED_REGISTRIES ) - findings.extend(_image_findings(spec.build.image, registries=registries)) + if spec.build.source == "image": + findings.extend(_image_findings(spec.build.image, registries=registries)) + else: + findings.extend( + _foreign_dockerfile_findings(spec.build.dockerfile, registries=registries) + ) python = spec.packages.python if python.manager not in SUPPORTED_PACKAGE_MANAGERS: diff --git a/docs/docs/environments/specification.mdx b/docs/docs/environments/specification.mdx index 410cd35..9ef79f7 100644 --- a/docs/docs/environments/specification.mdx +++ b/docs/docs/environments/specification.mdx @@ -42,7 +42,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `compatibility.variants.required` | `[datalayer]` | At least one of `datalayer`, `e2b`, `daytona`, `modal`. | | `compatibility.variants.optional` | none | The same variants, none of them also required. | | `compatibility.regions` | none | Region names. | -| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused. | +| `build.source` | `packages` | `packages`; `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer); `dockerfile` (`build.dockerfile.content`, inline: its `FROM` names an approved base with its channel as the tag, which is what resolves, and what its own instructions install is not locked, as the lock's header says); and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base). `packages`, a `requirements.txt`, a `pyproject.toml` with its `uv.lock`, a conda `environment.yml` and `dockerfile` build on the Datalayer and Daytona variants. A `pyproject.toml`'s `uv.lock` has to lock Datalayer's `jupyter-server` fork, which no package index serves: the project takes it by URL from [the fork's release](https://github.com/datalayer-externals/jupyter-server/releases/tag/v2.21.0-datalayer.1), in `[tool.uv.sources]`, which is also what makes `uv` record its hash. `image` resolves but does not yet start as a sandbox, because the imported image has no Datalayer runtime layer; `image` from a private registry, and a `COPY` from a Dockerfile's build context, are refused.; `foreignDockerfile` (a Dockerfile `FROM` one image in the registries `image` may import from, Datalayer builder only). An `image` or `foreignDockerfile` must be Debian-derived; the build layers `sandbox-contract/v1` on it from an approved base (the contract files, PID 1, the kernel stack and the contract user, installing over what the image holds), and installs Python from its distribution if it has none. | ## The spec digest diff --git a/pyproject.toml b/pyproject.toml index 22053fb..c0ca40f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ Home = "https://github.com/datalayer/code-sandboxes" path = "code_sandboxes/__version__.py" [tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" markers = [ "live: runs against the real sandbox providers; needs CODE_SANDBOXES_LIVE=1 and credentials", diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index 4b7040a..f823f34 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -123,7 +123,8 @@ "packages", "dependencyFile", "dockerfile", - "image" + "image", + "foreignDockerfile" ], "title": "Source", "type": "string" diff --git a/tests/test_environment_datalayer_builder.py b/tests/test_environment_datalayer_builder.py index 11b40b1..3074b71 100644 --- a/tests/test_environment_datalayer_builder.py +++ b/tests/test_environment_datalayer_builder.py @@ -170,6 +170,9 @@ def a_request(**changes) -> BuildRequest: return BuildRequest(**fields) +CONTRACT_BASE = "datalayer/python-cpu@sha256:" + "a" * 64 + + def a_builder(**changes) -> Builder: options = { "credential": Credential(), @@ -384,12 +387,10 @@ def test_an_undeclared_build_secret_id_mounts_nothing(self) -> None: dockerfile = a_builder().dockerfile(request) assert "--mount=type=secret" not in dockerfile - def test_an_imported_image_bootstraps_uv_from_its_own_wheelhouse(self) -> None: + def test_an_imported_image_bootstraps_uv_from_the_approved_bases_wheelhouse(self) -> None: """An imported image (E3-04) is not baked with `uv` or the fork's - wheel the way an approved base is (E1-05): both are brought to this - stage too, the same as the resolver's own solve. Found on PR #27's - Copilot review — the resolve half of this had already been fixed, - the actual build's Dockerfile had not.""" + wheel the way an approved base is (E1-05): the wheel comes with the + contract layer, and `uv` is installed for it.""" request = a_request( spec={ "packages": {}, @@ -399,12 +400,13 @@ def test_an_imported_image_bootstraps_uv_from_its_own_wheelhouse(self) -> None: }, } ) + request = request.model_copy( + update={"lock_text": f"# datalayer-contract-base: {CONTRACT_BASE}\n{request.lock_text}"} + ) dockerfile = a_builder().dockerfile(request) - assert "COPY wheelhouse/ /opt/datalayer/wheelhouse-import/" in dockerfile - assert 'RUN pip install --no-cache-dir "uv==0.12.11"' in dockerfile - assert "--find-links /opt/datalayer/wheelhouse-import" in dockerfile - # Never the approved base's own path: that one is not copied in here. - assert "--find-links /opt/datalayer/wheelhouse " not in dockerfile + assert f"COPY --from={CONTRACT_BASE} /opt/datalayer/wheelhouse/" in dockerfile + assert 'RUN python3 -m pip install --no-cache-dir "uv==0.12.11"' in dockerfile + assert "--find-links /opt/datalayer/wheelhouse " in dockerfile def test_a_packages_source_never_copies_a_wheelhouse_in(self) -> None: dockerfile = a_builder().dockerfile(a_request()) @@ -473,7 +475,13 @@ def test_the_section_4_1_example_is_supported(self) -> None: def test_this_variant_has_no_gpu_yet(self) -> None: capabilities = a_builder().capabilities() assert capabilities.supports_gpu is False - assert capabilities.build_sources == ("packages", "dependencyFile", "image", "dockerfile") + assert capabilities.build_sources == ( + "packages", + "dependencyFile", + "image", + "dockerfile", + "foreignDockerfile", + ) assert capabilities.package_managers == ("uv", "pip") @@ -500,35 +508,6 @@ def test_it_pushes_by_digest_with_its_attestations(self) -> None: assert artifact.size_class == "medium" assert artifact.contract_version == "sandbox-contract/v1" - def test_an_imported_image_build_copies_the_wheelhouse_into_the_context(self) -> None: - from code_sandboxes.environments.resolve import WHEELHOUSE_PATH - - expected = {wheel.name for wheel in WHEELHOUSE_PATH.glob("*.whl")} - assert expected, "the package's own wheelhouse must not be empty" - seen: dict[str, set[str]] = {} - - class RecordingBuildctl(Buildctl): - def __call__(self, argv, **kwargs): - result = super().__call__(argv, **kwargs) - assert self.context is not None - seen["wheelhouse"] = { - path.name for path in (self.context / "wheelhouse").glob("*.whl") - } - return result - - request = a_request( - spec={ - "packages": {}, - "build": { - "source": "image", - "image": {"reference": "python:3.12-slim-bookworm"}, - }, - } - ) - buildctl = RecordingBuildctl() - a_builder(run=buildctl).build(request) - assert seen["wheelhouse"] == expected - def test_a_packages_source_build_has_no_wheelhouse_in_its_context(self) -> None: seen: dict[str, bool] = {} diff --git a/tests/test_environment_foreign_contract_layer.py b/tests/test_environment_foreign_contract_layer.py new file mode 100644 index 0000000..c063913 --- /dev/null +++ b/tests/test_environment_foreign_contract_layer.py @@ -0,0 +1,491 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""A foreign image made a sandbox: `sandbox-contract/v1` layered on at build. + +What is rendered and refused, always. With `DATALAYER_ENVIRONMENT_DOCKER_TESTS` +set, the real thing: the resolver's and the builder's own Dockerfiles built by +Docker on real public images, the doctor run in the result, and a real Jupyter +server started by the image's own entrypoint, whose kernel runs a task's code. +A local image stands where the approved base does, since it is what the layer +copies from: built here from this package's doctor, constraints and wheelhouse, +and the plane's start script and config, which `PLANE_DOCKERFILES` names. +""" + +# ruff: noqa: S603, S607 +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import uuid +from pathlib import Path + +import httpx +import pytest + +from code_sandboxes.environments.adapters.datalayer import ( + CONTRACT_LAYER_FILES, + TASK_WORKDIR_FILE, + TINI_SHA256, + Builder, +) +from code_sandboxes.environments.builders import BuildRequest +from code_sandboxes.environments.contract import ( + foreign_dockerfile_findings, + pin_foreign_dockerfile_base, +) +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.image_import import DEFAULT_ALLOWED_REGISTRIES +from code_sandboxes.environments.resolve import ( + WHEELHOUSE_PATH, + BuildkitResolveRunner, + ResolveOutcome, + ResolveRequest, + contract_base_in, + foreign_bootstrap_lines, + lock_document, + merge_requirements, + resolve_environment, +) +from code_sandboxes.environments.spec import parse_environment, spec_findings + +BASE = "datalayer/python-cpu@sha256:" + "a" * 64 +FOREIGN = "docker.io/library/python@sha256:" + "b" * 64 +LOCK = f"# datalayer-contract-base: {BASE}\nipykernel==7.3.0 --hash=sha256:{'c' * 64}\n" +ENVIRONMENTS = Path(__file__).parents[1] / "code_sandboxes" / "environments" + + +def plane_of(environ): + """The plane's dockerfiles, when the variable names a directory that exists.""" + named = environ.get("PLANE_DOCKERFILES", "") + return Path(named) if named and Path(named).is_dir() else None + + +PLANE = plane_of(os.environ) +REAL = bool(os.environ.get("DATALAYER_ENVIRONMENT_DOCKER_TESTS")) + + +_TRANSPORT = httpx.MockTransport( + lambda request: httpx.Response( + 200, json={"token": "t"}, headers={"Docker-Content-Digest": "sha256:" + "b" * 64} + ) +) + + +def document(build): + return { + "apiVersion": "environments.datalayer.io/v1alpha1", + "kind": "Environment", + "metadata": {"name": "task-one"}, + "spec": { + "language": {"name": "python", "version": "3.12"}, + "base": {"ref": "datalayer/python-cpu", "channel": "2026.09"}, + "build": build, + }, + } + + +def image_source(reference): + return document({"source": "image", "image": {"reference": reference}}) + + +def dockerfile_source(content): + return document({"source": "foreignDockerfile", "dockerfile": {"content": content}}) + + +def request_of(spec, *, lock=LOCK, base=FOREIGN): + return BuildRequest( + environment_uid="e1", + version=1, + build_uid="b1", + owner_uid="o1", + variant="datalayer", + environment=parse_environment(spec), + lock_text=lock, + lock_digest="sha256:" + "d" * 64, + resolved_base=base, + ) + + +def findings(spec): + return [item.message for item in spec_findings(parse_environment(spec))] + + +class TestTheLayer: + def rendered(self, spec=None): + return Builder().dockerfile(request_of(spec or image_source("python:3.12-slim"))) + + def test_it_copies_the_contract_from_the_approved_base_it_was_resolved_against(self): + text = self.rendered() + for path in CONTRACT_LAYER_FILES: + assert f"COPY --from={BASE} {path} {path}" in text + + def test_it_installs_over_the_image_and_removes_nothing(self): + text = self.rendered() + assert "uv pip install --system --break-system-packages --require-hashes" in text + assert "uv pip sync" not in text + + def test_pid_1_is_fetched_by_its_checksum(self): + assert f"ADD --checksum=sha256:{TINI_SHA256} " in self.rendered() + + def test_it_ends_as_the_contracts_user_and_is_checked_by_the_doctor(self): + text = self.rendered() + assert text.index("usermod") < text.rindex("USER 1000:100") + assert text.rstrip().endswith("doctor --json > /tmp/doctor.json") + assert 'ENTRYPOINT ["/opt/datalayer/bin/tini", "-g", "--"]' in text + + def test_the_tasks_working_directory_is_kept(self): + assert f"pwd > {TASK_WORKDIR_FILE}" in self.rendered() + + def test_a_dockerfile_keeps_what_it_says_and_gets_the_same_layer(self): + spec = dockerfile_source("FROM python:3.12-slim AS task\nRUN echo hello > /hello\n") + text = self.rendered(spec) + assert f"FROM {FOREIGN} AS task" in text + assert text.index("RUN echo hello") < text.index("COPY --from=" + BASE) + + def test_a_lock_with_no_contract_base_is_resolved_again_not_guessed(self): + with pytest.raises(EnvironmentsError, match="resolve again"): + Builder().dockerfile(request_of(image_source("python:3.12-slim"), lock="a==1\n")) + + def test_the_lock_carries_the_base(self): + locked = lock_document( + ResolveOutcome(lock_text="a==1\n"), + python_version="3.12", + base_reference=FOREIGN, + merged=merge_requirements([]), + contract_base=BASE, + ) + assert contract_base_in(locked["content"]) == BASE + + +class TestTheReviewsFindings: + """Six things a reader found, each pinned here.""" + + @pytest.mark.parametrize( + "content", + [ + "FROM python:3.12\nCOPY --from=registry.example.com/team/tool:latest /tool /tool\n", + "FROM python:3.12\nRUN --mount=type=bind,from=registry.example.com/t:1,target=/t a\n", + "FROM python:3.12\nCOPY --from=later /a /a\nFROM python:3.12 AS later\n", + ], + ) + def test_nothing_is_pulled_past_the_base(self, content): + said = " ".join(item.message for item in _foreign(content)) + assert "earlier stage" in said + + @pytest.mark.parametrize( + "content", + [ + "FROM a:1 AS tools\nRUN true\nFROM tools\nCOPY --from=tools /a /a\n", + "FROM a:1 AS tools\nFROM tools\nCOPY --from=0 /a /a\n", + "FROM a:1 AS tools\nFROM tools\nRUN --mount=type=bind,from=tools,target=/t true\n", + "FROM a:1\nRUN --mount=type=cache,target=/c true\n", + ], + ) + def test_an_earlier_stage_is_fine(self, content): + assert _foreign(content) == [] + + @pytest.mark.parametrize( + "content", + [ + "FROM \\\n python:3.12-slim\nRUN true\n", + "FROM --platform=linux/amd64 \\\n python:3.12-slim \\\n AS task\nRUN true\n", + ], + ) + def test_a_base_on_a_continuation_line_is_pinned(self, content): + text = Builder().dockerfile(request_of(dockerfile_source(content))) + assert FOREIGN in text + assert "python:3.12-slim" not in text + + def test_the_base_is_replaced_as_a_token(self): + content = "FROM --platform=linux/amd64 a:1\nRUN echo a:1\n" + pinned = pin_foreign_dockerfile_base(content, FOREIGN) + assert f"FROM --platform=linux/amd64 {FOREIGN}" in pinned + assert "RUN echo a:1" in pinned + + def test_the_reserved_tree_and_what_holds_it_stay_roots(self): + text = Builder().dockerfile(request_of(image_source("python:3.12-slim"))) + assert "-path /opt/datalayer -prune" in text + assert "! -path /opt" in text + assert 'chown -R 1000:100 "$(pwd)"' not in text + + def test_the_lock_says_the_python_it_was_solved_for(self): + class Solved: + name = "fake" + + def solve(self, request, log=None): + return ResolveOutcome(lock_text="ipykernel==7.3.0\n", python_version="3.12") + + spec = image_source("python:3.12-slim") + spec["spec"]["language"]["version"] = "3.13" + locked = resolve_environment( + spec=spec, variants=["datalayer"], runner=Solved(), image_transport=_TRANSPORT + ) + assert locked["python_version"] == "3.12" + assert "# python: 3.12\n" in locked["content"] + + def test_the_solve_exports_the_python_it_saw(self): + text = BuildkitResolveRunner(buildctl="/usr/bin/true").dockerfile( + ResolveRequest( + python_version="3.13", + requirements=("ipykernel==7.3.0",), + constraints=(), + indexes=(), + base_reference=FOREIGN, + bootstrap_uv=True, + ) + ) + assert "COPY --from=solve /solve/python-version /python-version" in text + assert '--python-version "$(cat /solve/python-version)"' in text + + def test_pip_is_brought_to_the_python_the_image_kept(self): + text = "\n".join(foreign_bootstrap_lines()) + assert "python3 -m ensurepip --default-pip" in text + assert text.index("python3 -m ensurepip") < text.index("python3-pip") + + def test_an_unset_plane_is_no_plane(self): + assert plane_of({}) is None + assert plane_of({"PLANE_DOCKERFILES": ""}) is None + assert plane_of({"PLANE_DOCKERFILES": "/no/such/plane"}) is None + + +class TestTheRegistries: + def test_the_list_is_what_it_was(self): + assert DEFAULT_ALLOWED_REGISTRIES == ( + "docker.io", + "ghcr.io", + "quay.io", + "gcr.io", + "registry.k8s.io", + "public.ecr.aws", + ) + + def test_a_dockerfile_reaches_only_what_an_image_can(self): + assert findings(dockerfile_source("FROM python:3.12-slim\n")) == [] + refused = findings(dockerfile_source("FROM registry.example.com/team/task:1\n")) + assert "`registry.example.com` is not an allowed registry" in refused[0] + + def test_an_image_from_elsewhere_stays_refused(self): + assert "is not an allowed registry" in findings(image_source("registry.example.com/a:1"))[0] + + @pytest.mark.parametrize( + ("content", "said"), + [ + ("FROM ${IMAGE}\n", "build argument"), + ("FROM a:1 AS one\nFROM b:1 AS two\n", "second base"), + ("FROM datalayer/python-cpu:2026.09\n", "approved base"), + ("FROM python:3.12\nCOPY data.txt /data\n", "build context"), + ("FROM python:3.12\nVOLUME /data\n", "VOLUME is refused"), + ("RUN true\n", "names no base"), + ], + ) + def test_what_cannot_be_built_is_named(self, content, said): + assert said in " ".join(item.message for item in _foreign(content)) + + def test_a_stage_is_not_a_second_base(self): + assert _foreign("FROM python:3.12 AS one\nFROM one AS two\n") == [] + + def test_only_the_datalayer_builder_takes_it(self): + from code_sandboxes.environments.adapters.modal import Builder as Modal + + assert "foreignDockerfile" not in Modal().capabilities().build_sources + + +def _foreign(content): + return foreign_dockerfile_findings(content, registries=DEFAULT_ALLOWED_REGISTRIES) + + +def docker(*arguments, **options): + done = subprocess.run(["docker", *arguments], capture_output=True, text=True, **options) + assert done.returncode == 0, done.stderr[-3000:] + return done.stdout + + +def build(root, tag=None, output=None): + command = ["buildx", "build", "--builder", "default", "--progress", "plain"] + command += ["--output", f"type=local,dest={output}"] if output else ["--load", "-t", tag] + docker(*command, str(root), timeout=900) + + +@pytest.fixture(scope="module") +def approved(tmp_path_factory): + """What the layer copies from: the approved base's own paths, on a small image.""" + root = tmp_path_factory.mktemp("contract-source") + shutil.copytree(WHEELHOUSE_PATH, root / "wheelhouse") + shutil.copy(ENVIRONMENTS / "constraints" / "sandbox-contract-v1.txt", root) + shutil.copy(ENVIRONMENTS / "doctor" / "build.py", root) + shutil.copy(ENVIRONMENTS / "doctor" / "datalayer_sandbox.py", root) + shutil.copy(PLANE / "jupyter-python" / "etc" / "jupyter" / "config" / "jupyter_config.py", root) + shutil.copy(PLANE / "jupyter-python" / "etc" / "jupyter" / "scripts" / "start-jupyter.sh", root) + (root / "Dockerfile").write_text( + "FROM python:3.12-slim\n" + "COPY . /c/\n" + "RUN python3 /c/build.py /datalayer-sandbox\n" + "RUN mkdir -p /opt/datalayer/bin /opt/datalayer/constraints /opt/datalayer/wheelhouse\n" + "RUN cp /datalayer-sandbox /opt/datalayer/bin/ && cp /c/*.txt /opt/datalayer/constraints/ " + "&& cp /c/wheelhouse/*.whl /opt/datalayer/wheelhouse/\n" + "RUN mkdir -p /usr/local/sbin && cp /c/jupyter_config.py /usr/local/share/ " + "&& cp /c/start-jupyter.sh /usr/local/sbin/\n" + "RUN printf 'c.MappingKernelManager.root_dir = \"/home/datalayer\"\\n' " + ">> /usr/local/share/jupyter_config.py\n" + ) + build(root, tag="contract-source:test") + return "contract-source:test" + + +def solve_and_build(approved, tmp_path, spec, base): + """The real solve, then the real build, by the code that renders both.""" + merged = merge_requirements([]) + runner = BuildkitResolveRunner() + solve = tmp_path / "solve" + (solve / "wheelhouse").mkdir(parents=True) + for wheel in WHEELHOUSE_PATH.glob("*.whl"): + shutil.copy(wheel, solve / "wheelhouse") + (solve / "requirements.in").write_text("\n".join(merged.requirements) + "\n") + (solve / "constraints.txt").write_text("\n".join(merged.constraints) + "\n") + request = ResolveRequest( + python_version="3.12", + requirements=tuple(merged.requirements), + constraints=tuple(merged.constraints), + indexes=(), + base_reference=base, + bootstrap_uv=True, + ) + (solve / "Dockerfile").write_text(runner.dockerfile(request).replace("# syntax=", "# ")) + build(solve, output=tmp_path / "out") + solved = (tmp_path / "out" / "python-version").read_text().strip() + locked = lock_document( + ResolveOutcome(lock_text=(tmp_path / "out" / "lock.txt").read_text()), + python_version=solved, + base_reference=base, + merged=merged, + contract_base=approved, + ) + context = tmp_path / "context" + context.mkdir() + (context / "lock.txt").write_text(locked["content"]) + rendered = Builder().dockerfile(request_of(spec, lock=locked["content"], base=base)) + (context / "Dockerfile").write_text(rendered) + tag = f"foreign-layer:{uuid.uuid4().hex[:8]}" + build(context, tag=tag) + return tag + + +KERNEL = """ +import json, os, subprocess, sys, time, urllib.request +base = "http://127.0.0.1:2300/api/jupyter-server/api" +token = {"Authorization": "token t", "Content-Type": "application/json"} +def call(path, data=None): + body = json.dumps(data).encode() if data is not None else None + method = "POST" if body else "GET" + request = urllib.request.Request(base + path, data=body, headers=token, method=method) + return json.load(urllib.request.urlopen(request, timeout=30)) +for _ in range(60): + try: + print(json.dumps(call("/status"))) + break + except Exception as error: + last = error + time.sleep(1) +else: + sys.exit(f"the server never answered: {last}") +print(json.dumps(call("/kernels", {"name": "python3"}))) +""" + + +def served(tag): + """The image's own entrypoint, the doctor beside it, a kernel's answer.""" + name = f"foreign-{uuid.uuid4().hex[:8]}" + docker("run", "-d", "--rm", "--name", name, "-e", "JUPYTER_TOKEN=t", tag) + try: + doctor = subprocess.run( + ["docker", "exec", name, "/opt/datalayer/bin/datalayer-sandbox", "doctor", "--json"], + capture_output=True, + text=True, + timeout=60, + ) + report = json.loads(doctor.stdout) + status = docker("exec", name, "python3", "-c", KERNEL, timeout=120) + code = ( + "import jupyter_client\n" + "client = jupyter_client.BlockingKernelClient()\n" + "from jupyter_client.manager import start_new_kernel\n" + "manager, client = start_new_kernel(kernel_name='python3')\n" + "seen = []\n" + "client.execute_interactive('import os; print(os.getcwd(), 6 * 7)', " + "output_hook=lambda m: seen.append(m['content'].get('text', '')), timeout=60)\n" + "print(''.join(seen))\n" + "manager.shutdown_kernel(now=True)\n" + ) + answered = docker("exec", name, "python3", "-c", code, timeout=120) + return report, status, answered, doctor.returncode + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + +@pytest.mark.skipif(not (REAL and PLANE), reason="needs Docker, the network and the plane") +class TestARealImage: + def _digest(self, image): + docker("pull", "-q", image, timeout=300) + return docker("image", "inspect", "--format", "{{index .RepoDigests 0}}", image).strip() + + def _settled(self, tag): + report, status, answered, code = served(tag) + assert code == 0 and report["ok"] is True, json.dumps(report)[:1500] + assert len(report["rows"]) == 15 and all(row["ok"] for row in report["rows"]) + assert '"started"' in status + assert '"execution_state"' in status + assert "/home/datalayer 42" in answered + return report + + def test_a_python_image_keeps_its_packages_and_is_a_sandbox(self, approved, tmp_path): + base = self._digest("python:3.12-slim") + spec = dockerfile_source( + "FROM python:3.12-slim\nRUN pip install --no-cache-dir cowsay==6.1\n" + "WORKDIR /app\nRUN echo task > /app/file\nUSER nobody\n" + ) + tag = solve_and_build(approved, tmp_path, spec, base) + self._settled(tag) + assert ( + docker("run", "--rm", tag, "python3", "-c", "import cowsay; print('kept')").strip() + == "kept" + ) + assert docker("run", "--rm", tag, "cat", TASK_WORKDIR_FILE).strip() == "/app" + assert docker( + "run", "--rm", tag, "sh", "-c", "id -u; test -w /app/file && echo writable" + ).split() == ["1000", "writable"] + + def test_a_working_directory_above_the_reserved_tree_leaves_it_alone(self, approved, tmp_path): + base = self._digest("python:3.12-slim") + spec = dockerfile_source("FROM python:3.12-slim\nWORKDIR /opt\nRUN echo t > /opt/file\n") + tag = solve_and_build(approved, tmp_path, spec, base) + self._settled(tag) + owners = docker( + "run", "--rm", tag, "stat", "-c", "%U", "/opt", "/opt/datalayer", "/opt/file" + ) + assert owners.split() == ["root", "root", "datalayer"] + + def test_a_python_that_lost_its_pip_gets_it_back(self, approved, tmp_path): + base = self._digest("python:3.12-slim") + spec = dockerfile_source("FROM python:3.12-slim\nRUN python3 -m pip uninstall -y pip\n") + self._settled(solve_and_build(approved, tmp_path, spec, base)) + + def test_a_distribution_python_with_no_pip_gets_its_distributions(self, approved, tmp_path): + base = self._digest("ubuntu:24.04") + spec = dockerfile_source( + "FROM ubuntu:24.04\nRUN apt-get update -qq && apt-get install -y python3\n" + ) + self._settled(solve_and_build(approved, tmp_path, spec, base)) + + def test_an_image_with_no_python_gets_its_distributions(self, approved, tmp_path): + base = self._digest("ubuntu:24.04") + tag = solve_and_build(approved, tmp_path, image_source("ubuntu:24.04"), base) + self._settled(tag) + + def test_an_image_that_is_not_debian_derived_is_said_to_be(self, approved, tmp_path): + base = self._digest("alpine:3.20") + with pytest.raises(AssertionError, match="not Debian-derived"): + solve_and_build(approved, tmp_path, image_source("alpine:3.20"), base) diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 2f8ed21..2e172b5 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -1191,7 +1191,7 @@ def test_an_imported_image_bootstraps_uv_from_its_own_wheelhouse(self) -> None: bootstrap_uv=True, ) ) - assert 'RUN pip install --no-cache-dir "uv==0.12.11"' in dockerfile + assert 'RUN python3 -m pip install --no-cache-dir "uv==0.12.11"' in dockerfile assert "COPY wheelhouse/ ./wheelhouse/" in dockerfile assert "--find-links /solve/wheelhouse" in dockerfile assert "/opt/datalayer/wheelhouse" not in dockerfile