diff --git a/deploy/kubernetes/BETA_CHANGELOG.md b/deploy/kubernetes/BETA_CHANGELOG.md index d5931817f..2abb166bd 100644 --- a/deploy/kubernetes/BETA_CHANGELOG.md +++ b/deploy/kubernetes/BETA_CHANGELOG.md @@ -108,3 +108,16 @@ Verify both rollout revisions and ready Service endpoints, then check public hea Committed hosted Environments and pending inputs notify their existing lifecycle owner. Hints preserve the lease, placement, capacity and one-shot allocation checks, with periodic recovery and one extra scan per maintenance period. The E2B helper adds bounded Create stage timings, and subprocess output collection enforces its byte limit through both copy paths. The [Sandbox Provider guide](../../docs/sandbox-provider.md#per-node-lifecycle-workers) and [E2B helper guide](../../services/core/tools/e2b-provider/README.md#observations) own the behavior and timing limitations. Validation includes database-backed ordinary/stream creation and input recovery without advancing the maintenance clock, concurrent retry and lifecycle regressions, Go race checks, Python helper tests and independent review. Production timing must be verified after rollout; these checks do not establish an end-to-end latency guarantee. + +## 2026-10-08 — E2B initialization round trips + +| Item | Value | +| --- | --- | +| Fork beta baseline | `89f01a50a729d2a0d024aafc193215dd9c2a31cc` | +| Integration branch | `codex/e2b-create-roundtrips` | +| Schema migrations / SQL / protocol changes | None | +| Deployment requirements | Rebuild Core with its packaged E2B helper; existing Runtime template remains compatible | + +Create validates the managed entry point through SDK file information and a streaming open instead of starting a probe process. The initialization command returns its bounded durable receipt, removing the separate ready-file read on the successful path. Ownership and configuration checks remain mandatory; uncertain responses recover through inspection without replaying initialization. The [helper guide](../../services/core/tools/e2b-provider/README.md#create) owns these behaviors. + +Validation includes generated-contract checks, 11 template tests, 261 helper tests, SDK transport fixtures, real local subprocess failure/recovery checks and independent review. These checks do not establish production latency savings; compare Create stages and the full cold Session path after rollout. diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 4233406de..2ebdcefe4 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -42,9 +42,9 @@ A helper holds its allocation's lock until the SDK operation returns, even after 1. Record `create_pending` with the bootstrap identity, then call `Sandbox.create` with the template, the configured timeout, the ownership metadata, `on_timeout=kill` and auto-resume disabled. A definite rejection records a settled `rejected` receipt with no sandbox IDs. 2. Record the sandbox ID and connection material, check the sandbox domain, then read the sandbox by ID and check its ownership metadata, template and resources before writing any credential. A mismatch records a settled rejection and returns `CreateSettled` with the error. -3. Check that `/opt/oac-e2b/managed_init.py` is readable, write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. +3. Check that `/opt/oac-e2b/managed_init.py` has regular-file type through SDK file information, then open it through the SDK streaming file API as root and close the response without downloading its contents. This verifies readability without a remote probe process. Then write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. -`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when a later inspection reads that record with the expected identity; it does not prove enrollment or native readiness. +`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when Core validates that record with the expected identity. The initialization command returns the bounded durable receipt on its existing output stream, avoiding a separate file read on successful Create; Core still rereads cloud ownership and configuration before returning. If the command response is lost or its receipt is unreadable, inspection can recover the record without replaying startup; it does not prove enrollment or native readiness. An unknown Create is never repeated. A Create whose connection material was lost can be discovered and destroyed but cannot resume bootstrap, and an unconfirmed startup requires reclaiming the whole allocation. diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 241c4b934..47c7f46d6 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -9,7 +9,7 @@ from datetime import datetime, timezone from uuid import UUID -from e2b import Sandbox, SandboxQuery, SandboxState +from e2b import FileType, Sandbox, SandboxQuery, SandboxState from e2b.api.client.models.sandbox_metric import SandboxMetric from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException @@ -21,6 +21,20 @@ PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +# Execute the existing protected entry point and return its durable receipt on +# the same command stream. A read failure leaves successful startup recoverable +# through Inspect, without replaying initialization. +BOOTSTRAP_SCRIPT = """import runpy,sys +runpy.run_path('/opt/oac-e2b/managed_init.py', run_name='__main__') +try: + with open('/root/.oac/e2b/managed-ready.json', 'rb') as source: + receipt = source.read(4097) + if len(receipt) <= 4096: + sys.stdout.buffer.write(receipt) +except OSError: + pass +""" + @contextmanager def create_stage(stage): @@ -217,13 +231,17 @@ def inspect(self): except FileNotFoundException: receipt = None if receipt is not None: - expected = record.get('bootstrap_identity') - if (receipt.get('identity') != expected or receipt.get('status') != 'daemon_started' or - type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): - raise Failure('ownership') - self.receipt.save(settled=True, bootstrap_complete=True) + self.accept_bootstrap_receipt(receipt) return cloud + def accept_bootstrap_receipt(self, receipt): + expected = (self.receipt.data or {}).get('bootstrap_identity') + if (not isinstance(receipt, dict) or receipt.get('identity') != expected or + receipt.get('status') != 'daemon_started' or + type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): + raise Failure('ownership') + self.receipt.save(settled=True, bootstrap_complete=True) + def create(self): if self.receipt.data is not None: raise Failure('exists') @@ -259,12 +277,20 @@ def create(self): raise # Validate the current template entry point before writing any credential. with create_stage('template_check'): - check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', - "import os,sys; sys.exit(78 if not os.path.isfile('/opt/oac-e2b/managed_init.py') or not os.access('/opt/oac-e2b/managed_init.py', os.R_OK) else 0)"]}, - self.remaining, user='root') - if check['ExitCode'] != 0: + try: + entry = cloud.files.get_info('/opt/oac-e2b/managed_init.py', + user='root', request_timeout=self.remaining()) + if entry.type != FileType.FILE: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('template_invalid') + # A successful download can also refer to a special file. Check + # the type first, then verify readability without a probe process. + with cloud.files.read('/opt/oac-e2b/managed_init.py', format='stream', + user='root', request_timeout=self.remaining()): + pass + except FileNotFoundException: self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('template_invalid' if check['ExitCode'] == 78 else 'unconfirmed') + raise Failure('template_invalid') from None payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) del payload['CoreURL'], payload['Credential'], payload['Harness'] @@ -275,13 +301,18 @@ def create(self): user='root', request_timeout=self.remaining()) self.receipt.save(status='bootstrap_pending') with create_stage('bootstrap_run'): - result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '/opt/oac-e2b/managed_init.py']}, + result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]}, self.remaining, user='root') if result['ExitCode'] != 0: self.receipt.save(status='bootstrap_failed', settled=True) raise Failure('unconfirmed') self.receipt.save(status='bootstrap_exited', settled=True) with create_stage('ready_inspect'): + try: + receipt = json.loads(result['Stdout']) + except (ValueError, KeyError): + raise Failure('unconfirmed') from None + self.accept_bootstrap_receipt(receipt) return self.inspect() def renew(self): diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index feff8f1ba..bd4a63205 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -1,24 +1,63 @@ """Controlled SDK boundary failures; live qualification remains separate.""" import copy import io +import httpx from datetime import datetime, timedelta, timezone import json from pathlib import Path import tempfile +import subprocess +import sys from types import SimpleNamespace import unittest -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch from uuid import uuid4 -from e2b import SandboxState -from e2b.exceptions import AuthenticationException, SandboxNotFoundException +from e2b import FileType, SandboxState +from e2b.connection_config import ConnectionConfig +from e2b.sandbox_sync.filesystem.filesystem import Filesystem +from packaging.version import Version +from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException -from provider import Provider, create_stage +from provider import BOOTSTRAP_SCRIPT, Provider, create_stage from helper_contract_generated import PROTOCOL_VERSION from sdk import restore, run from state import Failure, Receipt +class BootstrapScriptTest(unittest.TestCase): + def execute(self, startup, receipt=None): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + entry = root / 'managed_init.py' + ready = root / 'managed-ready.json' + entry.write_text(startup.replace('READY_PATH', repr(str(ready)))) + if receipt is not None: + ready.write_bytes(receipt) + script = BOOTSTRAP_SCRIPT.replace('/opt/oac-e2b/managed_init.py', str(entry)).replace( + '/root/.oac/e2b/managed-ready.json', str(ready)) + return subprocess.run([sys.executable, '-I', '-c', script], + capture_output=True, timeout=5) + + def test_success_reads_receipt_after_initialization(self): + result = self.execute("from pathlib import Path; Path(READY_PATH).write_bytes(b'new receipt')", + receipt=b'stale receipt') + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'new receipt') + + def test_failed_initialization_cannot_return_old_receipt(self): + result = self.execute('raise SystemExit(7)', receipt=b'stale receipt') + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, b'') + + def test_missing_and_oversized_receipts_leave_startup_recoverable(self): + for receipt in (None, b'x' * 4097): + with self.subTest(size=len(receipt) if receipt else None): + result = self.execute('pass', receipt=receipt) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'') + + class ProviderTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -42,7 +81,12 @@ def setUp(self): self.identity = dict(self.reference, InstallationID=self.config['InstallationID'], SessionID=self.request['Bootstrap']['SessionID'], DeviceID=self.request['Bootstrap']['DeviceID']) self.ready = json.dumps({'identity': self.identity, 'status': 'daemon_started', 'daemon_pid': 123}) + self.cloud.files.get_info.return_value = SimpleNamespace(type=FileType.FILE) + self.template_stream = MagicMock() self.cloud.files.read.return_value = self.ready + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + self.template_stream if path == '/opt/oac-e2b/managed_init.py' + else self.cloud.files.read.return_value) self.api = Mock() self.api.create.return_value = self.cloud self.api.get_info.return_value = self.cloud @@ -53,7 +97,8 @@ def setUp(self): self.runtime = patch('provider.restore', return_value=self.cloud) self.runtime.start() self.addCleanup(self.runtime.stop) - self.command = patch('provider.run', return_value={'Stdout': '', 'Stderr': '', 'ExitCode': 0}) + self.command = patch('provider.run', side_effect=lambda *a, **k: { + 'Stdout': self.cloud.files.read.return_value, 'Stderr': '', 'ExitCode': 0}) self.command.start() self.addCleanup(self.command.stop) @@ -150,6 +195,7 @@ def test_create_refuses_other_owner_before_credentials(self): self.assertEqual(self.call('create')['ErrorCode'], 'ownership') self.api.get_info.assert_called_once() self.cloud.files.write.assert_not_called() + self.cloud.files.read.assert_not_called() def test_qualified_gateway_template_id_accepts_only_selected_build(self): self.config['Resources'] = {'cpus': 2, 'memory_mib': 2048} @@ -187,7 +233,7 @@ def test_create_refuses_foreign_data_plane_before_envd(self): self.cloud.commands.run.assert_not_called() def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(self): - with patch('provider.run', return_value={'ExitCode': 78, 'Stdout': '', 'Stderr': ''}): + with patch.object(self.cloud.files, 'read', side_effect=FileNotFoundException('missing')): result = self.call('create') self.assertEqual(result['ErrorCode'], 'template_invalid') self.assertTrue(result['Info']['CreateSettled']) @@ -197,6 +243,75 @@ def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(s self.api.kill.assert_not_called() self.assertEqual(self.call('create')['ErrorCode'], 'exists') + def test_template_non_regular_files_are_rejected_before_open_or_credentials(self): + for kind in (FileType.DIR, None): + with self.subTest(kind=kind): + self.reference['AllocationID'] = str(uuid4()) + self.request['Bootstrap']['AllocationID'] = self.reference['AllocationID'] + self.cloud.metadata = Provider(self.request).metadata + self.cloud.files.get_info.return_value = SimpleNamespace(type=kind) + self.assertEqual(self.call('create')['ErrorCode'], 'template_invalid') + self.cloud.files.read.assert_not_called() + self.cloud.files.write.assert_not_called() + + def test_template_read_is_closed_and_does_not_launch_a_probe(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': self.ready}) as command: + self.assertEqual(self.call('create')['ErrorCode'], '') + self.template_stream.__enter__.assert_called_once() + self.template_stream.__exit__.assert_called_once() + self.template_stream.__iter__.assert_not_called() + command.assert_called_once() + self.assertEqual(command.call_args.args[1]['Args'], + ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]) + args, options = self.cloud.files.read.call_args_list[0] + self.assertEqual(args, ('/opt/oac-e2b/managed_init.py',)) + self.assertEqual(options['format'], 'stream') + self.assertEqual(options['user'], 'root') + self.assertGreater(options['request_timeout'], 0) + + def test_template_probe_uses_sdk_stream_and_closes_without_downloading(self): + class Body(httpx.SyncByteStream): + closed = False + + def __iter__(self): + raise AssertionError('template contents must not be downloaded') + yield b'' + + def close(self): + self.closed = True + + body = Body() + requests = [] + + def respond(request): + requests.append(request) + return httpx.Response(200, stream=body) + + with httpx.Client(base_url='https://fixture.invalid', + transport=httpx.MockTransport(respond)) as client: + with patch('e2b.sandbox_sync.filesystem.filesystem.get_envd_api', return_value=client), \ + patch('e2b.sandbox_sync.filesystem.filesystem.create_rpc_client'): + files = Filesystem('https://fixture.invalid', Version('0.5.0'), + ConnectionConfig(api_key='fixture'), client) + original_read = self.cloud.files.read.side_effect + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + files.read(path, **kwargs) if path == '/opt/oac-e2b/managed_init.py' + else original_read(path, **kwargs)) + self.assertEqual(self.call('create')['ErrorCode'], '') + self.assertTrue(body.closed) + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0].method, 'GET') + self.assertEqual(requests[0].url.params['path'], '/opt/oac-e2b/managed_init.py') + self.assertEqual(requests[0].url.params['username'], 'root') + + def test_uncertain_template_read_never_writes_credentials_or_replays_create(self): + with patch.object(self.cloud.files, 'read', side_effect=TimeoutError('private secret')): + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + self.cloud.files.write.assert_not_called() + self.assertFalse(self.record().get('bootstrap_complete', False)) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + self.api.create.assert_called_once() + def test_unknown_create_empty_lookup_never_proves_cleanup(self): self.api.create.side_effect = TimeoutError('confidential SDK diagnostic') self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') @@ -281,6 +396,23 @@ def test_mismatched_receipt_cannot_prove_bootstrap_complete(self): self.assertTrue(self.record()['settled']) self.assertFalse(self.record()['bootstrap_complete']) + def test_successful_create_returns_receipt_without_remote_receipt_read(self): + self.assertTrue(self.call('create')['Info']['BootstrapComplete']) + self.assertEqual([call.args[0] for call in self.cloud.files.read.call_args_list], + ['/opt/oac-e2b/managed_init.py']) + self.assertEqual(self.api.get_info.call_count, 2) + + def test_lost_command_receipt_recovers_from_file_without_replaying_startup(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': ''}) as command: + result = self.call('create') + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertTrue(result['Info']['CreateSettled']) + self.assertFalse(result['Info']['BootstrapComplete']) + self.assertTrue(self.call('inspect')['Info']['BootstrapComplete']) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + command.assert_called_once() + self.api.create.assert_called_once() + def test_foreign_owner_prevents_renew_and_delete(self): self.call('create') self.cloud.metadata = {}